mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-07 10:45:59 -10:00
Make turn termination append-only and fence late writes
Problem: Stopping a turn and reopening another conversation could make durable history unreadable. Root cause: Cancellation replaced display history through divergent strict decoders and reset message-version watermarks; asynchronous terminal paths also lacked one durable acceptance boundary. Fix: Share event payload validation, preserve version watermarks, add required message retraction with revision-2 writer migration, and unify cleanup/recovery through atomic turn finalization. Fence stale generations and synchronize history, search, catalog and fork projections without rewriting pause history. Verification: Focused deterministic and race regressions, provider HTTP byte comparisons, root and Desktop module suites, frontend transcript/browser tests, old-binary compatibility probes, and macOS/Windows ARM64 portable production-mode pause/switch/reopen workflows passed. Final reply-anchor refinement passed its owning session suite. Installer lifecycle and Windows x64 hardware remain unverified; see docs/session-termination.md.
This commit is contained in:
1 parent
b21deef03e
commit
c3694a0e07
37 files changed
+3021
-265
No files matched your search
@@ -67,6 +67,34 @@ func TestDesktopHistorySliceUsesCanonicalDurableIndex(t *testing.T) {
|
||||
if stale := app.HistoryContentForTab(tab.ID, assistant.Refs[0], 0); !stale.Stale {
|
||||
t.Fatal("content ref from older durable snapshot must become stale after append")
|
||||
}
|
||||
// Stop/recovery keeps message identities while rewriting their transcript.
|
||||
// All desktop readers must accept the runtime's reason metadata and index
|
||||
// new versions without colliding with the previously displayed messages.
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"reason": "cancel-or-recovery-rewrite",
|
||||
"messages": []provider.Message{
|
||||
{ID: "history-user", Role: provider.RoleUser, Origin: provider.MessageOriginUser, Content: "durable user turn"},
|
||||
{ID: "history-assistant", Role: provider.RoleAssistant, Content: large},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runtime.Session().Append(t.Context(), session.Batch{OperationID: "cancel-rewrite", Events: []session.Event{{Kind: "history/replace", Payload: payload}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runtime.Session().Flush(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if page := app.HistorySliceForTab(tab.ID, HistorySliceRequest{Turns: 12}); page.Error != "" || len(page.Entries) != 2 {
|
||||
t.Fatalf("compatibility reader after cancel rewrite: %+v", page)
|
||||
}
|
||||
if page, err := app.SessionHistoryWindowForTab(tab.ID, session.HistoryWindowRequest{Anchor: "newest"}); err != nil || page.Status != "ready" || len(page.Messages) != 2 {
|
||||
t.Fatalf("window reader after cancel rewrite: %+v, %v", page, err)
|
||||
}
|
||||
if page, err := app.SessionHistoryPageForTab(tab.ID, "", 10); err != nil || page.Status != "ready" || len(page.Messages) != 2 {
|
||||
t.Fatalf("page reader after cancel rewrite: %+v, %v", page, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDesktopCanonicalHistoryRemainsReadableBeforeControllerReady(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
# History loading fails after Stop / 暂停后历史加载失败
|
||||
|
||||
This document records the original reader repair baseline. The subsequent
|
||||
append-only termination and revision-2 migration are documented in
|
||||
[Session termination](session-termination.md); its upgrade rules supersede the
|
||||
baseline compatibility table below.
|
||||
|
||||
本文记录最初的读取器修复基线。后续追加式终结和 revision 2 升级见
|
||||
[暂停与恢复机制](session-termination.md),升级兼容规则以新文档为准。
|
||||
|
||||
## Confirmed mechanism / 已确认机制
|
||||
|
||||
Stopping a turn can call `Controller.replaceSessionAfterCancel`, which records
|
||||
`history/replace` with the existing message IDs and `reason` metadata. Opening a
|
||||
session subsequently reads that event through the history index. Two independent
|
||||
reader defects made a valid durable transcript unreadable:
|
||||
|
||||
1. Execution/recovery accepted `reason` and `sourceSequences`, but the history
|
||||
and search decoders only declared `messages`. Strict decoding rejected the
|
||||
valid event with `json: unknown field "reason"`. Legacy imports had the same
|
||||
discrepancy for `source`, `goal`, `modelRef`, and `modelIdentity`.
|
||||
2. History replacement retired previous message rows but reset their version
|
||||
counters. Reusing the same ID inserted version 1 again and failed SQLite's
|
||||
`(message_id, version)` primary key. Both incremental indexing and rebuilding
|
||||
from the log failed. Loading only current-row versions also broke restoration
|
||||
of a message removed by an earlier replacement.
|
||||
|
||||
暂停回合可能调用 `Controller.replaceSessionAfterCancel`,写入保留原消息 ID、
|
||||
附带 `reason` 的 `history/replace` 事件。随后打开会话会通过历史索引读取此事件。
|
||||
两个独立的读取缺陷使合法记录无法显示:
|
||||
|
||||
1. 执行和恢复层接受 `reason`、`sourceSequences`,历史和搜索解码器却仅声明
|
||||
`messages`,严格解码直接报 `json: unknown field "reason"`。旧会话导入的
|
||||
`source`、`goal`、`modelRef`、`modelIdentity` 也存在同类字段不一致。
|
||||
2. 历史替换保留旧数据库行,却把版本计数清零;相同 ID 再写版本 1,触发
|
||||
`(message_id, version)` 主键冲突,增量更新和完整重建均失败。另外,只读取
|
||||
当前有效行的版本,也会让先删除、后恢复的消息发生冲突。
|
||||
|
||||
The fix shares the existing event schemas across execution, recovery, history,
|
||||
and search. History versions retain their high-water marks across replacement
|
||||
and are restored from all indexed versions, including retired identities.
|
||||
Unknown fields and missing/null message lists still fail validation.
|
||||
|
||||
修复复用已有事件字段定义,使执行、恢复、历史和搜索一致;替换时保留消息版本
|
||||
水位,增量读取时恢复全部身份的最大版本,包括已退出当前历史的消息。
|
||||
未知字段、缺失或为 null 的消息列表仍会被拒绝。
|
||||
|
||||
## Evidence and limits / 证据与边界
|
||||
|
||||
- Base checkout: `b21deef03eb859c2e923f1f6dd8a8ee1b0d4493e`.
|
||||
- Submitted diagnostics identify preview build `6e4617121fd4`. The real
|
||||
Stop → switch away → read sequence fails on that exact unmodified production
|
||||
code with `unknown field "reason"`; the same regression passes with this fix.
|
||||
- The recorder contains 1,521 events over 120,005 ms, including three
|
||||
`navigation.settle` / `data-failed` results and one `paint-ready` success.
|
||||
It contains no backend exception text, so it cannot independently establish
|
||||
the failing event or session for every reported incident.
|
||||
- The submitted report's index/log watermark differences are observations,
|
||||
not proof that every cache must update at each turn end. These indexes are
|
||||
derived data and can legitimately lag. The confirmed decoder/primary-key
|
||||
failures explain why a subsequent index read can repeatedly fail to catch up.
|
||||
- The v5 workspace registry owns session membership; fewer records in legacy
|
||||
topic tables do not by themselves prove missing conversations. Lease warnings
|
||||
and concurrent-version damage are not established causes by these artifacts.
|
||||
- The older base also treats history `preparing` as a failure; the submitted
|
||||
preview already has preparation waiting. No frontend retry change is included
|
||||
in this repair of the durable event readers.
|
||||
|
||||
当前基线为上述 `b21deef…`。在诊断指定的原始预览版 `6e4617121fd4` 上,真实
|
||||
“暂停 → 切走 → 读取”回归同样报 `unknown field "reason"`;应用本修复后通过。
|
||||
录制数据有三次 `data-failed`、一次 `paint-ready`,但没有后端异常内容,不能
|
||||
据此断言所有用户每一次失败都只有此原因。报告中的索引落后可作为现象,不能
|
||||
单独证明每回合结束时必须同步所有缓存。旧 topic 数量也不能替代 v5 workspace
|
||||
注册表判断会话是否丢失;租约告警、不同版本混用尚无直接因果证据。
|
||||
较老基线的 `preparing` 误报在用户预览版中已有等待处理,本修复未叠加前端重试。
|
||||
|
||||
## Compatibility / 兼容性
|
||||
|
||||
| Surface / 范围 | Behavior / 行为 |
|
||||
| --- | --- |
|
||||
| Durable events / 持久事件 | Existing fields and bytes unchanged; readers accept the already valid schema. / 字段与写入字节不变,读取已有合法格式。 |
|
||||
| SQLite history / 历史缓存 | Same schema and primary key; monotonically increasing versions. Failed index transactions roll back and can be retried by the fixed reader. / 表结构及主键不变,版本递增,失败事务回滚后可由修复版继续读取。 |
|
||||
| Earlier readers / 旧读取器 | Still contain these defects; a read fix does not repair an old executable. / 旧程序仍有本缺陷,不能靠新版读取修复旧程序。 |
|
||||
| Provider and RPC / 模型与接口 | No prompt, provider payload, RPC shape, or session identity change. / 不改提示词、模型请求、RPC 结构或会话身份。 |
|
||||
|
||||
All reproductions use disposable sessions. No submitted user data was modified.
|
||||
Tests cover repeated rewrites, removed/restored identities, search, imported
|
||||
metadata, cached restart, rebuild after restart, desktop page/window/compatibility
|
||||
readers, and Stop → switch → return → send. Subsequent Windows ARM64 packaged
|
||||
execution is recorded in [Session termination](session-termination.md). A retest
|
||||
of the reporters' original session logs remains external verification.
|
||||
|
||||
复现全部使用临时会话,未修改用户资料。回归覆盖重复重写、删除后恢复消息、
|
||||
搜索、导入元数据、保留缓存重启、重启后重建索引、Desktop 三种历史读取接口,
|
||||
以及暂停后切走、切回、再发送。后续 Windows ARM64 包运行验证见暂停与恢复文档;
|
||||
用户原始会话日志复测仍待外部验证。
|
||||
|
||||
## Local validation / 本地验证
|
||||
|
||||
- Passed: session/control regression tests on both the base checkout and the
|
||||
submitted preview commit with the patch; focused Go race tests; desktop
|
||||
session switch/cancel/history reader tests; `pnpm test:transcript`;
|
||||
`pnpm test:transcript-browser` (Chromium); repository lint and diff checks.
|
||||
- `go test ./...` passed the session, control, and other reported packages except
|
||||
`internal/cli`: three WSL clipboard fixture subprocesses exceeded their 5-second
|
||||
deadlines. The exact clipboard suite passed independently on both checkouts.
|
||||
The aggregate run is therefore not reported as green; no timeout or test was
|
||||
relaxed for this change.
|
||||
|
||||
两个代码基线应用补丁后的定向回归、Go 竞态测试、Desktop 会话切换/取消/历史读取、
|
||||
前端 transcript、Chromium 浏览器测试及仓库静态检查均通过。全仓 Go 检查中,
|
||||
`internal/cli` 的三个 WSL 剪贴板子进程超过 5 秒限制;该组测试在两个基线独立
|
||||
运行均通过,因此保留全仓首次检查未全绿的记录,未放宽超时或测试。
|
||||
@@ -0,0 +1,191 @@
|
||||
# Session termination / 暂停与恢复机制
|
||||
|
||||
## Contract / 契约
|
||||
|
||||
Reasonix keeps the Go runtime, v4 physical log, SQLite projections and Desktop
|
||||
Stop RPC. Cancellation now stages explicit message mutations and accepts them
|
||||
with the terminal facts in one atomic commit. `history/replace` remains available
|
||||
for import and intentional history edits, but is not the cancellation commit.
|
||||
`model/context-replace` intentionally retains the exact provider workset.
|
||||
|
||||
保留 Go 运行时、v4 物理日志、SQLite 投影和 Desktop Stop RPC。暂停清理先生成明确
|
||||
消息变更,再与回合终结事实一起原子提交;导入和显式编辑仍可使用 `history/replace`。
|
||||
暂停不替换整段展示历史,`model/context-replace` 继续保存精确模型工作集。
|
||||
|
||||
| Owner / 所有者 | Responsibility / 职责 |
|
||||
| --- | --- |
|
||||
| `internal/control/termination_policy.go` | Pure legacy cleanup policy, including fallback input, complete tool pairs, LocalOnly and compaction. / 无副作用的既有清理策略,包括真实输入兜底、完整工具对、LocalOnly 和摘要。 |
|
||||
| `internal/control/termination.go` | Detached plan, turn identity, atomic terminal acceptance and bounded Flush. / 独立计划快照、回合身份、原子终结和有界持久化。 |
|
||||
| `internal/session/recovery.go` | Deterministic tool/interaction/step closure shared with restart recovery; never reruns a tool. / 运行终结与重启共享确定性收尾,不重跑工具。 |
|
||||
| `internal/session/transcript_metadata.go` | Bounded input previews and stable identities survive externalized history; retractions repair catalog and reply anchors. / 历史外置后保留输入身份及截断预览,撤回同步修复目录及回复锚点。 |
|
||||
|
||||
The input-preview metadata grows with input identities, not message bodies.
|
||||
History pages and recent-message windows remain bounded. Synthetic cleanup also
|
||||
tracks the active turn's committed IDs, so compaction cannot hide messages that
|
||||
must be retracted; absence from the model working set does not authorize deleting
|
||||
older display history. Unresolved side-effect records use the existing
|
||||
`Session.Replace` retention policy before IDs are assigned and events prepared.
|
||||
|
||||
输入预览元数据随输入身份数量增长,不保留完整消息正文;分页和最近消息窗口保持
|
||||
有界。合成回合还按当前回合已提交的 ID 撤回,避免压缩移走模型上下文后漏删,也
|
||||
不会因旧消息不在模型窗口中而删除旧历史。未决副作用记录在分配稳定 ID、生成事件
|
||||
之前应用已有 `Session.Replace` 保留规则,避免提交后产生额外临时消息。
|
||||
|
||||
## Events and ordering / 事件与顺序
|
||||
|
||||
`message/retract` is required. Its strict payload is
|
||||
`{"messageIds":["stable-id"],"reason":"diagnostic-only"}`. IDs must be nonempty,
|
||||
trimmed and unique. Writers sort cleanup IDs. A missing/already retracted ID is a
|
||||
projection no-op. Later upsert restores the ID at a higher version; raw events and
|
||||
content blobs are retained. Unknown fields remain errors.
|
||||
|
||||
`message/retract` 为 required 事件。严格载荷如上,ID 非空、无重复且无首尾空白;
|
||||
清理写入按 ID 排序。投影对不在当前视图或已撤回的 ID 不操作;同 ID 可由后续
|
||||
upsert 以更高版本恢复。原始事件和正文不物理删除,未知字段仍拒绝。
|
||||
|
||||
Normal termination order is cancellation signal, worker exit, planning, terminal
|
||||
gate, atomic acceptance, executor synchronization, independent 15-second Flush,
|
||||
then completion publication. The existing cooperative cancellation grace remains
|
||||
15 seconds. Within the batch the order is message changes, model context, tool /
|
||||
interaction / step closure, recovery state, then turn end.
|
||||
|
||||
正常顺序为立即发取消信号、等待协作退出、生成计划、进入终结门闩、原子接受、
|
||||
同步 executor、独立 15 秒 Flush、发布完成;协作取消宽限期仍为 15 秒。Batch 内
|
||||
顺序为消息变更、模型上下文、工具/交互/步骤收尾、恢复状态、回合结束。
|
||||
|
||||
`turn-finalize:<turnID>` identifies the unique terminal operation. The runtime
|
||||
checks execution generation and open turn while holding its commit gate. The
|
||||
controller additionally checks the captured execution token. Accepted terminal
|
||||
retries only flush; persistence failure keeps `recovery_required` and disarms
|
||||
automatic Goal continuation. Repairs after a closed turn use a deterministic
|
||||
`turn-repair:<payload hash>` operation and never add another turn end.
|
||||
|
||||
唯一终结键为 `turn-finalize:<turnID>`。runtime 在提交门闩内检查执行代际及开放
|
||||
回合,controller 还检查执行 token;已接受终结只重试 Flush。持久化失败保持
|
||||
`recovery_required` 并停止 Goal 自动续跑。已关闭回合的修复使用确定性的
|
||||
`turn-repair:<载荷哈希>`,不重复追加 `turn/end`。
|
||||
|
||||
Watchdog uses accepted model state, captured input/prefix identities and accepted
|
||||
compaction summaries. Sealing rejects late business output; it does not prove a
|
||||
tool process exited. Started tools without a result remain unknown; dispatched
|
||||
tools without start evidence are not-started. Existing results are not duplicated.
|
||||
Stop receipts acknowledge acceptance only, and switching another session does
|
||||
not wait for this session's disk cleanup. Old callbacks retain their session and
|
||||
turn identity. No polling service or provider-visible diagnostic text is added.
|
||||
|
||||
watchdog 根据已接受模型状态、捕获的输入/前缀身份和摘要封存。封存拒绝迟到业务
|
||||
输出,但不代表工具进程已退出:已启动无结果记 unknown,仅派发无启动证据记
|
||||
not-started;已有结果不补写。Stop receipt 仅确认请求已接受,切换其他会话不等待
|
||||
本会话磁盘清理。旧回调保留原会话/回合身份,不新增轮询或模型可见诊断文字。
|
||||
|
||||
## Upgrade and rollback / 升级与回退
|
||||
|
||||
Physical codec stays v4; new sessions use storage revision 2. Readers accept
|
||||
revisions 1 and 2. A revision-1 writable open validates the log and acquires the
|
||||
exclusive lease before atomically publishing and syncing revision 2. It does not
|
||||
rewrite the event log. Read-only open neither upgrades nor repairs. Fault tests
|
||||
cover rejected leases, invalid logs and failed atomic manifest publication.
|
||||
|
||||
物理 codec 仍为 v4,新会话 revision 2;新版读取 revision 1、2。旧会话可写打开
|
||||
在日志验证和独占租约成立后原子同步新版 manifest,不改原日志;只读打开不升级、
|
||||
不写修复。故障测试覆盖租约冲突、损坏日志和 manifest 原子发布失败。
|
||||
|
||||
History projection is version 6, search version 2, recovery projection version 3,
|
||||
catalog metadata version 2. Old derived caches rebuild on demand. Retraction
|
||||
expires MVCC rows without resetting version watermarks; changed visible-turn
|
||||
ordinals get new rows so previous snapshot cursors keep their previous view.
|
||||
Retracted inputs and replies disappear from current catalog/fork navigation.
|
||||
|
||||
历史投影版本 6、搜索版本 2、恢复投影版本 3、目录元数据版本 2;旧缓存按需重建。
|
||||
撤回使 MVCC 当前行过期,不重置版本水位;变化的回合序号写新版本行,旧快照仍可读。
|
||||
撤回输入与回复后,当前目录和 fork 导航同步更新。
|
||||
|
||||
**Do not lower manifest revisions to roll back.** An old executable cannot safely
|
||||
read revision 2. A code rollback must retain revision-2 decoding and projections,
|
||||
or restore a complete pre-upgrade backup into a separate location. Do not mix an
|
||||
old manifest with a new event log.
|
||||
|
||||
**禁止降低 manifest 版本号实现降级。** 旧程序不能安全读取 revision 2。代码回退
|
||||
必须保留新版解码和投影,或将完整升级前备份恢复到独立位置;不得混用旧 manifest
|
||||
和新事件日志。
|
||||
|
||||
## Verification record / 验证记录
|
||||
|
||||
Baseline: `b21deef03eb859c2e923f1f6dd8a8ee1b0d4493e`, with the existing decoder and
|
||||
message-version repair retained. Packages were built from the implementation
|
||||
working tree before its PR commit; they are local verification artifacts.
|
||||
The local verification date is 2026-09-16, macOS arm64.
|
||||
|
||||
基线为上述提交,保留已有解码器与消息版本修复。验证包从 PR 提交前的实现工作树
|
||||
构建,仅作为本地验证产物。
|
||||
本地验证日期为 2026-09-16,平台为 macOS arm64。
|
||||
|
||||
| Evidence / 证据 | Result / 结果 |
|
||||
| --- | --- |
|
||||
| Policy and HTTP bytes / 策略及 HTTP 字节 | Seven fixtures through OpenAI-compatible, Anthropic thinking/signature and Responses adapters compare expected cleanup, new cleanup and persisted reopen requests byte-for-byte. / 七组夹具经三个真实 HTTP adapter 比较原预期、新清理及重开请求,逐字节一致。 |
|
||||
| Deterministic termination / 确定性终结 | Double Stop, watchdog late output, accepted-before-fsync failure, fallback input, synthetic compaction, side-effect recovery retention and zero pause history replacements pass. / 连续 Stop、watchdog 迟到输出、接受后 fsync 失败、输入兜底、合成压缩、副作用恢复记录和零暂停历史替换通过。 |
|
||||
| Projections / 投影 | History/search, snapshot MVCC, ID restoration, catalog/recent, fork targets, reply repair, checkpoint and cache-free reopen covered. / 历史搜索、快照、恢复 ID、目录、最近消息、fork、回复修复、检查点和无缓存重开均覆盖。 |
|
||||
| Old binary / 旧二进制 | Built from baseline: lease held prevents upgrade; after upgrade warm history, cold read and write return unsupported storage version. / 从基线编译:旧租约阻止升级;升级后旧版热缓存、冷读、写入均报不支持版本。 |
|
||||
| Frontend / 前端 | typecheck, transcript tests and Chromium transcript browser suite passed; no render errors, duplicate nodes or anchor drift. / 类型检查、transcript 单测和 Chromium 浏览器套件通过。 |
|
||||
| Full root module / 根模块全量 | Latest full invocation passed (control 208.632s, session 113.589s, agent 161.773s, CLI 130.414s). A previous full run failed at the known WSL clipboard timeout; its isolated pass was not counted as a full-suite pass. The final reply-anchor refinement also passed its focused session suite. / 最近全量通过,耗时如左;此前一轮已知 WSL 剪贴板超时及单独重跑记录仍保留,不以单独通过代替全量。最后的回复锚点细化另通过 session 定向套件。 |
|
||||
| Desktop module / Desktop 模块 | Independent `go test ./...` passed (277.720s); cancellation/cutover/recovery race suite passed (49.662s). The reversed compatibility-message order found in earlier runs was fixed before these passes. / 独立模块全量与相关 race 通过;此前发现的兼容消息反序问题已修复。 |
|
||||
| Runtime races / 运行时并发 | session/control targeted race passed; the final watchdog-summary and termination race suite passed (11.918s). / session/control 定向 race 通过,最终 watchdog 摘要及终结 race 通过。 |
|
||||
| Native macOS / macOS 原生 | Production-mode local ZIP exercised renderer→service Stop, old/new session switches, return, continuation, restart and clean exit using a local HTTP fixture. No real vendor credentials used. / 生产模式本地 ZIP 通过 renderer→service 暂停、旧/新会话切换、返回、继续、重启和正常退出,使用本地 HTTP 夹具,无真实厂商凭据。 |
|
||||
| Windows ARM64 / Windows ARM64 | Passed on the local Parallels Windows 11 ARM64 VM (10.0.26200): native session tests 37.118s, control tests 23.632s, six Desktop regressions 7.630s. The complete portable ZIP passed production-mode renderer/service Stop → old/new conversation → return → continue → restart, unique transcript record IDs, visible reopened content and normal shell/service exit. / 已在本机 Parallels Win11 ARM64(10.0.26200)通过原生 session、control 和六个 Desktop 定向回归;完整便携 ZIP 通过生产模式暂停→旧/新会话→返回→继续→重启,记录 ID 无重复、重开内容可见,壳及服务正常退出。 |
|
||||
|
||||
The local packages are verification evidence, not signed/notarized production
|
||||
releases. Native runtime reproduction is now covered on macOS ARM64 and Windows
|
||||
11 ARM64; Windows x64 hardware was not exercised. Check the task's final result
|
||||
for module runs and package identities; earlier interrupted/failed attempts are
|
||||
not counted as passes.
|
||||
|
||||
本地包仅作为验证证据,不是正式签名/公证发布。现已覆盖 macOS ARM64 与
|
||||
Windows 11 ARM64 的原生运行复现,未测试 Windows x64 硬件。最终模块运行结果
|
||||
及包身份以任务最终结果为准;中途失败或中断不算通过。
|
||||
|
||||
Final tested ZIP: `dist/Reasonix-darwin-arm64.zip`, version
|
||||
`v0.0.0-termination-local`, source revision `b21deef…+dirty`, SHA-256
|
||||
`a3640d502d784194a95e1b96c27eb0ae9f39212eead5698b638e3cb9a6aecd55`.
|
||||
It was extracted into a fresh directory and passed strict deep codesign
|
||||
verification before the production-mode renderer/service scenario. The DMG was
|
||||
built, but this record qualifies the ZIP, not a separate DMG install run.
|
||||
|
||||
最终验证对象为上述版本和哈希的 ZIP(包含未提交改动),重新解压到干净目录,
|
||||
通过严格深度签名结构检查后执行生产模式 renderer/service 流程。DMG 已构建,
|
||||
但没有把 ZIP 验证冒充为另一次 DMG 安装验证。
|
||||
|
||||
Windows follow-up (2026-09-16): all changed source files were SHA-256 checked
|
||||
against the macOS working tree before the VM build. The repository's complete
|
||||
`desktop-build.sh windows/arm64` flow produced both the NSIS installer and the
|
||||
portable archive. The initial NSIS permission failure was resolved by copying
|
||||
the system-owned toolkit into the dedicated test directory; product code and
|
||||
the original toolkit permissions were unchanged.
|
||||
|
||||
Windows 补验(2026-09-16):构建前逐一比对所有改动文件的 SHA-256,确认与 macOS
|
||||
工作树一致。在虚拟机内执行仓库完整 Windows ARM64 打包流程,生成 NSIS 安装器
|
||||
和便携 ZIP。最初 NSIS 工具目录权限不足,通过复制工具到专用测试目录解决,
|
||||
未修改产品代码或原工具目录权限。
|
||||
|
||||
Tested archive: `dist/Reasonix-windows-arm64.zip`, version
|
||||
`v0.0.0-termination-local`, SHA-256
|
||||
`859f720c74c59c86da62edc4a19bbf34aa166db63e3da0264771337ede253637`.
|
||||
The archive passed structural verification (463 entries), was freshly extracted
|
||||
to a dedicated disposable directory, and ran its bundled ARM64 Electron
|
||||
and Go binaries with no development or service-path overrides. Both the legacy
|
||||
submit RPC and the display/submission-ID RPC scenario passed; the latter also
|
||||
waited for the startup overlay to leave and checked visible restored content.
|
||||
The NSIS installer was built, but an install/uninstall cycle was not performed.
|
||||
|
||||
实测便携包版本及哈希如上,463 个包条目的结构校验通过,重新解压后直接运行包内
|
||||
ARM64 Electron 和 Go 服务,未使用开发模式或外部服务路径。旧提交 RPC 与带展示
|
||||
文本/提交 ID 的 RPC 两条流程均通过,后者还等待启动遮罩退出并确认恢复内容已
|
||||
渲染。NSIS 安装器已构建,但未执行安装/卸载周期。
|
||||
|
||||
Local evidence is retained separately: Windows build, session/control,
|
||||
Desktop and native-run logs, plus continued/restarted window screenshots. These
|
||||
use disposable homes and a loopback HTTP fixture, not user session data or vendor
|
||||
credentials. No production release is represented by this record.
|
||||
|
||||
本地另行保留构建、session/control、Desktop、原生运行日志以及继续和
|
||||
重启后的窗口截图。全部使用临时数据目录和回环 HTTP 夹具,没有使用用户会话数据
|
||||
或厂商凭据。本记录不代表正式发布。
|
||||
@@ -142,6 +142,12 @@ func unresolvedToolRecord(r provider.ToolCallRecord) bool {
|
||||
// Rewriting model history cannot erase an unresolved external-effect fact. A
|
||||
// local-only record survives compaction/rewind on the same session, but does
|
||||
// not restrict later tool admission.
|
||||
// PlanRetainedToolRecords applies the same retention rule as Session.Replace
|
||||
// without changing the session. Callers assign stable IDs before committing.
|
||||
func PlanRetainedToolRecords(previous, next []provider.Message) []provider.Message {
|
||||
return retainUnresolvedToolRecords(previous, next)
|
||||
}
|
||||
|
||||
func retainUnresolvedToolRecords(previous, next []provider.Message) []provider.Message {
|
||||
seen := map[string]bool{}
|
||||
for _, m := range next {
|
||||
|
||||
+18
-24
@@ -3,7 +3,6 @@ package control
|
||||
import (
|
||||
"reasonix/internal/agent"
|
||||
"reasonix/internal/event"
|
||||
"reasonix/internal/session"
|
||||
)
|
||||
|
||||
// CancelReceipt acknowledges a session-scoped Stop request. Accepted means the
|
||||
@@ -33,23 +32,16 @@ func (c *Controller) CancelSession() CancelReceipt {
|
||||
epoch := c.runtimeState.snapshot.RuntimeEpoch
|
||||
recoveryRequired := c.runtimeState.snapshot.Phase == "recovery_required"
|
||||
c.runtimeState.mu.Unlock()
|
||||
service, runtime, exclusive := c.v3Binding()
|
||||
_, runtime, exclusive := c.v3Binding()
|
||||
if exclusive && runtime != nil {
|
||||
sessionRef = runtime.Ref().SessionID
|
||||
headID = ""
|
||||
}
|
||||
cancelled := c.signalTurnCancel()
|
||||
if exclusive && runtime != nil && service != nil {
|
||||
if v3Receipt, err := service.CancelSession(runtime.Ref()); err == nil {
|
||||
epoch = v3Receipt.RuntimeEpoch
|
||||
alreadyIdle = v3Receipt.Phase == session.RuntimeIdle
|
||||
recoveryRequired = v3Receipt.Phase == session.RuntimeRecoveryRequired
|
||||
}
|
||||
}
|
||||
token, turnID, cancelled := c.signalTurnCancelIdentity()
|
||||
if cancelled {
|
||||
alreadyIdle = false
|
||||
}
|
||||
go c.finishCancellation(cancelled)
|
||||
go c.finishCancellation(token, turnID, cancelled)
|
||||
receipt := CancelReceipt{
|
||||
SessionRef: sessionRef, HeadID: headID, RuntimeEpoch: epoch,
|
||||
Accepted: true, AlreadyIdle: alreadyIdle, RecoveryRequired: recoveryRequired,
|
||||
@@ -75,26 +67,22 @@ func (c *Controller) cancelLocked() {
|
||||
// emit that follows is a synchronous event barrier, and a stalled event lane
|
||||
// must never keep the provider stream or a tool process alive after Stop.
|
||||
func (c *Controller) cancelTurnLocked() (string, bool) {
|
||||
cancelled := c.signalTurnCancel()
|
||||
_, turnID, cancelled := c.signalTurnCancelIdentity()
|
||||
if !cancelled {
|
||||
return "", false
|
||||
}
|
||||
turnID := ""
|
||||
if ledger := c.turnEventLedger(); ledger != nil {
|
||||
turnID = ledger.ActiveTurnID()
|
||||
}
|
||||
c.promptOwner.CancelAll()
|
||||
c.approval.clearAll()
|
||||
c.promptOwner.CancelTurn(turnID)
|
||||
return turnID, true
|
||||
}
|
||||
|
||||
func (c *Controller) finishCancellation(cancelled bool) {
|
||||
turnID := ""
|
||||
if ledger := c.turnEventLedger(); ledger != nil {
|
||||
turnID = ledger.ActiveTurnID()
|
||||
func (c *Controller) finishCancellation(token uint64, turnID string, cancelled bool) {
|
||||
c.mu.Lock()
|
||||
current := c.turns.token == token
|
||||
c.mu.Unlock()
|
||||
if !current {
|
||||
return
|
||||
}
|
||||
c.promptOwner.CancelAll()
|
||||
c.approval.clearAll()
|
||||
c.promptOwner.CancelTurn(turnID)
|
||||
c.finishCancel(turnID, cancelled)
|
||||
}
|
||||
|
||||
@@ -103,6 +91,12 @@ func (c *Controller) finishCancel(turnID string, cancelled bool) {
|
||||
if cancelled {
|
||||
c.emitTurnStatus(event.TurnCancelling, turnID)
|
||||
}
|
||||
c.mu.Lock()
|
||||
stale := turnID != "" && c.turns.turnID != "" && c.turns.turnID != turnID
|
||||
c.mu.Unlock()
|
||||
if stale {
|
||||
return
|
||||
}
|
||||
if c.goals.active() {
|
||||
c.stopGoal(GoalStatusStopped)
|
||||
}
|
||||
|
||||
@@ -171,7 +171,7 @@ func (c *Controller) validatedCheckpointTurn(completion *guardedTurnCompletion)
|
||||
if !c.checkpoints.matchesBoundary(candidate.store, candidate.turn, candidate.messageIndex) {
|
||||
return nil
|
||||
}
|
||||
messages := candidate.session.Snapshot()
|
||||
messages := c.terminationMessages()
|
||||
if candidate.messageIndex < 0 || candidate.messageIndex >= len(messages) {
|
||||
return nil
|
||||
}
|
||||
|
||||
+18
-114
@@ -3923,6 +3923,9 @@ func (c *Controller) stripTurnMessagesAfter(idx int) {
|
||||
}
|
||||
msgs := c.executor.Session().Snapshot()
|
||||
if len(msgs) <= idx {
|
||||
// Compaction may have removed the entire synthetic workset. The
|
||||
// explicit turn identities still need retraction from display history.
|
||||
c.replaceSessionAfterCancel(msgs)
|
||||
return
|
||||
}
|
||||
c.replaceSessionAfterCancel(msgs[:idx])
|
||||
@@ -3958,118 +3961,11 @@ func (c *Controller) stripCancelledVisibleTurnMessagesAfterWithFallbackAt(idx in
|
||||
if c.executor == nil {
|
||||
return
|
||||
}
|
||||
msgs := c.executor.Session().Snapshot()
|
||||
if start, ok := resolveInterruptedTurnStart(msgs, idx, true, startedAt, fallback); ok {
|
||||
idx = start
|
||||
before := c.executor.Session().Snapshot()
|
||||
next := planCancelledMessages(before, idx, fallback, startedAt, c.executor.CanReplayAssistantMessage, c.ledgerTailEvidence())
|
||||
if next != nil {
|
||||
c.replaceSessionAfterCancelFrom(before, next)
|
||||
}
|
||||
if idx < 0 {
|
||||
idx = 0
|
||||
}
|
||||
if idx > len(msgs) {
|
||||
idx = len(msgs)
|
||||
}
|
||||
next := append([]provider.Message{}, msgs[:idx]...)
|
||||
keptUser := false
|
||||
userEnd := idx
|
||||
for i, m := range msgs[idx:] {
|
||||
if !agent.IsUserAuthoredTurnMessage(m) {
|
||||
continue
|
||||
}
|
||||
m.Content = StripComposePrefixes(m.Content)
|
||||
next = append(next, m)
|
||||
keptUser = true
|
||||
userEnd = idx + i + 1
|
||||
break
|
||||
}
|
||||
if !keptUser && agent.IsUserAuthoredTurnMessage(fallback) {
|
||||
fallback.Content = StripComposePrefixes(fallback.Content)
|
||||
if strings.TrimSpace(fallback.Content) != "" {
|
||||
fallback.Images = append([]string(nil), fallback.Images...)
|
||||
next = append(next, fallback)
|
||||
keptUser = true
|
||||
userEnd = idx
|
||||
}
|
||||
}
|
||||
if !keptUser && len(msgs) <= idx {
|
||||
return
|
||||
}
|
||||
recovery := &provider.InterruptedTurnRecovery{Pending: true}
|
||||
localIndexes := make([]int, 0, 1)
|
||||
for i := userEnd; i < len(msgs); {
|
||||
m := msgs[i]
|
||||
if m.LocalOnly {
|
||||
m.Role = provider.RoleTool
|
||||
m.ToolCallID = provider.LocalOnlyToolID
|
||||
m.Name = provider.LocalOnlyToolName
|
||||
previousRecovery := m.InterruptedTurn
|
||||
m.InterruptedTurn = nil
|
||||
next = append(next, m)
|
||||
localIndexes = append(localIndexes, len(next)-1)
|
||||
recovery.DroppedPartialText = recovery.DroppedPartialText || strings.TrimSpace(m.Content) != ""
|
||||
recovery.DroppedPartialReasoning = recovery.DroppedPartialReasoning || strings.TrimSpace(m.ReasoningContent) != ""
|
||||
if previousRecovery != nil {
|
||||
recovery.CompletedTools = append(recovery.CompletedTools, previousRecovery.CompletedTools...)
|
||||
recovery.InterruptedTools = append(recovery.InterruptedTools, previousRecovery.InterruptedTools...)
|
||||
recovery.NotStartedTools = append(recovery.NotStartedTools, previousRecovery.NotStartedTools...)
|
||||
recovery.UnknownTools = append(recovery.UnknownTools, previousRecovery.UnknownTools...)
|
||||
} else {
|
||||
for _, call := range m.ToolCalls {
|
||||
provider.RecordToolRecovery(recovery, interruptedToolSummary(call), provider.ToolRunUnknown)
|
||||
}
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
// Auto-compaction can install a digest between the pinned current user
|
||||
// message and its recent tool tail. It summarizes pre-turn/current work
|
||||
// that is no longer present verbatim, so keep it provider-visible rather
|
||||
// than silently dropping context during recovery.
|
||||
if agent.IsCompactionSummary(m) {
|
||||
next = append(next, m)
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if m.Role == provider.RoleAssistant {
|
||||
recordInterruptedAssistantRecovery(recovery, msgs, i, c.ledgerTailEvidence())
|
||||
}
|
||||
if end, ok := completeToolTurnEnd(msgs, i); ok && c.executor.CanReplayAssistantMessage(m) {
|
||||
next = append(next, msgs[i:end]...)
|
||||
i = end
|
||||
continue
|
||||
}
|
||||
switch m.Role {
|
||||
case provider.RoleAssistant:
|
||||
local := m
|
||||
local.Role = provider.RoleTool
|
||||
local.LocalOnly = true
|
||||
local.ToolCallID = provider.LocalOnlyToolID
|
||||
local.Name = provider.LocalOnlyToolName
|
||||
local.InterruptedTurn = nil
|
||||
next = append(next, local)
|
||||
localIndexes = append(localIndexes, len(next)-1)
|
||||
recovery.DroppedPartialText = recovery.DroppedPartialText || strings.TrimSpace(local.Content) != ""
|
||||
recovery.DroppedPartialReasoning = recovery.DroppedPartialReasoning || strings.TrimSpace(local.ReasoningContent) != ""
|
||||
case provider.RoleTool:
|
||||
local := m
|
||||
local.LocalOnly = true
|
||||
local.ToolCalls = []provider.ToolCall{{ID: m.ToolCallID, Name: m.Name}}
|
||||
local.ToolCallID = provider.LocalOnlyToolID
|
||||
local.Name = provider.LocalOnlyToolName
|
||||
next = append(next, local)
|
||||
localIndexes = append(localIndexes, len(next)-1)
|
||||
}
|
||||
i++
|
||||
}
|
||||
if len(localIndexes) == 0 {
|
||||
next = append(next, provider.Message{
|
||||
Role: provider.RoleTool, ToolCallID: provider.LocalOnlyToolID,
|
||||
Name: provider.LocalOnlyToolName, LocalOnly: true,
|
||||
})
|
||||
localIndexes = append(localIndexes, len(next)-1)
|
||||
}
|
||||
c.applyLedgerRecoveryFacts(recovery)
|
||||
next[localIndexes[len(localIndexes)-1]].InterruptedTurn = recovery
|
||||
c.replaceSessionAfterCancel(next)
|
||||
}
|
||||
|
||||
func (c *Controller) inFlightTurnStartedAt() time.Time {
|
||||
@@ -4218,6 +4114,13 @@ func interruptedToolSummary(call provider.ToolCall) provider.InterruptedToolSumm
|
||||
}
|
||||
|
||||
func (c *Controller) replaceSessionAfterCancel(msgs []provider.Message) {
|
||||
if c.executor == nil {
|
||||
return
|
||||
}
|
||||
c.replaceSessionAfterCancelFromScoped(c.executor.Session().Snapshot(), msgs, true)
|
||||
}
|
||||
|
||||
func (c *Controller) replaceLegacySessionAfterCancel(msgs []provider.Message) {
|
||||
// The whole cleanup is a save/recovery handoff like snapshot's: hold
|
||||
// snapshotMu from the in-memory truncation onward. Truncating outside the
|
||||
// lock would let an in-flight save capture the shortened transcript, read
|
||||
@@ -4226,10 +4129,11 @@ func (c *Controller) replaceSessionAfterCancel(msgs []provider.Message) {
|
||||
// before the flush below could persist it.
|
||||
c.snapshotMu.Lock()
|
||||
defer c.snapshotMu.Unlock()
|
||||
c.replaceLegacySessionAfterCancelLocked(msgs)
|
||||
}
|
||||
|
||||
func (c *Controller) replaceLegacySessionAfterCancelLocked(msgs []provider.Message) {
|
||||
c.executor.Session().Replace(append([]provider.Message(nil), msgs...))
|
||||
if err := c.replaceSessionEventProjection(context.Background(), "cancel-or-recovery-rewrite", msgs); err != nil {
|
||||
slog.Warn("controller: record cancel/recovery transcript rewrite", "err", err)
|
||||
}
|
||||
// The mid-turn autosave may have already written a partial transcript to
|
||||
// disk. snapshotActivityIfChanged skips the write when messageCount()
|
||||
// returns to startMessages, so flush the cleaned transcript here. SaveRewrite
|
||||
|
||||
@@ -349,18 +349,33 @@ func (o *PendingPromptOwner) Clear() {
|
||||
o.mu.Unlock()
|
||||
}
|
||||
func (o *PendingPromptOwner) CancelAll() {
|
||||
o.cancelMatching(func(PromptIdentity) bool { return true })
|
||||
}
|
||||
|
||||
// CancelTurn cannot close prompts registered by a successor while an older
|
||||
// Stop request was waiting on its asynchronous publication lane.
|
||||
func (o *PendingPromptOwner) CancelTurn(turnID string) {
|
||||
o.cancelMatching(func(identity PromptIdentity) bool { return identity.TurnID == turnID })
|
||||
}
|
||||
|
||||
func (o *PendingPromptOwner) cancelMatching(matches func(PromptIdentity) bool) {
|
||||
o.mu.Lock()
|
||||
cancels := make([]func() error, 0, len(o.pending))
|
||||
identities := make([]PromptIdentity, 0, len(o.pending))
|
||||
prompts := make([]PendingPrompt, 0, len(o.pending))
|
||||
for _, prompt := range o.pending {
|
||||
if !matches(prompt.Identity) {
|
||||
continue
|
||||
}
|
||||
prompts = append(prompts, prompt)
|
||||
identities = append(identities, prompt.Identity)
|
||||
if prompt.Cancel != nil {
|
||||
cancels = append(cancels, prompt.Cancel)
|
||||
}
|
||||
}
|
||||
o.pending = make(map[string]PendingPrompt)
|
||||
for _, identity := range identities {
|
||||
delete(o.pending, identity.PromptID)
|
||||
}
|
||||
if o.resolved == nil {
|
||||
o.resolved = make(map[string]PromptResolution)
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ func (c *Controller) cancelledTurnWasSilent(completion *guardedTurnCompletion) b
|
||||
if c == nil || c.executor == nil || completion == nil || completion.checkpoint == nil {
|
||||
return false
|
||||
}
|
||||
msgs := c.executor.Session().Snapshot()
|
||||
msgs := c.terminationMessages()
|
||||
start := completion.checkpoint.messageIndex
|
||||
if start < 0 || start > len(msgs) {
|
||||
return false
|
||||
|
||||
@@ -404,16 +404,26 @@ func (c *Controller) appendSessionEventLocked(ctx context.Context, e event.Event
|
||||
if _, turnID, active := c.currentTurnToken(); active {
|
||||
e.TurnID = turnID
|
||||
}
|
||||
if e.TurnID == "" {
|
||||
e.TurnID = projection.TurnID
|
||||
}
|
||||
}
|
||||
events, err := c.v3EventsFor(e, projection)
|
||||
if err != nil || len(events) == 0 {
|
||||
return err
|
||||
}
|
||||
if e.Kind == event.TurnDone {
|
||||
if err := c.appendTerminationLocked(ctx, e, store, events); err != nil {
|
||||
return fmt.Errorf("%w: %w", turnevent.ErrTurnLedgerUnavailable, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
op := fmt.Sprintf("runtime:%s:%d:%d", e.TurnID, snapshot.EventSequence+1, e.Kind)
|
||||
_, err = c.appendSessionBatch(ctx, store, session.Batch{OperationID: op, TurnID: e.TurnID, Events: events})
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", turnevent.ErrTurnLedgerUnavailable, err)
|
||||
}
|
||||
c.noteCommittedMessagesLocked(events)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -538,13 +548,7 @@ func (c *Controller) v3EventsFor(e event.Event, projection session.Projection) (
|
||||
if e.Cancelled || e.Status == event.TurnInterrupted {
|
||||
interactionState = "cancelled"
|
||||
}
|
||||
for id := range projection.Interactions {
|
||||
interactionPayload, marshalErr := makePayload(map[string]any{"id": id, "state": interactionState})
|
||||
if marshalErr != nil {
|
||||
return nil, marshalErr
|
||||
}
|
||||
out = append(out, session.Event{Kind: "interaction/resolved", Payload: interactionPayload})
|
||||
}
|
||||
out = append(out, session.ClosureEvents(projection, interactionState, "turn ended before recording a result")...)
|
||||
if e.Recovery != nil && e.Recovery.State == "recovery_required" {
|
||||
recoveryPayload, marshalErr := makePayload(e.Recovery)
|
||||
if marshalErr != nil {
|
||||
@@ -606,6 +610,9 @@ func (c *Controller) RecordSessionMessages(ctx context.Context, reason string, m
|
||||
}
|
||||
c.turnEvents.commitMu.Lock()
|
||||
defer c.turnEvents.commitMu.Unlock()
|
||||
if !c.messageCommitAllowedLocked(ctx, store) {
|
||||
return nil
|
||||
}
|
||||
events := make([]session.Event, 0, len(messages))
|
||||
for _, message := range messages {
|
||||
if strings.TrimSpace(message.ID) == "" {
|
||||
@@ -620,6 +627,9 @@ func (c *Controller) RecordSessionMessages(ctx context.Context, reason string, m
|
||||
snapshot := store.ExecutionSnapshot()
|
||||
op := fmt.Sprintf("messages:%s:%d", reason, snapshot.EventSequence+1)
|
||||
_, err := c.appendSessionBatch(ctx, store, session.Batch{OperationID: op, TurnID: snapshot.Projection.TurnID, Events: events})
|
||||
if err == nil {
|
||||
c.noteCommittedMessagesLocked(events)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -644,6 +654,9 @@ func (c *Controller) RecordSessionMessageUpsert(ctx context.Context, reason stri
|
||||
}
|
||||
c.turnEvents.commitMu.Lock()
|
||||
defer c.turnEvents.commitMu.Unlock()
|
||||
if !c.messageCommitAllowedLocked(ctx, store) {
|
||||
return nil
|
||||
}
|
||||
snapshot := store.ExecutionSnapshot()
|
||||
op := fmt.Sprintf("message-upsert:%s:%s:%d", reason, message.ID, snapshot.EventSequence+1)
|
||||
_, err = c.appendSessionBatch(ctx, store, session.Batch{OperationID: op, TurnID: snapshot.Projection.TurnID, Events: []session.Event{{Kind: "message/upsert", Payload: payload}}})
|
||||
@@ -702,6 +715,9 @@ func (c *Controller) appendDomainState(kind string, payload json.RawMessage, rea
|
||||
}
|
||||
c.turnEvents.commitMu.Lock()
|
||||
defer c.turnEvents.commitMu.Unlock()
|
||||
if !c.messageCommitAllowedLocked(context.Background(), store) {
|
||||
return nil
|
||||
}
|
||||
snapshot := store.ExecutionSnapshot()
|
||||
op := fmt.Sprintf("domain:%s:%s:%d", kind, reason, snapshot.EventSequence+1)
|
||||
_, err := c.appendSessionBatch(context.Background(), store, session.Batch{OperationID: op, TurnID: snapshot.Projection.TurnID, Events: []session.Event{{Kind: kind, Payload: append(json.RawMessage(nil), payload...)}}})
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/agent"
|
||||
"reasonix/internal/event"
|
||||
"reasonix/internal/provider"
|
||||
"reasonix/internal/session"
|
||||
)
|
||||
|
||||
const terminationFlushTimeout = 15 * time.Second
|
||||
|
||||
var errTerminationDurability = errors.New("turn termination was not made durable")
|
||||
|
||||
type executionTokenKey struct{}
|
||||
|
||||
func (c *Controller) failTermination(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.enterRecoveryLocked("termination_commit_failed")
|
||||
c.mu.Unlock()
|
||||
c.disarmGoalLifecycle("persistence-error")
|
||||
c.failTurnEventLedger(fmt.Errorf("%w: %w", errTerminationDurability, err))
|
||||
}
|
||||
|
||||
// messageCommitAllowedLocked is checked after acquiring commitMu, so a
|
||||
// recorder waiting behind a terminal cannot revive its closed turn.
|
||||
func (c *Controller) messageCommitAllowedLocked(ctx context.Context, store *session.Session) bool {
|
||||
if !c.sessionEventCommitAllowed() {
|
||||
return false
|
||||
}
|
||||
token, turnID, _ := c.currentTurnToken()
|
||||
if ctx != nil {
|
||||
if origin, ok := ctx.Value(executionTokenKey{}).(uint64); ok && origin != token {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if turnID != "" && c.turnEvents.finalizedTurn == turnID {
|
||||
return false
|
||||
}
|
||||
projection := store.StateSnapshot().Projection
|
||||
return projection.Recovery == nil || projection.Recovery.State != "recovery_required"
|
||||
}
|
||||
|
||||
type terminationBoundary struct {
|
||||
token uint64
|
||||
prefix map[string]bool
|
||||
fallback provider.Message
|
||||
preserveUser bool
|
||||
}
|
||||
|
||||
func (c *Controller) noteTerminationBoundary(fallback provider.Message, preserveUser bool) {
|
||||
if c.executor == nil {
|
||||
return
|
||||
}
|
||||
token, _, _ := c.currentTurnToken()
|
||||
b := &terminationBoundary{token: token, prefix: map[string]bool{}, fallback: fallback, preserveUser: preserveUser}
|
||||
for _, message := range c.executor.Session().Snapshot() {
|
||||
b.prefix[message.ID] = true
|
||||
}
|
||||
c.turnEvents.commitMu.Lock()
|
||||
c.turnEvents.terminationBoundary = b
|
||||
c.turnEvents.commitMu.Unlock()
|
||||
}
|
||||
|
||||
// terminationMessages is for terminal metadata only. UI and execution readers
|
||||
// continue to observe the accepted projection until the terminal batch lands.
|
||||
func (c *Controller) terminationMessages() []provider.Message {
|
||||
c.turnEvents.commitMu.Lock()
|
||||
defer c.turnEvents.commitMu.Unlock()
|
||||
if p := c.turnEvents.pendingTermination; p != nil {
|
||||
return append([]provider.Message(nil), p.Messages...)
|
||||
}
|
||||
return c.executor.Session().Snapshot()
|
||||
}
|
||||
|
||||
// TerminationPlan is an owned snapshot of one explicit cleanup. Retractions
|
||||
// come only from its input workset, never from a diff against durable history.
|
||||
type TerminationPlan struct {
|
||||
SessionID string
|
||||
TurnID string
|
||||
Generation uint64
|
||||
Token uint64
|
||||
Messages []provider.Message
|
||||
Events []session.Event
|
||||
}
|
||||
|
||||
func buildTerminationPlan(before, after []provider.Message) (*TerminationPlan, error) {
|
||||
// Detach nested tool/reasoning records from the worker's mutable cache.
|
||||
data, err := json.Marshal(after)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(data, &after); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p := &TerminationPlan{Messages: after}
|
||||
old := make(map[string]provider.Message, len(before))
|
||||
for _, message := range before {
|
||||
old[message.ID] = message
|
||||
}
|
||||
kept := make(map[string]bool, len(after))
|
||||
for _, message := range after {
|
||||
if message.ID == "" {
|
||||
return nil, fmt.Errorf("termination message has no stable identity")
|
||||
}
|
||||
kept[message.ID] = true
|
||||
if previous, ok := old[message.ID]; ok && reflect.DeepEqual(previous, message) {
|
||||
continue
|
||||
}
|
||||
payload, err := json.Marshal(map[string]any{"message": message})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Events = append(p.Events, session.Event{Kind: "message/upsert", Payload: payload})
|
||||
}
|
||||
var removed []string
|
||||
for id := range old {
|
||||
if id != "" && !kept[id] {
|
||||
removed = append(removed, id)
|
||||
}
|
||||
}
|
||||
if len(removed) > 0 {
|
||||
sort.Strings(removed)
|
||||
payload, _ := json.Marshal(map[string]any{"messageIds": removed, "reason": "interrupted-turn-cleanup"})
|
||||
p.Events = append(p.Events, session.Event{Kind: "message/retract", Payload: payload, Required: true})
|
||||
}
|
||||
model := provider.ModelMessages(after)
|
||||
if model == nil {
|
||||
model = []provider.Message{}
|
||||
}
|
||||
payload, err := json.Marshal(map[string]any{"messages": model, "reason": "interrupted-turn-cleanup"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Events = append(p.Events, session.Event{Kind: "model/context-replace", Payload: payload})
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (c *Controller) replaceSessionAfterCancelFrom(before, after []provider.Message) {
|
||||
c.replaceSessionAfterCancelFromScoped(before, after, false)
|
||||
}
|
||||
|
||||
func (c *Controller) noteCommittedMessagesLocked(events []session.Event) {
|
||||
for _, e := range events {
|
||||
if e.Kind == "turn/start" {
|
||||
c.turnEvents.turnMessageIDs = map[string]bool{}
|
||||
}
|
||||
if e.Kind == "message/complete" {
|
||||
var payload struct {
|
||||
Message provider.Message `json:"message"`
|
||||
}
|
||||
if json.Unmarshal(e.Payload, &payload) == nil && payload.Message.ID != "" {
|
||||
if c.turnEvents.turnMessageIDs == nil {
|
||||
c.turnEvents.turnMessageIDs = map[string]bool{}
|
||||
}
|
||||
c.turnEvents.turnMessageIDs[payload.Message.ID] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) retractMissingTurnMessagesLocked(p *TerminationPlan) {
|
||||
kept := map[string]bool{}
|
||||
for _, m := range p.Messages {
|
||||
kept[m.ID] = true
|
||||
}
|
||||
var ids []string
|
||||
for id := range c.turnEvents.turnMessageIDs {
|
||||
if !kept[id] {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return
|
||||
}
|
||||
sort.Strings(ids)
|
||||
payload, _ := json.Marshal(map[string]any{"messageIds": ids, "reason": "synthetic-turn-interrupted"})
|
||||
p.Events = append([]session.Event{{Kind: "message/retract", Required: true, Payload: payload}}, p.Events...)
|
||||
}
|
||||
|
||||
func (c *Controller) replaceSessionAfterCancelFromScoped(before, after []provider.Message, retractTurn bool) {
|
||||
if c.executor == nil {
|
||||
return
|
||||
}
|
||||
after = agent.PlanRetainedToolRecords(before, after)
|
||||
for i := range after {
|
||||
if after[i].ID == "" {
|
||||
after[i].ID = agent.NewMessageID()
|
||||
}
|
||||
}
|
||||
p, err := buildTerminationPlan(before, after)
|
||||
if err != nil {
|
||||
c.failTermination(err)
|
||||
return
|
||||
}
|
||||
c.snapshotMu.Lock()
|
||||
store := c.sessionEventStore()
|
||||
if store == nil {
|
||||
c.replaceLegacySessionAfterCancelLocked(after)
|
||||
c.snapshotMu.Unlock()
|
||||
return
|
||||
}
|
||||
p.SessionID = store.ID()
|
||||
p.Token, p.TurnID, _ = c.currentTurnToken()
|
||||
p.Generation = c.ExecutionGeneration()
|
||||
c.turnEvents.commitMu.Lock()
|
||||
if !c.sessionEventCommitAllowed() || (p.TurnID != "" && c.turnEvents.finalizedTurn == p.TurnID) {
|
||||
c.turnEvents.commitMu.Unlock()
|
||||
c.snapshotMu.Unlock()
|
||||
return
|
||||
}
|
||||
projection := store.StateSnapshot().Projection
|
||||
if projection.Recovery != nil && projection.Recovery.State == "recovery_required" {
|
||||
c.turnEvents.commitMu.Unlock()
|
||||
c.snapshotMu.Unlock()
|
||||
return
|
||||
}
|
||||
if p.TurnID != "" && projection.TurnID == p.TurnID {
|
||||
if retractTurn {
|
||||
c.retractMissingTurnMessagesLocked(p)
|
||||
}
|
||||
// Compatibility runners may update the executor before their recorder
|
||||
// commits. Preserve these unchanged messages explicitly as well.
|
||||
known := map[string]bool{}
|
||||
for _, m := range store.ExecutionSnapshot().Projection.ModelMessages {
|
||||
known[m.ID] = true
|
||||
}
|
||||
var unrecorded []session.Event
|
||||
for _, m := range before {
|
||||
if known[m.ID] {
|
||||
continue
|
||||
}
|
||||
for _, retained := range after {
|
||||
if retained.ID == m.ID && reflect.DeepEqual(m, retained) {
|
||||
payload, _ := json.Marshal(map[string]any{"message": retained})
|
||||
unrecorded = append(unrecorded, session.Event{Kind: "message/upsert", Payload: payload})
|
||||
}
|
||||
}
|
||||
}
|
||||
p.Events = append(unrecorded, p.Events...)
|
||||
c.turnEvents.pendingTermination = p
|
||||
c.turnEvents.commitMu.Unlock()
|
||||
c.snapshotMu.Unlock()
|
||||
return
|
||||
}
|
||||
// Resume cleanup has no live turn to close. Its deterministic repair
|
||||
// operation never adds a second terminal record.
|
||||
data, _ := json.Marshal(p.Events)
|
||||
digest := sha256.Sum256(data)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), terminationFlushTimeout)
|
||||
_, err = c.appendSessionBatch(ctx, store, session.Batch{OperationID: fmt.Sprintf("turn-repair:%x", digest), Events: p.Events})
|
||||
if err == nil {
|
||||
c.executor.Session().Replace(p.Messages)
|
||||
_, err = store.Flush(ctx)
|
||||
}
|
||||
c.turnEvents.commitMu.Unlock()
|
||||
if err == nil && !c.sessionEngineEnabled() {
|
||||
c.replaceLegacySessionAfterCancelLocked(p.Messages)
|
||||
}
|
||||
c.snapshotMu.Unlock()
|
||||
cancel()
|
||||
if err != nil {
|
||||
c.failTermination(err)
|
||||
}
|
||||
}
|
||||
|
||||
// appendTerminationLocked runs under snapshotMu -> commitMu. The runtime gate
|
||||
// revalidates ownership and turn identity at acceptance, after preparation.
|
||||
func (c *Controller) appendTerminationLocked(ctx context.Context, e event.Event, store *session.Session, terminal []session.Event) error {
|
||||
if c.turnEvents.finalizedTurn == e.TurnID {
|
||||
_, err := store.Flush(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", errTerminationDurability, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
snapshot := store.StateSnapshot()
|
||||
if e.TurnID == "" || snapshot.Projection.TurnID != e.TurnID {
|
||||
return session.ErrStaleExecution
|
||||
}
|
||||
p := c.turnEvents.pendingTermination
|
||||
if e.Recovery != nil && e.Recovery.State == "recovery_required" {
|
||||
var err error
|
||||
p, err = c.watchdogTerminationPlanLocked(store, e.TurnID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if p != nil {
|
||||
token, turnID, _ := c.currentTurnToken()
|
||||
if p.SessionID != store.ID() || p.Generation != c.ExecutionGeneration() || p.Token != token || p.TurnID != turnID {
|
||||
return session.ErrStaleExecution
|
||||
}
|
||||
terminal = append(append([]session.Event(nil), p.Events...), terminal...)
|
||||
}
|
||||
batch := session.Batch{OperationID: "turn-finalize:" + e.TurnID, TurnID: e.TurnID, Events: terminal}
|
||||
prepared, err := store.PrepareBatchContext(ctx, batch.OperationID, batch)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || errors.Is(err, session.ErrOperationConflict) {
|
||||
return fmt.Errorf("%w: %w", errTerminationDurability, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
_, runtime, exclusive := c.v3Binding()
|
||||
if exclusive && runtime != nil {
|
||||
_, err = runtime.CommitPreparedForTurn(c.ExecutionGeneration(), e.TurnID, prepared)
|
||||
} else {
|
||||
_, err = store.CommitPrepared(prepared)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.turnEvents.finalizedTurn = e.TurnID
|
||||
c.turnEvents.pendingTermination = nil
|
||||
if p != nil && c.executor != nil {
|
||||
c.executor.Session().Replace(p.Messages)
|
||||
}
|
||||
_, err = store.Flush(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", errTerminationDurability, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Controller) watchdogTerminationPlanLocked(store *session.Session, turnID string) (*TerminationPlan, error) {
|
||||
// Seal accepted work only: an uncooperative worker may mutate its cache.
|
||||
before := store.ExecutionSnapshot().Projection.ModelMessages
|
||||
var next []provider.Message
|
||||
b := c.turnEvents.terminationBoundary
|
||||
token, _, _ := c.currentTurnToken()
|
||||
if b != nil && b.token == token {
|
||||
for _, message := range before {
|
||||
if b.prefix[message.ID] || agent.IsCompactionSummary(message) {
|
||||
next = append(next, message)
|
||||
}
|
||||
}
|
||||
if b.preserveUser && b.fallback.ID != "" {
|
||||
user := b.fallback
|
||||
for _, message := range before {
|
||||
if message.ID == user.ID {
|
||||
user = message
|
||||
break
|
||||
}
|
||||
}
|
||||
user.Content = StripComposePrefixes(user.Content)
|
||||
next = append(next, user)
|
||||
}
|
||||
} else {
|
||||
next = append([]provider.Message(nil), before...)
|
||||
}
|
||||
p, err := buildTerminationPlan(before, next)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.SessionID, p.TurnID, p.Generation, p.Token = store.ID(), turnID, c.ExecutionGeneration(), token
|
||||
if b != nil && !b.preserveUser {
|
||||
c.retractMissingTurnMessagesLocked(p)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
@@ -0,0 +1,535 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/agent"
|
||||
"reasonix/internal/event"
|
||||
"reasonix/internal/provider"
|
||||
"reasonix/internal/session"
|
||||
"reasonix/internal/tool"
|
||||
)
|
||||
|
||||
type terminationBlockingProvider struct{}
|
||||
|
||||
func (terminationBlockingProvider) Name() string { return "termination-test" }
|
||||
func (terminationBlockingProvider) Stream(ctx context.Context, _ provider.Request) (<-chan provider.Chunk, error) {
|
||||
chunks := make(chan provider.Chunk, 1)
|
||||
go func() {
|
||||
defer close(chunks)
|
||||
select {
|
||||
case chunks <- provider.Chunk{Type: provider.ChunkText, Text: "partial answer"}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
<-ctx.Done()
|
||||
chunks <- provider.Chunk{Type: provider.ChunkError, Err: ctx.Err()}
|
||||
}()
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
func terminationCommitHistory(t *testing.T, c *Controller) []session.Commit {
|
||||
t.Helper()
|
||||
store := c.sessionEventStore()
|
||||
if store == nil {
|
||||
t.Fatal("missing session store")
|
||||
}
|
||||
if _, err := store.Flush(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var commits []session.Commit
|
||||
var cursor uint64
|
||||
for {
|
||||
page, err := store.Handle().Read(t.Context(), cursor, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
commits = append(commits, page.Commits...)
|
||||
if !page.Truncated {
|
||||
break
|
||||
}
|
||||
if page.Next == cursor {
|
||||
t.Fatal("event pagination stalled")
|
||||
}
|
||||
cursor = page.Next
|
||||
}
|
||||
return commits
|
||||
}
|
||||
|
||||
func assertSingleTerminationCommit(t *testing.T, commits []session.Commit, turnID string, wantCleanup bool) {
|
||||
t.Helper()
|
||||
terminals, cleanups := 0, 0
|
||||
for _, commit := range commits {
|
||||
terminal, cleanup := false, false
|
||||
for _, e := range commit.Events {
|
||||
if e.Kind == "history/replace" {
|
||||
t.Fatal("pause emitted history/replace")
|
||||
}
|
||||
if commit.TurnID == turnID {
|
||||
terminal = terminal || e.Kind == "turn/end"
|
||||
cleanup = cleanup || e.Kind == "model/context-replace"
|
||||
}
|
||||
}
|
||||
if terminal {
|
||||
terminals++
|
||||
if commit.OperationID != "turn-finalize:"+turnID {
|
||||
t.Fatalf("unstable terminal operation %q", commit.OperationID)
|
||||
}
|
||||
}
|
||||
if cleanup {
|
||||
cleanups++
|
||||
if !terminal {
|
||||
t.Fatal("cleanup committed separately from turn/end")
|
||||
}
|
||||
}
|
||||
}
|
||||
if terminals != 1 {
|
||||
t.Fatalf("terminal commits=%d, want=1", terminals)
|
||||
}
|
||||
if wantCleanup && cleanups != 1 {
|
||||
t.Fatalf("cleanup commits=%d, want=1", cleanups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTerminationCommitRealSendPartialCancel(t *testing.T) {
|
||||
done := make(chan event.Event, 8)
|
||||
text := make(chan struct{}, 1)
|
||||
sink := event.FuncSink(func(e event.Event) {
|
||||
if e.Kind == event.TurnDone {
|
||||
done <- e
|
||||
}
|
||||
if e.Kind == event.Text {
|
||||
select {
|
||||
case text <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
})
|
||||
exec := agent.New(terminationBlockingProvider{}, tool.NewRegistry(), agent.NewSession("system"), agent.Options{}, sink)
|
||||
dir := t.TempDir()
|
||||
c := newOwnedTestController(t, Options{Runner: exec, Executor: exec, Sink: sink, SessionDir: dir, SessionPath: filepath.Join(dir, "session.jsonl")})
|
||||
c.Send("keep my question")
|
||||
awaitPromptLedgerTest(t, text, "partial output")
|
||||
c.CancelSession()
|
||||
c.CancelSession()
|
||||
terminal := waitTurnDoneEvent(t, done)
|
||||
waitIdleAdmission(t, c)
|
||||
assertSingleTerminationCommit(t, terminationCommitHistory(t, c), terminal.TurnID, true)
|
||||
kept, recovery := false, false
|
||||
for _, m := range c.sessionEventStore().Snapshot().Projection.Messages {
|
||||
kept = kept || (m.Role == provider.RoleUser && m.Content == "keep my question")
|
||||
// Existing sampling cancellation discards speculative result.text before
|
||||
// recordInterruptedDisplay (sampling_recovery.go). Preserve that policy:
|
||||
// this path retains a pending marker, not uncommitted stream text.
|
||||
recovery = recovery || (m.LocalOnly && m.InterruptedTurn != nil && m.InterruptedTurn.Pending)
|
||||
}
|
||||
if !kept || !recovery {
|
||||
t.Fatalf("retained user=%v pending recovery=%v", kept, recovery)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTerminationCommitFallbackAndSynthetic(t *testing.T) {
|
||||
for _, name := range []string{"fallback", "synthetic", "persisted-partial"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
synthetic := name == "synthetic"
|
||||
done := make(chan event.Event, 8)
|
||||
c, _, _ := exclusiveTestController(t, event.FuncSink(func(e event.Event) {
|
||||
if e.Kind == event.TurnDone {
|
||||
done <- e
|
||||
}
|
||||
}))
|
||||
started := make(chan struct{})
|
||||
question := provider.Message{ID: "current-user", Role: provider.RoleUser, Content: "preserve exact input"}
|
||||
c.runGuarded(func(ctx context.Context) error {
|
||||
start := c.executor.Session().Len()
|
||||
if synthetic {
|
||||
c.executor.Session().Add(provider.Message{ID: "synthetic", Role: provider.RoleUser, Content: "internal generated input"})
|
||||
if err := c.RecordSessionMessages(ctx, "synthetic-input", c.executor.Session().Snapshot()[start:]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if name == "persisted-partial" {
|
||||
c.executor.Session().Add(question)
|
||||
c.executor.Session().Add(provider.Message{ID: "partial", Role: provider.RoleAssistant, Content: "committed fragment", ReasoningContent: "partial reasoning"})
|
||||
if err := c.RecordSessionMessages(ctx, "partial-input", c.executor.Session().Snapshot()[start:]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
if synthetic {
|
||||
c.stripTurnMessagesAfter(start)
|
||||
} else {
|
||||
c.stripCancelledVisibleTurnMessagesAfterWithFallback(start, question)
|
||||
}
|
||||
return ctx.Err()
|
||||
})
|
||||
awaitPromptLedgerTest(t, started, "turn start")
|
||||
c.CancelSession()
|
||||
first := waitTurnDoneEvent(t, done)
|
||||
waitIdleAdmission(t, c)
|
||||
commits := terminationCommitHistory(t, c)
|
||||
assertSingleTerminationCommit(t, commits, first.TurnID, true)
|
||||
found, retracted := false, false
|
||||
for _, m := range c.sessionEventStore().Snapshot().Projection.Messages {
|
||||
if m.ID == question.ID {
|
||||
found = true
|
||||
}
|
||||
if m.ID == "synthetic" {
|
||||
t.Fatal("synthetic message survived cancellation")
|
||||
}
|
||||
}
|
||||
for _, commit := range commits {
|
||||
for _, e := range commit.Events {
|
||||
retracted = retracted || e.Kind == "message/retract"
|
||||
}
|
||||
}
|
||||
if synthetic && !retracted {
|
||||
t.Fatal("synthetic cancellation omitted retraction")
|
||||
}
|
||||
if !synthetic && !found {
|
||||
t.Fatal("pre-executor fallback lost")
|
||||
}
|
||||
if name == "persisted-partial" {
|
||||
local := false
|
||||
for _, m := range c.sessionEventStore().Snapshot().Projection.Messages {
|
||||
local = local || (m.ID == "partial" && m.LocalOnly && m.Content == "committed fragment" && m.ReasoningContent == "partial reasoning")
|
||||
}
|
||||
if !local {
|
||||
t.Fatal("persisted partial fragment lost during cleanup")
|
||||
}
|
||||
}
|
||||
if got := c.runGuarded(func(context.Context) error { return nil }); got != turnStarted {
|
||||
t.Fatalf("next admission=%v", got)
|
||||
}
|
||||
second := waitTurnDoneEvent(t, done)
|
||||
waitIdleAdmission(t, c)
|
||||
if second.TurnID == first.TurnID {
|
||||
t.Fatal("next turn reused terminated ID")
|
||||
}
|
||||
assertSingleTerminationCommit(t, terminationCommitHistory(t, c), first.TurnID, true)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTerminationCommitWatchdogRejectsLateWorker(t *testing.T) {
|
||||
states := make(chan event.RuntimeStateSnapshot, 32)
|
||||
terminals := make(chan event.Event, 4)
|
||||
c, _, _ := exclusiveTestController(t, &runtimeStateTestSink{Sink: event.FuncSink(func(e event.Event) {
|
||||
if e.Kind == event.TurnDone {
|
||||
terminals <- e
|
||||
}
|
||||
}), states: states})
|
||||
c.testCancelGrace = time.Nanosecond
|
||||
started, release, returned := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
t.Cleanup(func() { once.Do(func() { close(release) }) })
|
||||
c.runGuarded(func(context.Context) error {
|
||||
close(started)
|
||||
<-release
|
||||
defer close(returned)
|
||||
c.sink.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{ID: "late", Name: "todo_write", TodoWritten: true, Todos: []event.Todo{{Content: "late mutation", Status: "completed"}}}})
|
||||
return nil
|
||||
})
|
||||
awaitPromptLedgerTest(t, started, "uncooperative turn start")
|
||||
c.mu.Lock()
|
||||
idleDone := c.turns.finishingBound.idleDone
|
||||
c.mu.Unlock()
|
||||
c.CancelSession()
|
||||
runtimeStateAwait(t, states, func(s event.RuntimeStateSnapshot) bool { return s.Phase == "recovery_required" })
|
||||
awaitPromptLedgerTest(t, terminals, "watchdog terminal publication")
|
||||
before := terminationCommitHistory(t, c)
|
||||
var turnID string
|
||||
for _, commit := range before {
|
||||
for _, e := range commit.Events {
|
||||
if e.Kind == "turn/end" {
|
||||
turnID = commit.TurnID
|
||||
}
|
||||
}
|
||||
}
|
||||
if turnID == "" {
|
||||
t.Fatal("watchdog published recovery before terminal commit")
|
||||
}
|
||||
c.CancelSession()
|
||||
once.Do(func() { close(release) })
|
||||
awaitPromptLedgerTest(t, returned, "late worker body return")
|
||||
awaitPromptLedgerTest(t, idleDone, "sealed worker finalization")
|
||||
assertSingleTerminationCommit(t, terminationCommitHistory(t, c), turnID, false)
|
||||
if p := c.sessionEventStore().Snapshot().Projection; p.TodoWritten || len(p.Todos) > 0 {
|
||||
b, _ := json.Marshal(p.Todos)
|
||||
t.Fatalf("late worker changed todos: %s", b)
|
||||
}
|
||||
if got := c.runGuarded(func(context.Context) error { return nil }); got != turnDroppedWriteAuthority {
|
||||
t.Fatalf("sealed runtime admitted another turn: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdogPlanPreservesAcceptedCompactionSummary(t *testing.T) {
|
||||
c, _, _ := exclusiveTestController(t, event.Discard)
|
||||
input := provider.Message{ID: "watchdog-input", Role: provider.RoleUser, Content: "real input"}
|
||||
c.noteTerminationBoundary(input, true)
|
||||
summary := provider.Message{ID: "watchdog-summary", Role: provider.RoleUser, Content: "<compaction-summary>\nearlier work\n</compaction-summary>"}
|
||||
if err := c.replaceSessionModelContext(t.Context(), []provider.Message{summary, input}, "test-compaction"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.turnEvents.commitMu.Lock()
|
||||
p, err := c.watchdogTerminationPlanLocked(c.sessionEventStore(), "watchdog-turn")
|
||||
c.turnEvents.commitMu.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(p.Messages) != 2 || p.Messages[0].ID != summary.ID || p.Messages[1].ID != input.ID {
|
||||
t.Fatalf("watchdog lost accepted context: %+v", p.Messages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTerminationCommitStorageFailureRequiresRecovery(t *testing.T) {
|
||||
states := make(chan event.RuntimeStateSnapshot, 32)
|
||||
c, _, _ := exclusiveTestController(t, &runtimeStateTestSink{Sink: event.Discard, states: states})
|
||||
started := make(chan struct{})
|
||||
c.runGuarded(func(ctx context.Context) error {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
c.stripCancelledVisibleTurnMessagesAfterWithFallback(c.executor.Session().Len(), provider.Message{ID: "input", Role: provider.RoleUser, Content: "keep input"})
|
||||
return ctx.Err()
|
||||
})
|
||||
awaitPromptLedgerTest(t, started, "turn start")
|
||||
blockPromptTestLedger(t, c, "")
|
||||
c.CancelSession()
|
||||
runtimeStateAwait(t, states, func(s event.RuntimeStateSnapshot) bool { return s.Phase == "recovery_required" })
|
||||
if got := c.runGuarded(func(context.Context) error { return nil }); got != turnDroppedWriteAuthority {
|
||||
t.Fatalf("failed terminal allowed send: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type terminationSyncFailurePersistence struct {
|
||||
*session.FilesystemPersistence
|
||||
armed atomic.Bool
|
||||
store *session.Session
|
||||
failed chan session.Snapshot
|
||||
}
|
||||
|
||||
func (p *terminationSyncFailurePersistence) Create(options session.CreateOptions) (*session.Session, error) {
|
||||
store, err := session.CreateWithOptions(filepath.Join(p.Root, options.SessionID), options.SessionID, session.OpenOptions{Sync: func(file *os.File) error {
|
||||
if p.armed.Load() {
|
||||
snapshot := p.store.StateSnapshot()
|
||||
// Fail only after the terminal was accepted into the execution
|
||||
// projection; earlier status/autosave flushes must remain healthy.
|
||||
if snapshot.Projection.TurnID == "" {
|
||||
select {
|
||||
case p.failed <- snapshot:
|
||||
default:
|
||||
}
|
||||
return errors.New("injected accepted-terminal fsync failure")
|
||||
}
|
||||
}
|
||||
return file.Sync()
|
||||
}})
|
||||
if err == nil {
|
||||
p.store = store
|
||||
}
|
||||
return store, err
|
||||
}
|
||||
|
||||
func TestTerminationCommitAcceptedFlushFailureRequiresRecovery(t *testing.T) {
|
||||
persistence := &terminationSyncFailurePersistence{FilesystemPersistence: session.NewFilesystemPersistence(filepath.Join(t.TempDir(), "sessions")), failed: make(chan session.Snapshot, 4)}
|
||||
service, err := session.NewService("desktop", persistence)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { persistence.armed.Store(false); _ = service.CloseAll(context.Background()) })
|
||||
runtime, err := service.Create(t.Context(), session.CreateOptions{SessionID: "flush-failure"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
states := make(chan event.RuntimeStateSnapshot, 32)
|
||||
sink := &runtimeStateTestSink{Sink: event.Discard, states: states}
|
||||
exec := agent.New(nil, tool.NewRegistry(), agent.NewSession("system"), agent.Options{}, sink)
|
||||
c := newOwnedTestController(t, Options{Executor: exec, Sink: sink, SessionService: service, SessionRuntime: runtime, ExclusiveSession: true})
|
||||
t.Cleanup(func() { persistence.armed.Store(false) })
|
||||
started := make(chan struct{})
|
||||
c.runGuarded(func(ctx context.Context) error {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
c.stripCancelledVisibleTurnMessagesAfterWithFallback(c.executor.Session().Len(), provider.Message{ID: "accepted-input", Role: provider.RoleUser, Content: "accepted input"})
|
||||
persistence.armed.Store(true)
|
||||
return ctx.Err()
|
||||
})
|
||||
awaitPromptLedgerTest(t, started, "turn start")
|
||||
_, terminatingTurnID, _ := c.currentTurnToken()
|
||||
c.CancelSession()
|
||||
accepted := awaitPromptLedgerTest(t, persistence.failed, "terminal accepted before fsync failure")
|
||||
if accepted.EventSequence <= accepted.DurableSequence {
|
||||
t.Fatalf("injection did not cover accepted-only tail: %+v", accepted)
|
||||
}
|
||||
runtimeStateAwait(t, states, func(s event.RuntimeStateSnapshot) bool { return s.Phase == "recovery_required" })
|
||||
if got := c.runGuarded(func(context.Context) error { return nil }); got != turnDroppedWriteAuthority {
|
||||
t.Fatalf("failed durability admitted another turn: %v", got)
|
||||
}
|
||||
// Retry only the persistence barrier, never synthesize a second terminal.
|
||||
persistence.armed.Store(false)
|
||||
assertSingleTerminationCommit(t, terminationCommitHistory(t, c), terminatingTurnID, true)
|
||||
}
|
||||
|
||||
func TestTerminationCommitSyntheticCompactedOutOfModelStillRetracts(t *testing.T) {
|
||||
done := make(chan event.Event, 8)
|
||||
c, _, _ := exclusiveTestController(t, event.FuncSink(func(e event.Event) {
|
||||
if e.Kind == event.TurnDone {
|
||||
done <- e
|
||||
}
|
||||
}))
|
||||
prefix := []provider.Message{
|
||||
{ID: "prefix-system", Role: provider.RoleSystem, Content: "system"},
|
||||
{ID: "old-user", Role: provider.RoleUser, Content: "old real question"},
|
||||
{ID: "old-answer", Role: provider.RoleAssistant, Content: "old real answer"},
|
||||
}
|
||||
if err := c.RecordSessionMessages(t.Context(), "old-history", prefix); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.restoreExecutorFromSessionEvents()
|
||||
started := make(chan struct{})
|
||||
c.runGuarded(func(ctx context.Context) error {
|
||||
start := c.executor.Session().Len()
|
||||
input := provider.Message{ID: "synthetic-input", Role: provider.RoleUser, Content: "internal generated task"}
|
||||
c.noteTerminationBoundary(input, false)
|
||||
current := []provider.Message{input, {ID: "synthetic-answer", Role: provider.RoleAssistant, Content: "unfinished generated answer"}}
|
||||
for _, m := range current {
|
||||
c.executor.Session().Add(m)
|
||||
}
|
||||
if err := c.RecordSessionMessages(ctx, "synthetic-before-compaction", current); err != nil {
|
||||
return err
|
||||
}
|
||||
// Compaction removes both old history and this turn's records from the
|
||||
// model workset; neither absence alone authorizes deleting old history.
|
||||
compacted := []provider.Message{prefix[0], {ID: "summary", Role: provider.RoleUser, Content: "<compaction-summary>\nearlier work\n</compaction-summary>"}}
|
||||
if err := c.replaceSessionModelContext(ctx, compacted, "test-auto-compaction"); err != nil {
|
||||
return err
|
||||
}
|
||||
c.executor.Session().Replace(compacted)
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
c.stripInterruptedSyntheticTurnMessagesAfter(start)
|
||||
return ctx.Err()
|
||||
})
|
||||
awaitPromptLedgerTest(t, started, "compacted synthetic turn")
|
||||
c.CancelSession()
|
||||
terminal := waitTurnDoneEvent(t, done)
|
||||
waitIdleAdmission(t, c)
|
||||
commits := terminationCommitHistory(t, c)
|
||||
// The earlier context replacement is compaction, not terminal cleanup.
|
||||
var ends int
|
||||
for _, commit := range commits {
|
||||
for _, e := range commit.Events {
|
||||
if e.Kind == "history/replace" {
|
||||
t.Fatal("synthetic cancellation rewrote history")
|
||||
}
|
||||
if e.Kind == "turn/end" && commit.TurnID == terminal.TurnID {
|
||||
ends++
|
||||
}
|
||||
}
|
||||
}
|
||||
if ends != 1 {
|
||||
t.Fatalf("terminal count=%d", ends)
|
||||
}
|
||||
projection := c.sessionEventStore().Snapshot().Projection
|
||||
retained := map[string]bool{}
|
||||
for _, m := range projection.Messages {
|
||||
retained[m.ID] = true
|
||||
if m.ID == "synthetic-input" || m.ID == "synthetic-answer" {
|
||||
t.Fatalf("compacted current-turn record survived: %s", m.ID)
|
||||
}
|
||||
}
|
||||
if !retained["old-user"] || !retained["old-answer"] {
|
||||
t.Fatalf("compaction prefix was incorrectly retracted: %+v", retained)
|
||||
}
|
||||
summaryKept := false
|
||||
for _, m := range projection.ModelMessages {
|
||||
summaryKept = summaryKept || m.ID == "summary"
|
||||
}
|
||||
if !summaryKept {
|
||||
t.Fatal("compaction summary was removed from model context")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTerminationCommitRetainsUnresolvedSideEffectOnce(t *testing.T) {
|
||||
done := make(chan event.Event, 8)
|
||||
c, _, _ := exclusiveTestController(t, event.FuncSink(func(e event.Event) {
|
||||
if e.Kind == event.TurnDone {
|
||||
done <- e
|
||||
}
|
||||
}))
|
||||
record := provider.ToolCallRecord{Identity: provider.ActionIdentity{AttemptID: "attempt-write", CallID: "write-1"}, Arguments: json.RawMessage(`{"path":"output.txt"}`), State: provider.ToolRunUnknown, ReadOnly: false, EffectSummary: "effect_unknown"}
|
||||
started := make(chan struct{})
|
||||
c.runGuarded(func(ctx context.Context) error {
|
||||
start := c.executor.Session().Len()
|
||||
input := provider.Message{ID: "synthetic-input", Role: provider.RoleUser, Content: "internal task"}
|
||||
c.noteTerminationBoundary(input, false)
|
||||
messages := []provider.Message{input, {ID: "side-effect-call", Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: "write-1", Name: "write_file", Arguments: `{"path":"output.txt"}`, Recovery: &record}}}}
|
||||
for _, message := range messages {
|
||||
c.executor.Session().Add(message)
|
||||
}
|
||||
if err := c.RecordSessionMessages(ctx, "side-effect-record", messages); err != nil {
|
||||
return err
|
||||
}
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
c.stripTurnMessagesAfter(start)
|
||||
return ctx.Err()
|
||||
})
|
||||
awaitPromptLedgerTest(t, started, "side-effect record")
|
||||
c.CancelSession()
|
||||
c.CancelSession()
|
||||
terminal := waitTurnDoneEvent(t, done)
|
||||
waitIdleAdmission(t, c)
|
||||
assertSingleTerminationCommit(t, terminationCommitHistory(t, c), terminal.TurnID, true)
|
||||
findRecord := func(messages []provider.Message) string {
|
||||
t.Helper()
|
||||
count := 0
|
||||
id := ""
|
||||
for _, message := range messages {
|
||||
if message.ID == "synthetic-input" || message.ID == "side-effect-call" {
|
||||
t.Fatalf("synthetic source survived: %s", message.ID)
|
||||
}
|
||||
for _, call := range message.ToolCalls {
|
||||
if call.Recovery != nil && call.Recovery.Identity.AttemptID == "attempt-write" {
|
||||
count++
|
||||
id = message.ID
|
||||
if !message.LocalOnly || id == "" || call.Recovery.State != provider.ToolRunUnknown || string(call.Recovery.Arguments) != string(record.Arguments) {
|
||||
t.Fatalf("retained record changed: %+v", message)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("retained side effect count=%d want=1; messages=%+v", count, messages)
|
||||
}
|
||||
return id
|
||||
}
|
||||
durableID := findRecord(c.sessionEventStore().Snapshot().Projection.Messages)
|
||||
if executorID := findRecord(c.executor.Session().Snapshot()); executorID != durableID {
|
||||
t.Fatalf("executor invented ghost ID %q, durable=%q", executorID, durableID)
|
||||
}
|
||||
if pending := c.executor.PendingToolRecovery(); len(pending) != 1 || pending[0].Identity.AttemptID != "attempt-write" {
|
||||
t.Fatalf("pending side-effect fact=%+v", pending)
|
||||
}
|
||||
// Reinstalling the accepted context must not invoke Replace's retention
|
||||
// fallback a second time and generate an uncommitted LocalOnly ghost.
|
||||
c.executor.Session().Replace(c.executor.Session().Snapshot())
|
||||
if id := findRecord(c.executor.Session().Snapshot()); id != durableID {
|
||||
t.Fatalf("second Replace invented ghost ID %q", id)
|
||||
}
|
||||
if err := c.Snapshot(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id := findRecord(c.sessionEventStore().Snapshot().Projection.Messages); id != durableID {
|
||||
t.Fatalf("autosave changed retained identity %q", id)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"reasonix/internal/agent"
|
||||
"reasonix/internal/provider"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// planCancelledMessages preserves the existing provider replay policy without
|
||||
// mutating the executor, storage, or UI. Its result belongs to the caller.
|
||||
func planCancelledMessages(msgs []provider.Message, idx int, fallback provider.Message, startedAt time.Time, canReplay func(provider.Message) bool, evidence *interruptedTailEvidence) []provider.Message {
|
||||
if start, ok := resolveInterruptedTurnStart(msgs, idx, true, startedAt, fallback); ok {
|
||||
idx = start
|
||||
}
|
||||
if idx < 0 {
|
||||
idx = 0
|
||||
}
|
||||
if idx > len(msgs) {
|
||||
idx = len(msgs)
|
||||
}
|
||||
next := append([]provider.Message{}, msgs[:idx]...)
|
||||
keptUser := false
|
||||
userEnd := idx
|
||||
for i, m := range msgs[idx:] {
|
||||
if !agent.IsUserAuthoredTurnMessage(m) {
|
||||
continue
|
||||
}
|
||||
m.Content = StripComposePrefixes(m.Content)
|
||||
next = append(next, m)
|
||||
keptUser = true
|
||||
userEnd = idx + i + 1
|
||||
break
|
||||
}
|
||||
if !keptUser && agent.IsUserAuthoredTurnMessage(fallback) {
|
||||
fallback.Content = StripComposePrefixes(fallback.Content)
|
||||
if strings.TrimSpace(fallback.Content) != "" {
|
||||
fallback.Images = append([]string(nil), fallback.Images...)
|
||||
next = append(next, fallback)
|
||||
keptUser = true
|
||||
userEnd = idx
|
||||
}
|
||||
}
|
||||
if !keptUser && len(msgs) <= idx {
|
||||
return nil
|
||||
}
|
||||
recovery := &provider.InterruptedTurnRecovery{Pending: true}
|
||||
localIndexes := make([]int, 0, 1)
|
||||
for i := userEnd; i < len(msgs); {
|
||||
m := msgs[i]
|
||||
if m.LocalOnly {
|
||||
m.Role = provider.RoleTool
|
||||
m.ToolCallID = provider.LocalOnlyToolID
|
||||
m.Name = provider.LocalOnlyToolName
|
||||
previousRecovery := m.InterruptedTurn
|
||||
m.InterruptedTurn = nil
|
||||
next = append(next, m)
|
||||
localIndexes = append(localIndexes, len(next)-1)
|
||||
recovery.DroppedPartialText = recovery.DroppedPartialText || strings.TrimSpace(m.Content) != ""
|
||||
recovery.DroppedPartialReasoning = recovery.DroppedPartialReasoning || strings.TrimSpace(m.ReasoningContent) != ""
|
||||
if previousRecovery != nil {
|
||||
recovery.CompletedTools = append(recovery.CompletedTools, previousRecovery.CompletedTools...)
|
||||
recovery.InterruptedTools = append(recovery.InterruptedTools, previousRecovery.InterruptedTools...)
|
||||
recovery.NotStartedTools = append(recovery.NotStartedTools, previousRecovery.NotStartedTools...)
|
||||
recovery.UnknownTools = append(recovery.UnknownTools, previousRecovery.UnknownTools...)
|
||||
} else {
|
||||
for _, call := range m.ToolCalls {
|
||||
provider.RecordToolRecovery(recovery, interruptedToolSummary(call), provider.ToolRunUnknown)
|
||||
}
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
// Keep compaction digests between the pinned input and recent tool tail;
|
||||
// their summarized work is no longer available verbatim.
|
||||
if agent.IsCompactionSummary(m) {
|
||||
next = append(next, m)
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if m.Role == provider.RoleAssistant {
|
||||
recordInterruptedAssistantRecovery(recovery, msgs, i, evidence)
|
||||
}
|
||||
if end, ok := completeToolTurnEnd(msgs, i); ok && canReplay(m) {
|
||||
next = append(next, msgs[i:end]...)
|
||||
i = end
|
||||
continue
|
||||
}
|
||||
switch m.Role {
|
||||
case provider.RoleAssistant:
|
||||
local := m
|
||||
local.Role = provider.RoleTool
|
||||
local.LocalOnly = true
|
||||
local.ToolCallID = provider.LocalOnlyToolID
|
||||
local.Name = provider.LocalOnlyToolName
|
||||
local.InterruptedTurn = nil
|
||||
next = append(next, local)
|
||||
localIndexes = append(localIndexes, len(next)-1)
|
||||
recovery.DroppedPartialText = recovery.DroppedPartialText || strings.TrimSpace(local.Content) != ""
|
||||
recovery.DroppedPartialReasoning = recovery.DroppedPartialReasoning || strings.TrimSpace(local.ReasoningContent) != ""
|
||||
case provider.RoleTool:
|
||||
local := m
|
||||
local.LocalOnly = true
|
||||
local.ToolCalls = []provider.ToolCall{{ID: m.ToolCallID, Name: m.Name}}
|
||||
local.ToolCallID = provider.LocalOnlyToolID
|
||||
local.Name = provider.LocalOnlyToolName
|
||||
next = append(next, local)
|
||||
localIndexes = append(localIndexes, len(next)-1)
|
||||
}
|
||||
i++
|
||||
}
|
||||
if len(localIndexes) == 0 {
|
||||
next = append(next, provider.Message{
|
||||
Role: provider.RoleTool, ToolCallID: provider.LocalOnlyToolID,
|
||||
Name: provider.LocalOnlyToolName, LocalOnly: true,
|
||||
})
|
||||
localIndexes = append(localIndexes, len(next)-1)
|
||||
}
|
||||
if evidence != nil {
|
||||
recovery.Cause = "runtime_restart"
|
||||
recovery.TurnID = evidence.turnID
|
||||
recovery.SilentInterruption = len(recovery.ToolCalls) == 0 && len(recovery.CompletedTools) == 0 && !recovery.DroppedPartialText && !recovery.DroppedPartialReasoning
|
||||
}
|
||||
next[localIndexes[len(localIndexes)-1]].InterruptedTurn = recovery
|
||||
return next
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/provider"
|
||||
"reasonix/internal/provider/anthropic"
|
||||
"reasonix/internal/provider/openai"
|
||||
"reasonix/internal/provider/responses"
|
||||
"reasonix/internal/session"
|
||||
)
|
||||
|
||||
func TestTerminationPolicyPreservesSemanticsAndProviderBytes(t *testing.T) {
|
||||
system := provider.Message{ID: "system", Role: provider.RoleSystem, Content: "stable system"}
|
||||
user := provider.Message{ID: "user", Role: provider.RoleUser, Content: "update files"}
|
||||
call := provider.Message{ID: "assistant", Role: provider.RoleAssistant, Content: "calling", ReasoningContent: "original reasoning", ReasoningID: "reason-1", ReasoningStatus: "completed", ReasoningSignature: "original-proof", ToolCalls: []provider.ToolCall{{ID: "done", Name: "lookup", Arguments: `{"q":"value"}`, ThoughtSignature: "thought-proof"}}}
|
||||
result := provider.Message{ID: "result", Role: provider.RoleTool, ToolCallID: "done", Name: "lookup", Content: "result", ToolRunState: provider.ToolRunCompleted}
|
||||
partial := provider.Message{ID: "partial", Role: provider.RoleAssistant, Content: "unfinished", ReasoningContent: "partial reasoning", ReasoningSignature: "partial-proof"}
|
||||
summary := provider.Message{ID: "summary", Role: provider.RoleUser, Content: "<compaction-summary>\nearlier work\n</compaction-summary>"}
|
||||
local := provider.Message{ID: "local", Role: provider.RoleTool, Content: "old partial", LocalOnly: true, ToolCallID: provider.LocalOnlyToolID, Name: provider.LocalOnlyToolName, InterruptedTurn: &provider.InterruptedTurnRecovery{Pending: true, DroppedPartialText: true}}
|
||||
partialBatch := call
|
||||
partialBatch.ToolCalls = append(append([]provider.ToolCall{}, call.ToolCalls...), provider.ToolCall{ID: "unknown", Name: "lookup", Arguments: `{}`})
|
||||
fixtures := []struct {
|
||||
name string
|
||||
input, expected []provider.Message
|
||||
fallback provider.Message
|
||||
replay bool
|
||||
droppedReasoning bool
|
||||
}{
|
||||
{name: "partial-reasoning", input: []provider.Message{system, user, partial}, expected: []provider.Message{system, user}, replay: true, droppedReasoning: true},
|
||||
{name: "paired-tool-round", input: []provider.Message{system, user, call, result, partial}, expected: []provider.Message{system, user, call, result}, replay: true, droppedReasoning: true},
|
||||
{name: "partial-tool-batch", input: []provider.Message{system, user, partialBatch, result}, expected: []provider.Message{system, user}, replay: true, droppedReasoning: true},
|
||||
{name: "unreplayable-pair", input: []provider.Message{system, user, call, result}, expected: []provider.Message{system, user}, replay: false, droppedReasoning: true},
|
||||
{name: "compaction-and-partial", input: []provider.Message{system, user, summary, call, result, partial}, expected: []provider.Message{system, user, summary, call, result}, replay: true, droppedReasoning: true},
|
||||
{name: "existing-local-only", input: []provider.Message{system, user, local}, expected: []provider.Message{system, user}, replay: true},
|
||||
{name: "pre-executor-fallback", input: []provider.Message{system}, expected: []provider.Message{system, user}, fallback: user, replay: true},
|
||||
}
|
||||
for _, fixture := range fixtures {
|
||||
t.Run(fixture.name, func(t *testing.T) {
|
||||
before, err := json.Marshal(fixture.input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fallbackBefore, _ := json.Marshal(fixture.fallback)
|
||||
got := planCancelledMessages(fixture.input, 1, fixture.fallback, time.Time{}, func(provider.Message) bool { return fixture.replay }, nil)
|
||||
after, _ := json.Marshal(fixture.input)
|
||||
fallbackAfter, _ := json.Marshal(fixture.fallback)
|
||||
if !bytes.Equal(before, after) || !bytes.Equal(fallbackBefore, fallbackAfter) {
|
||||
t.Fatal("planner mutated its inputs")
|
||||
}
|
||||
expected := provider.ModelMessages(fixture.expected)
|
||||
if !reflect.DeepEqual(provider.ModelMessages(got), expected) {
|
||||
t.Fatalf("model projection differs from explicit legacy-policy fixture:\ngot=%+v\nwant=%+v", provider.ModelMessages(got), expected)
|
||||
}
|
||||
if len(got) == 0 || got[len(got)-1].InterruptedTurn == nil {
|
||||
t.Fatalf("missing recovery handoff: %+v", got)
|
||||
}
|
||||
recovery := got[len(got)-1].InterruptedTurn
|
||||
if !recovery.Pending || recovery.DroppedPartialReasoning != fixture.droppedReasoning {
|
||||
t.Fatalf("recovery=%+v", recovery)
|
||||
}
|
||||
if fixture.name == "paired-tool-round" && len(recovery.CompletedTools) != 1 {
|
||||
t.Fatalf("lost completed tool fact: %+v", recovery)
|
||||
}
|
||||
if fixture.name == "partial-tool-batch" && (len(recovery.CompletedTools) != 1 || len(recovery.UnknownTools) != 1) {
|
||||
t.Fatalf("partial batch facts: %+v", recovery)
|
||||
}
|
||||
reopenedMessages := persistTerminationPolicyModel(t, got)
|
||||
for _, adapter := range []string{"openai", "anthropic", "responses"} {
|
||||
t.Run(adapter, func(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var bodies [][]byte
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
mu.Lock()
|
||||
bodies = append(bodies, body)
|
||||
mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = io.WriteString(w, `{"error":{"message":"captured"}}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
var p provider.Provider
|
||||
switch adapter {
|
||||
case "openai":
|
||||
p, err = openai.New(provider.Config{Name: "openai", BaseURL: server.URL, Model: "cache-model", APIKey: "test"})
|
||||
case "anthropic":
|
||||
p, err = anthropic.New(provider.Config{Name: "anthropic", BaseURL: server.URL, Model: "cache-model", APIKey: "test"})
|
||||
case "responses":
|
||||
p = responses.New(responses.Config{Name: "responses", BaseURL: server.URL, Model: "cache-model", APIKey: "test", Mode: "stateless"})
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, messages := range [][]provider.Message{fixture.expected, got, reopenedMessages} {
|
||||
stream, err := p.Stream(t.Context(), provider.Request{Messages: messages, Tools: []provider.ToolSchema{{Name: "lookup", Description: "look up", Parameters: json.RawMessage(`{"type":"object","properties":{"q":{"type":"string"}}}`)}}})
|
||||
if err == nil {
|
||||
for range stream {
|
||||
}
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(bodies) != 3 {
|
||||
t.Fatalf("captured %d requests, want 3", len(bodies))
|
||||
}
|
||||
if !bytes.Equal(bodies[0], bodies[1]) {
|
||||
t.Fatalf("provider bytes changed:\nexpected:%s\nactual:%s", bodies[0], bodies[1])
|
||||
}
|
||||
if !bytes.Equal(bodies[0], bodies[2]) {
|
||||
t.Fatalf("provider bytes changed after reopen:\nexpected:%s\nactual:%s", bodies[0], bodies[2])
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func persistTerminationPolicyModel(t *testing.T, messages []provider.Message) []provider.Message {
|
||||
t.Helper()
|
||||
dir := filepath.Join(t.TempDir(), "termination-policy")
|
||||
store, err := session.CreateStore(dir, "termination-policy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload, err := json.Marshal(map[string]any{"messages": messages})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.Append(t.Context(), session.Batch{OperationID: "model-cleanup", Events: []session.Event{{Kind: "model/context-replace", Payload: payload}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.Flush(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Close(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, err := session.Open(dir, "termination-policy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reopened.Close(context.Background())
|
||||
return reopened.DeriveMessages()
|
||||
}
|
||||
|
||||
func TestTerminationPolicyFallbackOwnsImageSlice(t *testing.T) {
|
||||
fallback := provider.Message{ID: "user", Role: provider.RoleUser, Content: "inspect image", Images: []string{"original"}}
|
||||
got := planCancelledMessages(nil, 0, fallback, time.Time{}, func(provider.Message) bool { return true }, nil)
|
||||
if len(got) < 1 || len(got[0].Images) != 1 {
|
||||
t.Fatalf("fallback missing: %+v", got)
|
||||
}
|
||||
got[0].Images[0] = "changed"
|
||||
if fallback.Images[0] != "original" {
|
||||
t.Fatal("fallback image slice aliases input")
|
||||
}
|
||||
}
|
||||
@@ -57,6 +57,10 @@ type turnEventState struct {
|
||||
// candidate and consumed atomically with Runtime execution activation.
|
||||
// commitMu owns it and its queue reservation.
|
||||
pendingExecutionCommit *session.PreparedBatch
|
||||
pendingTermination *TerminationPlan
|
||||
turnMessageIDs map[string]bool
|
||||
finalizedTurn string
|
||||
terminationBoundary *terminationBoundary
|
||||
}
|
||||
|
||||
// projectVolatileTodo keeps the same event-derived projection for controllers
|
||||
@@ -244,7 +248,6 @@ func (s *turnEventSink) persistAndPublish(e event.Event) error {
|
||||
status = event.TurnWaitingUser
|
||||
case event.TurnDone:
|
||||
status = terminalTurnStatus(e)
|
||||
e.ReadCompletion = s.c.updateTurnLedgerTranscript(ledger)
|
||||
case event.TurnStatusChanged:
|
||||
// The emitter supplied the exact transition in e.Status.
|
||||
}
|
||||
@@ -290,15 +293,28 @@ func lateBusinessEvent(kind event.Kind) bool {
|
||||
}
|
||||
|
||||
func (s *turnEventSink) commitEnvelope(ledger *turnevent.Ledger, e event.Event, status event.TurnStatus) (event.Event, turnevent.Envelope, bool, error) {
|
||||
if e.Kind == event.TurnDone {
|
||||
s.c.snapshotMu.Lock()
|
||||
defer s.c.snapshotMu.Unlock()
|
||||
}
|
||||
s.c.turnEvents.commitMu.Lock()
|
||||
defer s.c.turnEvents.commitMu.Unlock()
|
||||
if s.c.discardLateTurnEvent(e) {
|
||||
slog.Info("controller: discarded late turn event", "kind", e.Kind, "turnId", e.TurnID)
|
||||
return e, turnevent.Envelope{}, false, nil
|
||||
}
|
||||
if err := s.c.appendSessionEventLocked(context.Background(), e); err != nil {
|
||||
ctx := context.Background()
|
||||
if e.Kind == event.TurnDone {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, terminationFlushTimeout)
|
||||
defer cancel()
|
||||
}
|
||||
if err := s.c.appendSessionEventLocked(ctx, e); err != nil {
|
||||
return e, turnevent.Envelope{}, false, err
|
||||
}
|
||||
if e.Kind == event.TurnDone {
|
||||
e.ReadCompletion = s.c.updateTurnLedgerTranscript(ledger)
|
||||
}
|
||||
stamped, envelope, ok, err := ledger.AppendEnvelope(e, status)
|
||||
if err != nil || !ok || stamped.Sequence == 0 {
|
||||
return stamped, envelope, ok, err
|
||||
@@ -336,6 +352,7 @@ func (s *turnEventDurableSink) EmitChecked(e event.Event) error {
|
||||
s.owner.c.mu.Lock()
|
||||
s.owner.c.enterRecoveryLocked("terminal_commit_failed")
|
||||
s.owner.c.mu.Unlock()
|
||||
s.owner.c.disarmGoalLifecycle("persistence-error")
|
||||
}
|
||||
// Async stream callers cannot observe checked errors. Fail the Turn here so
|
||||
// a poisoned WAL immediately cancels provider, prompt, and process work.
|
||||
@@ -600,6 +617,9 @@ func classifyCommitError(err error) commitFailureKind {
|
||||
if err == nil {
|
||||
return commitOK
|
||||
}
|
||||
if errors.Is(err, errTerminationDurability) {
|
||||
return commitUnexpected
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return commitLifecycle
|
||||
}
|
||||
|
||||
@@ -257,6 +257,7 @@ func (c *Controller) startTurnLocked(parent context.Context, next queuedTurn) (c
|
||||
c.turns.phase = session.RuntimeRunning
|
||||
c.turns.cancelRequested = false
|
||||
c.turns.token++
|
||||
ctx = context.WithValue(ctx, executionTokenKey{}, c.turns.token)
|
||||
c.turns.turnID = ""
|
||||
return ctx, cancel, true
|
||||
}
|
||||
@@ -281,7 +282,13 @@ func (c *Controller) queueTurnLocked(item queuedTurn) {
|
||||
}
|
||||
|
||||
func (c *Controller) signalTurnCancel() bool {
|
||||
_, _, cancelled := c.signalTurnCancelIdentity()
|
||||
return cancelled
|
||||
}
|
||||
|
||||
func (c *Controller) signalTurnCancelIdentity() (uint64, string, bool) {
|
||||
c.mu.Lock()
|
||||
token, turnID := c.turns.token, c.turns.turnID
|
||||
cancel := c.turns.cancel
|
||||
first := cancel != nil && c.turns.phase == session.RuntimeRunning
|
||||
if cancel != nil && (c.turns.phase == session.RuntimeRunning || c.turns.phase == session.RuntimeCancelling) {
|
||||
@@ -291,13 +298,13 @@ func (c *Controller) signalTurnCancel() bool {
|
||||
done := c.turns.done
|
||||
c.mu.Unlock()
|
||||
if cancel == nil {
|
||||
return false
|
||||
return token, turnID, false
|
||||
}
|
||||
cancel()
|
||||
if first {
|
||||
c.startCancellationWatchdog(done)
|
||||
}
|
||||
return true
|
||||
return token, turnID, true
|
||||
}
|
||||
|
||||
func (c *Controller) enterRecoveryLocked(reason string) {
|
||||
@@ -507,7 +514,7 @@ func (c *Controller) startCancellationWatchdog(done chan struct{}) {
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
stillRunning := c.turns.phase == session.RuntimeRunning || c.turns.phase == session.RuntimeCancelling
|
||||
stillRunning := c.turns.done == done && (c.turns.phase == session.RuntimeRunning || c.turns.phase == session.RuntimeCancelling)
|
||||
turnID := c.turns.turnID
|
||||
if stillRunning {
|
||||
c.enterRecoveryLocked("cancellation_grace_expired")
|
||||
|
||||
@@ -199,11 +199,23 @@ func TestEachStartedTurnHasOneTerminalEvent(t *testing.T) {
|
||||
|
||||
func TestStopHistoryReplaceThenSendGetsNewTurnID(t *testing.T) {
|
||||
done := make(chan event.Event, 8)
|
||||
c, _, runtime := exclusiveTestController(t, event.FuncSink(func(e event.Event) {
|
||||
c, service, runtime := exclusiveTestController(t, event.FuncSink(func(e event.Event) {
|
||||
if e.Kind == event.TurnDone {
|
||||
done <- e
|
||||
}
|
||||
}))
|
||||
if err := c.RecordSessionMessages(t.Context(), "before-stop", []provider.Message{
|
||||
{ID: "retained-question", Role: provider.RoleUser, Content: "keep this question"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.restoreExecutorFromSessionEvents()
|
||||
if _, err := runtime.Session().Flush(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := service.Query().HistoryShape(t.Context(), runtime.Ref()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started := make(chan struct{})
|
||||
c.runGuarded(func(ctx context.Context) error {
|
||||
close(started)
|
||||
@@ -221,6 +233,33 @@ func TestStopHistoryReplaceThenSendGetsNewTurnID(t *testing.T) {
|
||||
if err := c.turnEventLedgerError(); err != nil {
|
||||
t.Fatalf("cancel poisoned admission: %v", err)
|
||||
}
|
||||
// Switching after Stop reads the same durable rewrite through the history
|
||||
// index. Admission alone used to pass while these reads failed with a
|
||||
// duplicate (message_id, version) key.
|
||||
other, err := service.Create(t.Context(), session.CreateOptions{SessionID: "other"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := c.OpenSession(t.Context(), other.Ref()); err != nil {
|
||||
t.Fatalf("switch away after stop: %v", err)
|
||||
}
|
||||
if _, err := service.Query().HistoryShape(t.Context(), runtime.Ref()); err != nil {
|
||||
t.Fatalf("read stopped session after switching away: %v", err)
|
||||
}
|
||||
if _, err := c.OpenSession(t.Context(), runtime.Ref()); err != nil {
|
||||
t.Fatalf("switch back after stop: %v", err)
|
||||
}
|
||||
page, err := service.Query().ReadHistoryWindow(t.Context(), runtime.Ref(), session.HistoryWindowRequest{Anchor: "newest"})
|
||||
if err != nil || page.Status != "ready" {
|
||||
t.Fatalf("reopened stopped history: %+v, %v", page, err)
|
||||
}
|
||||
found := false
|
||||
for _, message := range page.Messages {
|
||||
found = found || message.MessageID == "retained-question"
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("stopped history lost the retained question")
|
||||
}
|
||||
secondStarted := make(chan struct{})
|
||||
if got := c.runGuarded(func(context.Context) error {
|
||||
close(secondStarted)
|
||||
|
||||
@@ -198,6 +198,9 @@ func (o *turnOrchestrator) runOrchestratedTurn(ctx context.Context, turn orchest
|
||||
return nil
|
||||
}
|
||||
startMessages := c.messageCount()
|
||||
fallback := persistedUserTurn(input, turn.raw, userImages, time.Now().UnixMilli())
|
||||
fallback.ID = userMessageID
|
||||
c.noteTerminationBoundary(fallback, !turn.synthetic)
|
||||
var marker agent.InFlightTurnMeta
|
||||
defer func() { c.finishInFlightTurn(startMessages, marker) }()
|
||||
defer c.recordDisplayForNewUser(startMessages, turn.display)
|
||||
|
||||
@@ -13,7 +13,20 @@ import (
|
||||
"reasonix/internal/provider"
|
||||
)
|
||||
|
||||
const catalogMetadataVersion = 1
|
||||
const catalogMetadataVersion = 2
|
||||
|
||||
// metadataForDurable publishes catalog metadata only for a durable prefix.
|
||||
func (s *Session) metadataForDurable(durable uint64) (catalogMetadata, bool) {
|
||||
if s == nil {
|
||||
return catalogMetadata{}, false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if durable+1 != s.next {
|
||||
return catalogMetadata{}, false
|
||||
}
|
||||
return metadataFromProjection(s.manifest, durable, s.projection), true
|
||||
}
|
||||
|
||||
const (
|
||||
MetadataReady = "ready"
|
||||
@@ -50,9 +63,11 @@ func metadataFromProjection(manifest Manifest, sequence uint64, projection Proje
|
||||
CreatedAt: manifest.CreatedAt.UTC().Format(time.RFC3339Nano), Sequence: sequence,
|
||||
Title: projection.Title, ModelRef: projection.ModelRef, ModelIdentity: projection.ModelIdentity,
|
||||
}
|
||||
for _, turn := range projection.Turns {
|
||||
if turn.EndSequence != 0 {
|
||||
metadata.Turns++
|
||||
metadata.Turns = visibleBoundaryCount(projection, true)
|
||||
for _, input := range projection.TranscriptInputs {
|
||||
if input.Preview != "" {
|
||||
metadata.Preview = input.Preview
|
||||
return metadata
|
||||
}
|
||||
}
|
||||
for _, message := range projection.Messages {
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"reasonix/internal/provider"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestClosureEventsEvidenceOrderingAndIdempotence(t *testing.T) {
|
||||
event := func(kind, payload string) Event { return Event{Kind: kind, Payload: json.RawMessage(payload)} }
|
||||
initial := Commit{Events: []Event{
|
||||
event("tool/call", `{"id":"z-unstarted","name":"lookup"}`),
|
||||
event("tool/call", `{"id":"a-started","name":"write"}`),
|
||||
event("tool/start", `{"id":"a-started","name":"write"}`),
|
||||
event("tool/call", `{"id":"done","name":"read"}`),
|
||||
event("tool/start", `{"id":"done","name":"read"}`),
|
||||
event("tool/result", `{"id":"done","name":"read","runState":"completed"}`),
|
||||
event("interaction/created", `{"id":"z-interaction"}`),
|
||||
event("interaction/created", `{"id":"a-interaction"}`),
|
||||
event("step/start", `{"id":"z-step"}`),
|
||||
event("step/start", `{"id":"a-step"}`),
|
||||
event("step/start", `{"id":"done-step"}`),
|
||||
event("step/end", `{"id":"done-step"}`),
|
||||
}}
|
||||
projection, err := Project([]Commit{initial})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _ := json.Marshal(projection)
|
||||
closures := ClosureEvents(projection, "unavailable", "runtime exited")
|
||||
after, _ := json.Marshal(projection)
|
||||
if !bytes.Equal(before, after) {
|
||||
t.Fatal("closure planner mutated input projection")
|
||||
}
|
||||
var got []string
|
||||
for _, e := range closures {
|
||||
var body struct {
|
||||
ID string `json:"id"`
|
||||
RunState provider.ToolRunState `json:"runState"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if err := json.Unmarshal(e.Payload, &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got = append(got, e.Kind+":"+body.ID)
|
||||
switch body.ID {
|
||||
case "a-started":
|
||||
if body.RunState != provider.ToolRunUnknown {
|
||||
t.Fatalf("started tool state=%q", body.RunState)
|
||||
}
|
||||
case "z-unstarted":
|
||||
if body.RunState != provider.ToolRunNotStarted {
|
||||
t.Fatalf("unstarted tool state=%q", body.RunState)
|
||||
}
|
||||
}
|
||||
if e.Kind == "interaction/resolved" && body.State != "unavailable" {
|
||||
t.Fatalf("interaction state=%q", body.State)
|
||||
}
|
||||
}
|
||||
want := []string{"tool/result:a-started", "tool/result:z-unstarted", "interaction/resolved:a-interaction", "interaction/resolved:z-interaction", "step/end:a-step", "step/end:z-step"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("closures=%v, want=%v", got, want)
|
||||
}
|
||||
second := ClosureEvents(projection, "unavailable", "runtime exited")
|
||||
if !reflect.DeepEqual(second, closures) {
|
||||
t.Fatal("repeated plan changed ordering or payload")
|
||||
}
|
||||
closed, err := Project([]Commit{initial, {Events: closures}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again := ClosureEvents(closed, "unavailable", "runtime exited"); len(again) != 0 {
|
||||
t.Fatalf("closed authority generated duplicate closures: %+v", again)
|
||||
}
|
||||
if got := ClosureEvents(Projection{}, "unavailable", "runtime exited"); len(got) != 0 {
|
||||
t.Fatalf("empty projection invented authority: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -133,6 +133,23 @@ func (r *Runtime) CommitPreparedForExecution(generation uint64, prepared Prepare
|
||||
return r.session.CommitPrepared(prepared)
|
||||
}
|
||||
|
||||
// CommitPreparedForTurn makes the owner/turn check and terminal acceptance one
|
||||
// critical section. A prepared completion cannot close a successor's turn.
|
||||
func (r *Runtime) CommitPreparedForTurn(generation uint64, turnID string, prepared PreparedBatch) (Commit, error) {
|
||||
if r == nil || generation == 0 {
|
||||
prepared.Release()
|
||||
return Commit{}, ErrStaleExecution
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
cur := r.execution.Load()
|
||||
if cur == nil || cur.generation != generation || r.session.StateSnapshot().Projection.TurnID != turnID {
|
||||
prepared.Release()
|
||||
return Commit{}, ErrStaleExecution
|
||||
}
|
||||
return r.session.CommitPrepared(prepared)
|
||||
}
|
||||
|
||||
// UnbindExecution releases only the exact generation. A superseded controller
|
||||
// cannot clear the replacement's control binding.
|
||||
func (r *Runtime) UnbindExecution(generation uint64) {
|
||||
|
||||
@@ -79,9 +79,12 @@ func forkProjectionAvailability(projection Projection, boundary uint64) ForkAvai
|
||||
// forkable yet without disabling the turns that already finished.
|
||||
func ForkTargets(projection Projection) ForkTargetSet {
|
||||
targets := make([]ForkTarget, 0, len(projection.Turns)+1)
|
||||
for index, turn := range projection.Turns {
|
||||
for _, turn := range projection.Turns {
|
||||
if projection.HiddenTurns[turn.TurnID] {
|
||||
continue
|
||||
}
|
||||
target := ForkTarget{
|
||||
TurnID: turn.TurnID, TurnNumber: index + 1,
|
||||
TurnID: turn.TurnID, TurnNumber: len(targets) + 1,
|
||||
StartSequence: turn.StartSequence, EndSequence: turn.EndSequence,
|
||||
BoundarySequence: turn.BoundarySequence,
|
||||
Status: turn.Status, MessageID: turn.MessageID,
|
||||
@@ -112,6 +115,9 @@ func ForkSequence(projection Projection, turnID string) (uint64, ForkAvailabilit
|
||||
if turnID == "" {
|
||||
return 0, "", fmt.Errorf("session: fork needs a turn id")
|
||||
}
|
||||
if projection.HiddenTurns[turnID] {
|
||||
return 0, ForkHistoryUnverifiable, nil
|
||||
}
|
||||
if projection.TurnID == turnID {
|
||||
return 0, ForkTurnOpen, nil
|
||||
}
|
||||
|
||||
@@ -240,7 +240,13 @@ func incrementHistoryIndex(ctx context.Context, dir, path, sessionID string, rev
|
||||
|
||||
func loadCurrentHistoryState(ctx context.Context, db *sql.DB) (historyBuildState, error) {
|
||||
state := historyBuildState{positions: map[string]int64{}, turns: map[string]int{}, versions: map[string]int{}}
|
||||
rows, err := db.QueryContext(ctx, `SELECT message_id,position,visible_turn,version FROM messages WHERE current=1`)
|
||||
// Versions belong to the stable message identity, including retired rows.
|
||||
// A later rewrite can restore a removed message; its next version must not
|
||||
// collide with the versions retained for older snapshots.
|
||||
rows, err := db.QueryContext(ctx, `SELECT message_id,
|
||||
COALESCE(MAX(CASE WHEN current=1 THEN position END),0),
|
||||
COALESCE(MAX(CASE WHEN current=1 THEN visible_turn END),0),MAX(version)
|
||||
FROM messages GROUP BY message_id`)
|
||||
if err != nil {
|
||||
return historyBuildState{}, err
|
||||
}
|
||||
@@ -252,7 +258,11 @@ func loadCurrentHistoryState(ctx context.Context, db *sql.DB) (historyBuildState
|
||||
_ = rows.Close()
|
||||
return historyBuildState{}, err
|
||||
}
|
||||
state.positions[id], state.turns[id], state.versions[id] = position, visibleTurn, version
|
||||
state.versions[id] = version
|
||||
if position == 0 {
|
||||
continue
|
||||
}
|
||||
state.positions[id], state.turns[id] = position, visibleTurn
|
||||
state.nextPosition = max(state.nextPosition, position)
|
||||
state.visibleTurn = max(state.visibleTurn, visibleTurn)
|
||||
}
|
||||
@@ -454,7 +464,7 @@ func flushHistoryBuildRows(ctx context.Context, tx *sql.Tx, state *historyBuildS
|
||||
if err := insertHistoryRows(ctx, tx, `INSERT OR IGNORE INTO content_refs(digest,bytes,index_digest) VALUES `, 3, state.contentRefs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := insertHistoryRows(ctx, tx, `INSERT INTO messages(message_id,version,position,event_sequence,valid_to,role,preview,inline,content_digest,content_bytes,content_index_digest,current,search_text,visible_turn) VALUES `, 14, state.messages); err != nil {
|
||||
if err := insertHistoryRows(ctx, tx, `INSERT INTO messages(message_id,version,position,event_sequence,valid_to,role,preview,inline,content_digest,content_bytes,content_index_digest,current,search_text,visible_turn,visible_user) VALUES `, 15, state.messages); err != nil {
|
||||
return err
|
||||
}
|
||||
state.transactions = state.transactions[:0]
|
||||
@@ -496,7 +506,7 @@ func insertHistoryRows(ctx context.Context, tx *sql.Tx, prefix string, columns i
|
||||
}
|
||||
|
||||
func indexMessageEvent(ctx context.Context, content *sessioncontent.Store, state *historyBuildState, event Event) error {
|
||||
if event.Kind != "message/complete" && event.Kind != "message/upsert" && event.Kind != "history/replace" && event.Kind != "legacy/import" {
|
||||
if event.Kind != "message/complete" && event.Kind != "message/upsert" && event.Kind != "message/retract" && event.Kind != "history/replace" && event.Kind != "legacy/import" {
|
||||
return nil
|
||||
}
|
||||
payload := event.Payload
|
||||
@@ -508,6 +518,22 @@ func indexMessageEvent(ctx context.Context, content *sessioncontent.Store, state
|
||||
}
|
||||
}
|
||||
switch event.Kind {
|
||||
case "message/retract":
|
||||
ids, err := retractedMessageIDs(event, payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := flushHistoryBuildRows(ctx, state.tx, state); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range ids {
|
||||
if _, err := state.statements.expire.ExecContext(ctx, event.Sequence, id); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(state.positions, id)
|
||||
delete(state.turns, id)
|
||||
}
|
||||
return renumberVisibleTurns(ctx, state, event.Sequence)
|
||||
case "message/complete", "message/upsert":
|
||||
var body struct {
|
||||
Message *provider.Message `json:"message"`
|
||||
@@ -516,22 +542,12 @@ func indexMessageEvent(ctx context.Context, content *sessioncontent.Store, state
|
||||
return damagedPayload(event, err)
|
||||
}
|
||||
return indexOneMessage(ctx, content, state, *body.Message, event.Sequence, event.Kind == "message/upsert")
|
||||
case "history/replace":
|
||||
var body struct {
|
||||
Messages []provider.Message `json:"messages"`
|
||||
case "history/replace", "legacy/import":
|
||||
messages, err := replacementEventMessages(event, payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := strictPayload(payload, &body); err != nil || body.Messages == nil {
|
||||
return damagedPayload(event, err)
|
||||
}
|
||||
return replaceIndexedMessages(ctx, content, state, body.Messages, event.Sequence)
|
||||
case "legacy/import":
|
||||
var body struct {
|
||||
Messages []provider.Message `json:"messages"`
|
||||
}
|
||||
if err := strictPayload(payload, &body); err != nil || body.Messages == nil {
|
||||
return damagedPayload(event, err)
|
||||
}
|
||||
return replaceIndexedMessages(ctx, content, state, body.Messages, event.Sequence)
|
||||
return replaceIndexedMessages(ctx, content, state, messages, event.Sequence)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -547,7 +563,8 @@ func replaceIndexedMessages(ctx context.Context, content *sessioncontent.Store,
|
||||
state.visibleTurn = 0
|
||||
state.positions = map[string]int64{}
|
||||
state.turns = map[string]int{}
|
||||
state.versions = map[string]int{}
|
||||
// Keep each identity's version watermark when replacing its visible row.
|
||||
// Retired versions remain in SQLite for fixed-snapshot readers.
|
||||
for _, message := range messages {
|
||||
if err := indexOneMessage(ctx, content, state, message, sequence, false); err != nil {
|
||||
return err
|
||||
@@ -596,7 +613,51 @@ func indexOneMessage(ctx context.Context, content *sessioncontent.Store, state *
|
||||
if err := insertContentRef(ctx, state, ref); err != nil {
|
||||
return err
|
||||
}
|
||||
state.messages = append(state.messages, []any{id, version, position, sequence, 0, string(message.Role), messagePreview(message), nil, ref.Digest, ref.Bytes, ref.IndexDigest, 1, "", visibleTurn})
|
||||
visibleUser := 0
|
||||
if agent.IsUserAuthoredTurnMessage(message) {
|
||||
visibleUser = 1
|
||||
}
|
||||
state.messages = append(state.messages, []any{id, version, position, sequence, 0, string(message.Role), messagePreview(message), nil, ref.Digest, ref.Bytes, ref.IndexDigest, 1, "", visibleTurn, visibleUser})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Re-number only rows whose visible user boundary changed. New versions keep
|
||||
// previous numbering available to fixed-snapshot cursors.
|
||||
func renumberVisibleTurns(ctx context.Context, state *historyBuildState, sequence uint64) error {
|
||||
rows, err := state.tx.QueryContext(ctx, `SELECT message_id,ordinal FROM (SELECT message_id,visible_turn,SUM(visible_user) OVER (ORDER BY position) AS ordinal FROM messages WHERE current=1) WHERE visible_turn<>ordinal`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type changedTurn struct {
|
||||
id string
|
||||
turn int
|
||||
}
|
||||
var changed []changedTurn
|
||||
for rows.Next() {
|
||||
var item changedTurn
|
||||
if err := rows.Scan(&item.id, &item.turn); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
changed = append(changed, item)
|
||||
}
|
||||
if err := errors.Join(rows.Err(), rows.Close()); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range changed {
|
||||
version := state.versions[item.id] + 1
|
||||
if _, err := state.statements.expire.ExecContext(ctx, sequence, item.id); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := state.tx.ExecContext(ctx, `INSERT INTO messages(message_id,version,position,event_sequence,valid_to,role,preview,inline,content_digest,content_bytes,content_index_digest,current,search_text,visible_turn,visible_user) SELECT message_id,?,position,?,0,role,preview,inline,content_digest,content_bytes,content_index_digest,1,search_text,?,visible_user FROM messages WHERE message_id=? ORDER BY version DESC LIMIT 1`, version, sequence, item.turn, item.id); err != nil {
|
||||
return err
|
||||
}
|
||||
state.versions[item.id], state.turns[item.id] = version, item.turn
|
||||
}
|
||||
state.visibleTurn = 0
|
||||
for _, turn := range state.turns {
|
||||
state.visibleTurn = max(state.visibleTurn, turn)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
historyIndexVersion = 5
|
||||
historyIndexVersion = 6
|
||||
HistoryPageDefaultLimit = 100
|
||||
HistoryPageMaxLimit = 500
|
||||
HistoryPageMaxBytes = 2 << 20
|
||||
@@ -182,6 +182,9 @@ var historyMigrations = []projectiondb.Migration{{Version: 1, Apply: func(ctx co
|
||||
// rebuild metadata version forces old indexes through an atomic rebuild.
|
||||
_, err := tx.ExecContext(ctx, `SELECT 1`)
|
||||
return err
|
||||
}}, {Version: 6, Apply: func(ctx context.Context, tx *sql.Tx) error {
|
||||
_, err := tx.ExecContext(ctx, `ALTER TABLE messages ADD COLUMN visible_user INTEGER NOT NULL DEFAULT 0`)
|
||||
return err
|
||||
}}}
|
||||
|
||||
type SearchHistoryHit struct {
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/provider"
|
||||
)
|
||||
|
||||
func TestHistoryReplacementPreservesMessageVersions(t *testing.T) {
|
||||
for _, incremental := range []bool{false, true} {
|
||||
name := "rebuild"
|
||||
if incremental {
|
||||
name = "incremental"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "sessions")
|
||||
service, err := NewService("local", NewFilesystemPersistence(root))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = service.CloseAll(context.Background()) })
|
||||
runtime, err := service.Create(t.Context(), CreateOptions{SessionID: "replace"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendWindowMessages(t, runtime, "retained", "removed")
|
||||
query, ref := service.Query(), runtime.Ref()
|
||||
read := func() HistoryWindowPage {
|
||||
t.Helper()
|
||||
// Prepare synchronously: the test orders commits and index reads,
|
||||
// without depending on a background preparation worker or sleeps.
|
||||
if _, _, err := query.prepareHistoryIndex(t.Context(), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
page, err := query.ReadHistoryWindow(t.Context(), ref, HistoryWindowRequest{Anchor: "newest"})
|
||||
if err != nil || page.Status != "ready" {
|
||||
t.Fatalf("read history: %+v, %v", page, err)
|
||||
}
|
||||
return page
|
||||
}
|
||||
if incremental {
|
||||
read()
|
||||
searchHistoryReady(t, query, ref, "body", "", 10)
|
||||
}
|
||||
for i, ids := range [][]string{{"retained"}, {"retained", "removed"}, {"retained", "removed"}} {
|
||||
messages := make([]provider.Message, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
messages = append(messages, provider.Message{ID: id, Role: provider.RoleUser, Content: "rewritten " + id})
|
||||
}
|
||||
payload, err := json.Marshal(map[string]any{"messages": messages, "reason": "cancel-or-recovery-rewrite", "sourceSequences": []uint64{1}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runtime.Session().Append(t.Context(), Batch{OperationID: fmt.Sprintf("replace-%d", i), Events: []Event{{Kind: "history/replace", Payload: payload}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runtime.Session().Flush(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if incremental || i == 2 {
|
||||
page := read()
|
||||
if !idsEqual(windowIDs(t, page), ids) {
|
||||
t.Fatalf("replacement %d ids: %v", i, windowIDs(t, page))
|
||||
}
|
||||
if page.Messages[0].Version != i+2 {
|
||||
t.Fatalf("retained version = %d, want %d", page.Messages[0].Version, i+2)
|
||||
}
|
||||
if page := searchHistoryReady(t, query, ref, "rewritten", "", 10); len(page.Hits) != len(ids) {
|
||||
t.Fatalf("search replacement %d: %+v", i, page)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The same derived index serves the older page API and search.
|
||||
if page := historyPageReady(t, query, ref, "", 10); len(page.Messages) != 2 {
|
||||
t.Fatalf("history page: %+v", page)
|
||||
}
|
||||
if page := searchHistoryReady(t, query, ref, "rewritten", "", 10); len(page.Hits) != 2 {
|
||||
t.Fatalf("search page: %+v", page)
|
||||
}
|
||||
if err := service.CloseAll(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Cached and rebuilt readers must both recover the existing durable
|
||||
// transcript after restart; no rewrite of user data is needed.
|
||||
for _, removeCache := range []bool{false, true} {
|
||||
if removeCache {
|
||||
if err := os.RemoveAll(filepath.Join(root, ".query-cache")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
reopened, err := NewService("local", NewFilesystemPersistence(root))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
query = reopened.Query()
|
||||
if page := read(); len(page.Messages) != 2 || page.Messages[0].Version != 4 {
|
||||
t.Fatalf("restart (cache removed=%v): %+v", removeCache, page)
|
||||
}
|
||||
if err := reopened.CloseAll(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/provider"
|
||||
)
|
||||
|
||||
type messageRetractPayload struct {
|
||||
MessageIDs []string `json:"messageIds"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
func retractedMessageIDs(event Event, payload json.RawMessage) ([]string, error) {
|
||||
var body messageRetractPayload
|
||||
if err := strictPayload(payload, &body); err != nil {
|
||||
return nil, damagedPayload(event, err)
|
||||
}
|
||||
if len(body.MessageIDs) == 0 {
|
||||
return nil, damagedPayload(event, fmt.Errorf("empty messageIds"))
|
||||
}
|
||||
seen := make(map[string]bool, len(body.MessageIDs))
|
||||
for _, id := range body.MessageIDs {
|
||||
if strings.TrimSpace(id) == "" || strings.TrimSpace(id) != id || seen[id] {
|
||||
return nil, damagedPayload(event, fmt.Errorf("invalid or duplicate message id"))
|
||||
}
|
||||
seen[id] = true
|
||||
}
|
||||
return body.MessageIDs, nil
|
||||
}
|
||||
|
||||
// Execution, recovery, history and search must decode the same durable event
|
||||
// schema. Projection-specific DTOs used to reject valid rewrite/import metadata.
|
||||
type historyReplacePayload struct {
|
||||
Messages []provider.Message `json:"messages"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Sources []uint64 `json:"sourceSequences,omitempty"`
|
||||
}
|
||||
|
||||
type legacyImportPayload struct {
|
||||
Source Source `json:"source"`
|
||||
Messages []provider.Message `json:"messages"`
|
||||
Goal json.RawMessage `json:"goal,omitempty"`
|
||||
ModelRef string `json:"modelRef,omitempty"`
|
||||
ModelIdentity string `json:"modelIdentity,omitempty"`
|
||||
}
|
||||
|
||||
func replacementEventMessages(event Event, payload json.RawMessage) ([]provider.Message, error) {
|
||||
var messages []provider.Message
|
||||
var err error
|
||||
if event.Kind == "legacy/import" {
|
||||
var body legacyImportPayload
|
||||
err = strictPayload(payload, &body)
|
||||
messages = body.Messages
|
||||
} else {
|
||||
var body historyReplacePayload
|
||||
err = strictPayload(payload, &body)
|
||||
messages = body.Messages
|
||||
}
|
||||
if err != nil || messages == nil {
|
||||
return nil, damagedPayload(event, err)
|
||||
}
|
||||
return messages, nil
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHistoryReadersAcceptLegacyImportMetadata(t *testing.T) {
|
||||
service, err := NewService("local", NewFilesystemPersistence(filepath.Join(t.TempDir(), "sessions")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = service.CloseAll(context.Background()) })
|
||||
runtime, err := service.Create(t.Context(), CreateOptions{SessionID: "imported"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Use the wire fields, not the shared decoder type: this fixture guards
|
||||
// the actual import contract independently of its implementation.
|
||||
payload := json.RawMessage(`{"source":{},"messages":[{"id":"imported-message","role":"user","content":"imported question"}],"goal":{"goal":"example"},"modelRef":"test","modelIdentity":"test-model"}`)
|
||||
if _, err := runtime.Session().Append(t.Context(), Batch{OperationID: "import", Events: []Event{{Kind: "legacy/import", Payload: payload}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runtime.Session().Flush(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
page := historyPageReady(t, service.Query(), runtime.Ref(), "", 10)
|
||||
if len(page.Messages) != 1 || page.Messages[0].MessageID != "imported-message" {
|
||||
t.Fatalf("imported history: %+v", page)
|
||||
}
|
||||
if page := searchHistoryReady(t, service.Query(), runtime.Ref(), "imported", "", 10); len(page.Hits) != 1 {
|
||||
t.Fatalf("imported search: %+v", page)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplacementEventSchemaRemainsStrict(t *testing.T) {
|
||||
for _, kind := range []string{"history/replace", "legacy/import"} {
|
||||
for _, payload := range []string{`{"messages":[],"typo":true}`, `{"messages":null}`, `{}`} {
|
||||
event := Event{Kind: kind, Payload: json.RawMessage(payload)}
|
||||
if _, err := replacementEventMessages(event, event.Payload); !errors.Is(err, ErrDamagedStore) {
|
||||
t.Fatalf("%s accepted invalid payload %s: %v", kind, payload, err)
|
||||
}
|
||||
}
|
||||
messages, err := replacementEventMessages(Event{Kind: kind}, json.RawMessage(`{"messages":[]}`))
|
||||
if err != nil || messages == nil || len(messages) != 0 {
|
||||
t.Fatalf("%s rejected valid empty history: %v", kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/provider"
|
||||
)
|
||||
|
||||
func TestMessageRetractStrictSchema(t *testing.T) {
|
||||
for _, payload := range []string{`{}`, `{"messageIds":null}`, `{"messageIds":[]}`, `{"messageIds":[""]}`, `{"messageIds":[" "]}`, `{"messageIds":["a","a"]}`, `{"messageIds":["a"],"typo":true}`, `{"messageIds":"a"}`} {
|
||||
t.Run(payload, func(t *testing.T) {
|
||||
_, err := Project([]Commit{{Events: []Event{{Kind: "message/retract", Payload: json.RawMessage(payload)}}}})
|
||||
if !errors.Is(err, ErrDamagedStore) {
|
||||
t.Fatalf("invalid retract payload accepted: %s, err=%v", payload, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageRetractProjectionAndRestore(t *testing.T) {
|
||||
messageEvent := func(kind, id string) Event {
|
||||
payload, err := json.Marshal(map[string]any{"message": provider.Message{ID: id, Role: provider.RoleUser, Content: id}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Event{Kind: kind, Payload: payload}
|
||||
}
|
||||
commits := []Commit{{Events: []Event{messageEvent("message/complete", "retained"), messageEvent("message/complete", "removed")}}, {Events: []Event{{Kind: "message/retract", Payload: json.RawMessage(`{"messageIds":["removed"],"reason":"synthetic-turn-interrupted"}`)}}}}
|
||||
check := func(want []string) {
|
||||
t.Helper()
|
||||
p, err := Project(commits)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, messages := range map[string][]provider.Message{"history": p.Messages, "model": p.ModelMessages} {
|
||||
ids := make([]string, len(messages))
|
||||
for i := range messages {
|
||||
ids[i] = messages[i].ID
|
||||
}
|
||||
if !idsEqual(ids, want) {
|
||||
t.Fatalf("%s ids=%v, want=%v", name, ids, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
check([]string{"retained"})
|
||||
commits = append(commits, commits[1])
|
||||
check([]string{"retained"})
|
||||
commits = append(commits, Commit{Events: []Event{messageEvent("message/upsert", "removed")}})
|
||||
check([]string{"retained", "removed"})
|
||||
}
|
||||
|
||||
func TestRecoveredLocalOnlyMessageClearsOnlyItsClosedReplyAnchor(t *testing.T) {
|
||||
_, _, runtime := newSourceService(t, "recovery-anchor")
|
||||
appendCompletedTurn(t, runtime, "first", "first-reply")
|
||||
appendCompletedTurn(t, runtime, "second", "second-reply")
|
||||
payload, _ := json.Marshal(map[string]any{"message": provider.Message{ID: "second-reply", Role: provider.RoleTool, LocalOnly: true}})
|
||||
if _, err := runtime.Session().Append(t.Context(), Batch{OperationID: "repair", Events: []Event{{Kind: "message/upsert", Payload: payload}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := runtime.Session().Snapshot().Projection
|
||||
if p.Turns[0].MessageID != "first-reply" || p.Turns[1].MessageID != "" {
|
||||
t.Fatalf("invalid recovery anchors: %+v", p.Turns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageRetractHistorySearchAndRestart(t *testing.T) {
|
||||
for _, incremental := range []bool{false, true} {
|
||||
name := "rebuild"
|
||||
if incremental {
|
||||
name = "incremental"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "sessions")
|
||||
service, err := NewService("local", NewFilesystemPersistence(root))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = service.CloseAll(context.Background()) })
|
||||
runtime, err := service.Create(t.Context(), CreateOptions{SessionID: "retracted"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendWindowMessages(t, runtime, "retained", "removed", "tail")
|
||||
query, ref := service.Query(), runtime.Ref()
|
||||
read := func(req HistoryWindowRequest) HistoryWindowPage {
|
||||
t.Helper()
|
||||
if _, _, err := query.prepareHistoryIndex(t.Context(), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
page, err := query.ReadHistoryWindow(t.Context(), ref, req)
|
||||
if err != nil || page.Status != "ready" {
|
||||
t.Fatalf("history: %+v, %v", page, err)
|
||||
}
|
||||
return page
|
||||
}
|
||||
var oldCursor string
|
||||
if incremental {
|
||||
oldCursor = read(HistoryWindowRequest{Anchor: "newest", Limit: 1}).OlderCursor
|
||||
if page := searchHistoryReady(t, query, ref, "body", "", 10); len(page.Hits) != 3 {
|
||||
t.Fatalf("initial search: %+v", page)
|
||||
}
|
||||
}
|
||||
appendEvent := func(op, kind, payload string) {
|
||||
t.Helper()
|
||||
if _, err := runtime.Session().Append(t.Context(), Batch{OperationID: op, Events: []Event{{Kind: kind, Payload: json.RawMessage(payload)}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runtime.Session().Flush(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
appendEvent("retract", "message/retract", `{"messageIds":["removed"],"reason":"synthetic-turn-interrupted"}`)
|
||||
page := read(HistoryWindowRequest{Anchor: "newest"})
|
||||
if !idsEqual(windowIDs(t, page), []string{"retained", "tail"}) {
|
||||
t.Fatalf("retracted ids=%v", windowIDs(t, page))
|
||||
}
|
||||
if found := searchHistoryReady(t, query, ref, "removed", "", 10); len(found.Hits) != 0 {
|
||||
t.Fatalf("retracted search: %+v", found)
|
||||
}
|
||||
if incremental {
|
||||
old := read(HistoryWindowRequest{Anchor: "cursor", Cursor: oldCursor, Limit: 10})
|
||||
if !idsEqual(windowIDs(t, old), []string{"retained", "removed"}) {
|
||||
t.Fatalf("fixed snapshot changed: %v", windowIDs(t, old))
|
||||
}
|
||||
}
|
||||
appendEvent("retract-again", "message/retract", `{"messageIds":["removed"]}`)
|
||||
read(HistoryWindowRequest{Anchor: "newest"})
|
||||
appendEvent("restore", "message/upsert", `{"message":{"id":"removed","role":"user","content":"restored body-removed"}}`)
|
||||
checkRestored := func() {
|
||||
t.Helper()
|
||||
page := read(HistoryWindowRequest{Anchor: "newest"})
|
||||
if !idsEqual(windowIDs(t, page), []string{"retained", "tail", "removed"}) {
|
||||
t.Fatalf("restored ids=%v", windowIDs(t, page))
|
||||
}
|
||||
if page.Messages[2].Version != 2 {
|
||||
t.Fatalf("restored version=%d, want=2", page.Messages[2].Version)
|
||||
}
|
||||
if found := searchHistoryReady(t, query, ref, "restored", "", 10); len(found.Hits) != 1 {
|
||||
t.Fatalf("restored search: %+v", found)
|
||||
}
|
||||
}
|
||||
checkRestored()
|
||||
if err := service.CloseAll(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, removeCache := range []bool{false, true} {
|
||||
if removeCache {
|
||||
if err := os.RemoveAll(filepath.Join(root, ".query-cache")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
reopened, err := NewService("local", NewFilesystemPersistence(root))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
query = reopened.Query()
|
||||
checkRestored()
|
||||
if err := reopened.CloseAll(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageRetractRenumbersTurnsWithoutChangingOldSnapshot(t *testing.T) {
|
||||
for _, removeID := range []string{"u1", "u2"} {
|
||||
for _, withAnswer := range []bool{false, true} {
|
||||
name := removeID
|
||||
if withAnswer {
|
||||
name += "-with-answer"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
service, err := NewService("local", NewFilesystemPersistence(filepath.Join(t.TempDir(), "sessions")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = service.CloseAll(context.Background()) })
|
||||
runtime, err := service.Create(t.Context(), CreateOptions{SessionID: "turn-renumber"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendMessage := func(op, kind, id string, role provider.Role) {
|
||||
t.Helper()
|
||||
payload, err := json.Marshal(map[string]any{"message": provider.Message{ID: id, Role: role, Content: "body-" + id}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runtime.Session().Append(t.Context(), Batch{OperationID: op, Events: []Event{{Kind: kind, Payload: payload}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"u1", "a1", "u2", "a2", "u3", "a3"} {
|
||||
role := provider.RoleUser
|
||||
if id[0] == 'a' {
|
||||
role = provider.RoleAssistant
|
||||
}
|
||||
appendMessage(id, "message/complete", id, role)
|
||||
}
|
||||
read := func(req HistoryWindowRequest) HistoryWindowPage {
|
||||
t.Helper()
|
||||
if _, err := runtime.Session().Flush(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := service.Query().prepareHistoryIndex(t.Context(), runtime.Ref()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
page, err := service.Query().ReadHistoryWindow(t.Context(), runtime.Ref(), req)
|
||||
if err != nil || page.Status != "ready" {
|
||||
t.Fatalf("page=%+v err=%v", page, err)
|
||||
}
|
||||
return page
|
||||
}
|
||||
old := read(HistoryWindowRequest{Anchor: "newest", Limit: 1})
|
||||
if old.TotalTurns != 3 || old.Messages[0].VisibleTurn != 3 {
|
||||
t.Fatalf("initial turns=%+v", old)
|
||||
}
|
||||
ids := []string{removeID}
|
||||
if withAnswer {
|
||||
ids = append(ids, "a"+removeID[1:])
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]any{"messageIds": ids})
|
||||
if _, err := runtime.Session().Append(t.Context(), Batch{OperationID: "retract-user", Events: []Event{{Kind: "message/retract", Payload: payload}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
page := read(HistoryWindowRequest{Anchor: "newest"})
|
||||
if page.TotalTurns != 2 {
|
||||
t.Fatalf("current total turns=%d want=2", page.TotalTurns)
|
||||
}
|
||||
turn := 0
|
||||
for _, m := range page.Messages {
|
||||
if m.Role == string(provider.RoleUser) {
|
||||
turn++
|
||||
}
|
||||
if m.VisibleTurn != turn {
|
||||
t.Fatalf("message %s turn=%d want=%d", m.MessageID, m.VisibleTurn, turn)
|
||||
}
|
||||
}
|
||||
past := read(HistoryWindowRequest{Anchor: "cursor", Cursor: old.OlderCursor, Limit: 10})
|
||||
if past.TotalTurns != 3 || !idsEqual(windowIDs(t, past), []string{"u1", "a1", "u2", "a2", "u3"}) {
|
||||
t.Fatalf("old snapshot changed: %+v", past)
|
||||
}
|
||||
for i, m := range past.Messages {
|
||||
if m.VisibleTurn != i/2+1 {
|
||||
t.Fatalf("old snapshot %s turn=%d", m.MessageID, m.VisibleTurn)
|
||||
}
|
||||
}
|
||||
appendMessage("restore-user", "message/upsert", removeID, provider.RoleUser)
|
||||
restored := read(HistoryWindowRequest{Anchor: "newest"})
|
||||
last := restored.Messages[len(restored.Messages)-1]
|
||||
if restored.TotalTurns != 3 || last.MessageID != removeID || last.VisibleTurn != 3 || last.Version != 2 {
|
||||
t.Fatalf("restored user=%+v total=%d", last, restored.TotalTurns)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageRetractCatalogAndRecentFollowVisibleHistory(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "sessions")
|
||||
service, err := NewService("local", NewFilesystemPersistence(root))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = service.CloseAll(context.Background()) })
|
||||
runtime, err := service.Create(t.Context(), CreateOptions{SessionID: "recent-retraction"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range []string{"first", "second"} {
|
||||
user, _ := json.Marshal(map[string]any{"message": provider.Message{ID: id, Role: provider.RoleUser, Content: id + " question"}})
|
||||
answer, _ := json.Marshal(map[string]any{"message": provider.Message{ID: id + "-answer", Role: provider.RoleAssistant, Content: id + " answer"}})
|
||||
if _, err := runtime.Session().Append(t.Context(), Batch{OperationID: id, TurnID: id + "-turn", Events: []Event{{Kind: "turn/start", Payload: json.RawMessage(`{}`)}, {Kind: "message/complete", Payload: user}, {Kind: "message/complete", Payload: answer}, {Kind: "turn/end", Payload: json.RawMessage(`{"status":"completed"}`)}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := runtime.Session().Flush(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := runtime.Session().RecentSnapshot(); got.TotalTurns != 2 {
|
||||
t.Fatalf("initial recent turns=%d", got.TotalTurns)
|
||||
}
|
||||
if _, err := runtime.Session().Append(t.Context(), Batch{OperationID: "remove-first", Events: []Event{{Kind: "message/retract", Payload: json.RawMessage(`{"messageIds":["first","first-answer"]}`)}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runtime.Session().Flush(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check := func(label string, recent RecentSnapshot, info SessionInfo) {
|
||||
t.Helper()
|
||||
if recent.TotalTurns != 1 {
|
||||
t.Errorf("%s recent total=%d want=1", label, recent.TotalTurns)
|
||||
}
|
||||
if info.Turns != 1 || info.Preview != "second question" {
|
||||
t.Errorf("%s catalog turns=%d preview=%q", label, info.Turns, info.Preview)
|
||||
}
|
||||
ids := make([]string, 0, len(recent.Entries))
|
||||
for _, m := range recent.Entries {
|
||||
ids = append(ids, m.MessageID)
|
||||
if m.VisibleTurn != 1 {
|
||||
t.Errorf("%s recent %s visible turn=%d", label, m.MessageID, m.VisibleTurn)
|
||||
}
|
||||
}
|
||||
if !idsEqual(ids, []string{"second", "second-answer"}) {
|
||||
t.Errorf("%s recent ids=%v", label, ids)
|
||||
}
|
||||
}
|
||||
info, err := service.Query().Stat(t.Context(), runtime.Ref())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check("live", runtime.Session().RecentSnapshot(), info)
|
||||
targets := ForkTargets(runtime.Session().Snapshot().Projection)
|
||||
if len(targets.Targets) != 1 || targets.Targets[0].TurnID != "second-turn" || targets.Targets[0].TurnNumber != 1 || targets.Targets[0].MessageID != "second-answer" {
|
||||
t.Fatalf("withdrawn turn remained forkable or numbering changed: %+v", targets)
|
||||
}
|
||||
if seq, reason, err := ForkSequence(runtime.Session().Snapshot().Projection, "first-turn"); err != nil || seq != 0 || reason != ForkHistoryUnverifiable {
|
||||
t.Fatalf("withdrawn turn fork: seq=%d reason=%s err=%v", seq, reason, err)
|
||||
}
|
||||
if seq, reason, err := ForkSequence(runtime.Session().Snapshot().Projection, "second-turn"); err != nil || seq == 0 || reason != ForkAvailable {
|
||||
t.Fatalf("remaining turn fork: seq=%d reason=%s err=%v", seq, reason, err)
|
||||
}
|
||||
ref := runtime.Ref()
|
||||
if err := service.Close(t.Context(), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
view, err := service.OpenSession(t.Context(), ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err = service.Query().Stat(t.Context(), ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check("cold", view.Recent, info)
|
||||
// Reopen a writer and remove every remaining input. Neither a cached
|
||||
// first-preview fallback nor old turn boundaries may resurrect the list
|
||||
// preview or leave a phantom visible turn.
|
||||
binding, err := service.EnsureExecution(t.Context(), ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
live, ok := service.Runtime(ref)
|
||||
if !ok {
|
||||
t.Fatal("reopened runtime missing")
|
||||
}
|
||||
if _, err := live.Session().Append(t.Context(), Batch{OperationID: "remove-all-inputs", Events: []Event{{Kind: "message/retract", Payload: json.RawMessage(`{"messageIds":["second","second-answer"]}`)}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := live.Session().Flush(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := binding.Release(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkEmpty := func(label string, recent RecentSnapshot, info SessionInfo) {
|
||||
t.Helper()
|
||||
if recent.TotalTurns != 0 || len(recent.Entries) != 0 {
|
||||
t.Errorf("%s recent resurrected empty history: %+v", label, recent)
|
||||
}
|
||||
if info.Turns != 0 || info.Preview != "" {
|
||||
t.Errorf("%s catalog resurrected empty history: turns=%d preview=%q", label, info.Turns, info.Preview)
|
||||
}
|
||||
}
|
||||
info, err = service.Query().Stat(t.Context(), ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkEmpty("empty-live", live.Session().RecentSnapshot(), info)
|
||||
checkNoForks := func(projection Projection) {
|
||||
t.Helper()
|
||||
if targets := ForkTargets(projection); len(targets.Targets) != 0 {
|
||||
t.Fatalf("empty history exposes fork targets: %+v", targets)
|
||||
}
|
||||
for _, id := range []string{"first-turn", "second-turn"} {
|
||||
if seq, reason, err := ForkSequence(projection, id); err != nil || seq != 0 || reason != ForkHistoryUnverifiable {
|
||||
t.Fatalf("empty history fork %s: seq=%d reason=%s err=%v", id, seq, reason, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
checkNoForks(live.Session().Snapshot().Projection)
|
||||
if err := service.Close(t.Context(), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, removeCache := range []bool{false, true} {
|
||||
label := "empty-cached"
|
||||
if removeCache {
|
||||
label = "empty-rebuilt"
|
||||
for _, dir := range []string{filepath.Join(root, ".query-cache", ref.SessionID), recoveryCacheDir(filepath.Join(root, ref.SessionID))} {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// EnsureExecution deterministically completes checkpoint rebuild rather
|
||||
// than racing a cold reader's background preparation notification.
|
||||
binding, err := service.EnsureExecution(t.Context(), ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, ok := service.Runtime(ref)
|
||||
if !ok {
|
||||
t.Fatal("reopened runtime missing")
|
||||
}
|
||||
info, err := service.Query().Stat(t.Context(), ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkEmpty(label, reopened.Session().RecentSnapshot(), info)
|
||||
checkNoForks(reopened.Session().Snapshot().Projection)
|
||||
if err := binding.Release(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := service.Close(t.Context(), ref); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cold, err := service.OpenSession(t.Context(), ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err = service.Query().Stat(t.Context(), ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkEmpty(label+"-cold", cold.Recent, info)
|
||||
}
|
||||
}
|
||||
@@ -543,7 +543,7 @@ func readManifest(path string) (Manifest, error) {
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
if m.SchemaVersion != SchemaVersion || m.Codec != Codec || m.StorageRevision != StorageRevision {
|
||||
if !currentStoredManifest(m) {
|
||||
return Manifest{}, fmt.Errorf("%w: manifest schema or codec", ErrUnsupportedVersion)
|
||||
}
|
||||
return m, nil
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/event"
|
||||
@@ -13,6 +14,8 @@ import (
|
||||
)
|
||||
|
||||
type Projection struct {
|
||||
TranscriptInputs []transcriptInput
|
||||
HiddenTurns map[string]bool
|
||||
CommittedSequence uint64
|
||||
TurnID string
|
||||
TurnStatus event.TurnStatus
|
||||
@@ -31,6 +34,8 @@ type Projection struct {
|
||||
TodoWritten bool
|
||||
Interactions map[string]string
|
||||
ActiveTools map[string]string
|
||||
StartedTools map[string]bool
|
||||
ActiveSteps map[string]bool
|
||||
Recovery *event.RecoveryStatus
|
||||
PlanState json.RawMessage
|
||||
GoalState json.RawMessage
|
||||
@@ -59,7 +64,7 @@ type TurnBoundary struct {
|
||||
}
|
||||
|
||||
var ProjectionKinds = map[string]bool{
|
||||
"message/complete": true, "message/upsert": true, "assistant/attempt": true,
|
||||
"message/complete": true, "message/upsert": true, "message/retract": true, "assistant/attempt": true,
|
||||
"tool/call": true, "tool/start": true, "tool/result": true,
|
||||
"turn/start": true, "turn/end": true, "step/start": true, "step/end": true,
|
||||
"todo/write": true, "interaction/created": true, "interaction/resolved": true,
|
||||
@@ -87,12 +92,26 @@ func Project(commits []Commit) (Projection, error) {
|
||||
}
|
||||
|
||||
func applyProjectionCommit(projection *Projection, commit Commit) error {
|
||||
initializeProjectionMaps(projection)
|
||||
return applyProjectionEvents(projection, commit)
|
||||
}
|
||||
|
||||
func initializeProjectionMaps(projection *Projection) {
|
||||
if projection.Interactions == nil {
|
||||
projection.Interactions = map[string]string{}
|
||||
}
|
||||
if projection.ActiveTools == nil {
|
||||
projection.ActiveTools = map[string]string{}
|
||||
}
|
||||
if projection.StartedTools == nil {
|
||||
projection.StartedTools = map[string]bool{}
|
||||
}
|
||||
if projection.ActiveSteps == nil {
|
||||
projection.ActiveSteps = map[string]bool{}
|
||||
}
|
||||
}
|
||||
|
||||
func applyProjectionEvents(projection *Projection, commit Commit) error {
|
||||
closedBefore := len(projection.Turns)
|
||||
for _, ev := range commit.Events {
|
||||
projection.CommittedSequence = ev.Sequence
|
||||
@@ -104,6 +123,8 @@ func applyProjectionCommit(projection *Projection, commit Commit) error {
|
||||
err = projectMessageComplete(projection, commit, ev)
|
||||
case "message/upsert":
|
||||
err = projectMessageUpsert(projection, commit, ev)
|
||||
case "message/retract":
|
||||
err = projectMessageRetract(projection, commit, ev)
|
||||
case "assistant/attempt":
|
||||
err = projectAssistantAttempt(projection, commit, ev)
|
||||
case "history/replace":
|
||||
@@ -146,6 +167,7 @@ func applyProjectionCommit(projection *Projection, commit Commit) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
applyTranscriptMetadata(projection, commit, ev)
|
||||
}
|
||||
// turn/end can be followed by more events in the same atomic commit. Only
|
||||
// after the whole commit is projected do we know whether its cut leaves a
|
||||
@@ -157,13 +179,7 @@ func applyProjectionCommit(projection *Projection, commit Commit) error {
|
||||
}
|
||||
|
||||
func projectLegacyImport(projection *Projection, commit Commit, ev Event) error {
|
||||
var body struct {
|
||||
Source Source `json:"source"`
|
||||
Messages []provider.Message `json:"messages"`
|
||||
Goal json.RawMessage `json:"goal,omitempty"`
|
||||
ModelRef string `json:"modelRef,omitempty"`
|
||||
ModelIdentity string `json:"modelIdentity,omitempty"`
|
||||
}
|
||||
var body legacyImportPayload
|
||||
if err := strictPayload(ev.Payload, &body); err != nil || body.Messages == nil {
|
||||
return damagedPayload(ev, err)
|
||||
}
|
||||
@@ -228,10 +244,33 @@ func projectMessageUpsert(projection *Projection, commit Commit, ev Event) error
|
||||
// append case is retained for explicitly-created records.
|
||||
projection.ModelMessages = append(projection.ModelMessages, visible[0])
|
||||
}
|
||||
if projection.CurrentTurnMessageID == body.Message.ID && (body.Message.LocalOnly || body.Message.Role != provider.RoleAssistant) {
|
||||
projection.CurrentTurnMessageID = ""
|
||||
}
|
||||
projection.recordTurnReply(*body.Message)
|
||||
return nil
|
||||
}
|
||||
|
||||
func projectMessageRetract(projection *Projection, commit Commit, ev Event) error {
|
||||
ids, err := retractedMessageIDs(ev, ev.Payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
removed := make(map[string]bool, len(ids))
|
||||
for _, id := range ids {
|
||||
removed[id] = true
|
||||
}
|
||||
filter := func(messages []provider.Message) []provider.Message {
|
||||
return slices.DeleteFunc(messages, func(message provider.Message) bool { return removed[message.ID] })
|
||||
}
|
||||
projection.Messages = filter(projection.Messages)
|
||||
projection.ModelMessages = filter(projection.ModelMessages)
|
||||
if removed[projection.CurrentTurnMessageID] {
|
||||
projection.CurrentTurnMessageID = ""
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func projectAssistantAttempt(projection *Projection, commit Commit, ev Event) error {
|
||||
var body struct {
|
||||
ID string `json:"id"`
|
||||
@@ -248,11 +287,7 @@ func projectAssistantAttempt(projection *Projection, commit Commit, ev Event) er
|
||||
}
|
||||
|
||||
func projectHistoryReplace(projection *Projection, commit Commit, ev Event) error {
|
||||
var body struct {
|
||||
Messages []provider.Message `json:"messages"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Sources []uint64 `json:"sourceSequences,omitempty"`
|
||||
}
|
||||
var body historyReplacePayload
|
||||
if err := strictPayload(ev.Payload, &body); err != nil || body.Messages == nil {
|
||||
return damagedPayload(ev, err)
|
||||
}
|
||||
@@ -262,11 +297,7 @@ func projectHistoryReplace(projection *Projection, commit Commit, ev Event) erro
|
||||
}
|
||||
|
||||
func projectModelContextReplace(projection *Projection, commit Commit, ev Event) error {
|
||||
var body struct {
|
||||
Messages []provider.Message `json:"messages"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Sources []uint64 `json:"sourceSequences,omitempty"`
|
||||
}
|
||||
var body historyReplacePayload
|
||||
if err := strictPayload(ev.Payload, &body); err != nil || body.Messages == nil {
|
||||
return damagedPayload(ev, err)
|
||||
}
|
||||
@@ -329,6 +360,11 @@ func projectStepStart(projection *Projection, commit Commit, ev Event) error {
|
||||
if err := strictPayload(ev.Payload, &body); err != nil || body.ID == "" {
|
||||
return damagedPayload(ev, err)
|
||||
}
|
||||
if ev.Kind == "step/start" {
|
||||
projection.ActiveSteps[body.ID] = true
|
||||
} else {
|
||||
delete(projection.ActiveSteps, body.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -368,6 +404,7 @@ func projectToolCall(projection *Projection, commit Commit, ev Event) error {
|
||||
if err := strictPayload(ev.Payload, &body); err != nil || body.ID == "" || body.Name == "" {
|
||||
return damagedPayload(ev, err)
|
||||
}
|
||||
projection.ActiveTools[body.ID] = body.Name
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -380,6 +417,7 @@ func projectToolStart(projection *Projection, commit Commit, ev Event) error {
|
||||
return damagedPayload(ev, err)
|
||||
}
|
||||
projection.ActiveTools[body.ID] = body.Name
|
||||
projection.StartedTools[body.ID] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -425,6 +463,7 @@ func projectToolResult(projection *Projection, commit Commit, ev Event) error {
|
||||
return damagedPayload(ev, err)
|
||||
}
|
||||
delete(projection.ActiveTools, body.ID)
|
||||
delete(projection.StartedTools, body.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -541,6 +580,8 @@ func projectionMessageIndex(messages []provider.Message, id string) int {
|
||||
}
|
||||
|
||||
func cloneProjection(projection Projection) Projection {
|
||||
projection.TranscriptInputs = append([]transcriptInput(nil), projection.TranscriptInputs...)
|
||||
projection.HiddenTurns = maps.Clone(projection.HiddenTurns)
|
||||
projection.Messages = append([]provider.Message(nil), projection.Messages...)
|
||||
projection.ModelMessages = append([]provider.Message(nil), projection.ModelMessages...)
|
||||
projection.Turns = append([]TurnBoundary(nil), projection.Turns...)
|
||||
@@ -551,6 +592,8 @@ func cloneProjection(projection Projection) Projection {
|
||||
tools := make(map[string]string, len(projection.ActiveTools))
|
||||
maps.Copy(tools, projection.ActiveTools)
|
||||
projection.ActiveTools = tools
|
||||
projection.StartedTools = maps.Clone(projection.StartedTools)
|
||||
projection.ActiveSteps = maps.Clone(projection.ActiveSteps)
|
||||
if projection.Recovery != nil {
|
||||
recovery := *projection.Recovery
|
||||
projection.Recovery = &recovery
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"sort"
|
||||
|
||||
"reasonix/internal/event"
|
||||
"reasonix/internal/provider"
|
||||
)
|
||||
|
||||
// RecoverInterrupted closes persisted runtime authority that cannot survive a
|
||||
@@ -30,32 +31,10 @@ func (s *Session) RecoverInterrupted(ctx context.Context) (Commit, bool, error)
|
||||
if turnID == "" {
|
||||
return Commit{}, false, nil
|
||||
}
|
||||
events := make([]Event, 0, len(snapshot.Projection.ActiveTools)+len(snapshot.Projection.Interactions)+1)
|
||||
toolIDs := make([]string, 0, len(snapshot.Projection.ActiveTools))
|
||||
for id := range snapshot.Projection.ActiveTools {
|
||||
toolIDs = append(toolIDs, id)
|
||||
}
|
||||
sort.Strings(toolIDs)
|
||||
for _, id := range toolIDs {
|
||||
payload, _ := json.Marshal(map[string]any{"id": id, "name": snapshot.Projection.ActiveTools[id], "state": "result_unknown", "error": "previous runtime exited before recording a result"})
|
||||
events = append(events, Event{Kind: "tool/result", Payload: payload})
|
||||
}
|
||||
requestIDs := make([]string, 0, len(snapshot.Projection.Interactions))
|
||||
for id := range snapshot.Projection.Interactions {
|
||||
requestIDs = append(requestIDs, id)
|
||||
}
|
||||
sort.Strings(requestIDs)
|
||||
for _, id := range requestIDs {
|
||||
payload, _ := json.Marshal(map[string]any{"id": id, "state": "unavailable"})
|
||||
events = append(events, Event{Kind: "interaction/resolved", Payload: payload})
|
||||
}
|
||||
// turn/end uses a strict stable payload; diagnostic detail remains outside
|
||||
// the required event until the codec explicitly versions that field.
|
||||
events := ClosureEvents(snapshot.Projection, "unavailable", "previous runtime exited before recording a result")
|
||||
terminal, _ := json.Marshal(map[string]any{"status": event.TurnInterrupted})
|
||||
events = append(events, Event{Kind: "turn/end", Payload: terminal})
|
||||
manifest := s.Manifest()
|
||||
operationID := fmt.Sprintf("restart-recovery:%d:%s", manifest.WriterGeneration, turnID)
|
||||
commit, err := s.Append(ctx, Batch{OperationID: operationID, TurnID: turnID, Events: events})
|
||||
commit, err := s.Append(ctx, Batch{OperationID: "turn-finalize:" + turnID, TurnID: turnID, Events: events})
|
||||
if err != nil {
|
||||
return Commit{}, false, err
|
||||
}
|
||||
@@ -64,3 +43,43 @@ func (s *Session) RecoverInterrupted(ctx context.Context) (Commit, bool, error)
|
||||
}
|
||||
return commit, true, nil
|
||||
}
|
||||
|
||||
// ClosureEvents derives deterministic terminal facts without executing tools
|
||||
// or changing the provider workset. Live termination and restart share it.
|
||||
func ClosureEvents(projection Projection, interactionState, detail string) []Event {
|
||||
events := make([]Event, 0, len(projection.ActiveTools)+len(projection.Interactions)+len(projection.ActiveSteps))
|
||||
toolIDs := make([]string, 0, len(projection.ActiveTools))
|
||||
for id := range projection.ActiveTools {
|
||||
toolIDs = append(toolIDs, id)
|
||||
}
|
||||
sort.Strings(toolIDs)
|
||||
for _, id := range toolIDs {
|
||||
state := provider.ToolRunNotStarted
|
||||
legacyState := "not_started"
|
||||
if projection.StartedTools[id] {
|
||||
state = provider.ToolRunUnknown
|
||||
legacyState = "result_unknown"
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]any{"id": id, "name": projection.ActiveTools[id], "state": legacyState, "runState": state, "error": detail})
|
||||
events = append(events, Event{Kind: "tool/result", Payload: payload})
|
||||
}
|
||||
requestIDs := make([]string, 0, len(projection.Interactions))
|
||||
for id := range projection.Interactions {
|
||||
requestIDs = append(requestIDs, id)
|
||||
}
|
||||
sort.Strings(requestIDs)
|
||||
for _, id := range requestIDs {
|
||||
payload, _ := json.Marshal(map[string]any{"id": id, "state": interactionState})
|
||||
events = append(events, Event{Kind: "interaction/resolved", Payload: payload})
|
||||
}
|
||||
stepIDs := make([]string, 0, len(projection.ActiveSteps))
|
||||
for id := range projection.ActiveSteps {
|
||||
stepIDs = append(stepIDs, id)
|
||||
}
|
||||
sort.Strings(stepIDs)
|
||||
for _, id := range stepIDs {
|
||||
payload, _ := json.Marshal(map[string]any{"id": id})
|
||||
events = append(events, Event{Kind: "step/end", Payload: payload})
|
||||
}
|
||||
return events
|
||||
}
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
// commit. Version 1 checkpoints decode that field as empty, so accepting one
|
||||
// would incorrectly mark previously completed turns unverifiable until a full
|
||||
// replay. Rejecting the old projection rebuilds it from the durable log.
|
||||
const recoveryProjectionVersion = 2
|
||||
const recoveryProjectionVersion = 3
|
||||
|
||||
const (
|
||||
recoveryFormatVersion = 1
|
||||
@@ -445,7 +445,7 @@ func (s *recoveryStore) publish(ctx context.Context, checkpoint recoveryCheckpoi
|
||||
StorageGeneration: checkpoint.StorageGeneration, DurableSequence: checkpoint.DurableSequence,
|
||||
Title: checkpoint.Projection.Title,
|
||||
ModelRef: checkpoint.Projection.ModelRef, ModelIdentity: checkpoint.Projection.ModelIdentity,
|
||||
TotalTurns: len(checkpoint.Projection.Turns),
|
||||
TotalTurns: visibleBoundaryCount(checkpoint.Projection, false),
|
||||
}
|
||||
recent.Entries, err = buildRecentEntries(ctx, s.sessionDir, checkpoint.RecentMessages, checkpoint.DurableSequence, recent.TotalTurns)
|
||||
if err != nil {
|
||||
@@ -677,7 +677,7 @@ func applyRecentCommit(messages *[]provider.Message, commit Commit) error {
|
||||
recent.Events = nil
|
||||
for _, event := range commit.Events {
|
||||
switch event.Kind {
|
||||
case "message/complete", "message/upsert", "history/replace", "legacy/import":
|
||||
case "message/complete", "message/upsert", "message/retract", "history/replace", "legacy/import":
|
||||
recent.Events = append(recent.Events, event)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
"reasonix/internal/sessioncontent"
|
||||
)
|
||||
|
||||
const searchIndexVersion = 1
|
||||
const searchIndexVersion = 2
|
||||
|
||||
var searchMigrations = []projectiondb.Migration{{Version: 1, Apply: func(ctx context.Context, tx *sql.Tx) error {
|
||||
for _, statement := range []string{
|
||||
@@ -336,7 +336,7 @@ func populateSearchIndex(ctx context.Context, dir string, db *sql.DB, startOffse
|
||||
}
|
||||
|
||||
func indexSearchEvent(ctx context.Context, tx *sql.Tx, content *sessioncontent.Store, state *searchBuildState, event Event) error {
|
||||
if event.Kind != "message/complete" && event.Kind != "message/upsert" && event.Kind != "history/replace" && event.Kind != "legacy/import" {
|
||||
if event.Kind != "message/complete" && event.Kind != "message/upsert" && event.Kind != "message/retract" && event.Kind != "history/replace" && event.Kind != "legacy/import" {
|
||||
return nil
|
||||
}
|
||||
payload := event.Payload
|
||||
@@ -348,6 +348,18 @@ func indexSearchEvent(ctx context.Context, tx *sql.Tx, content *sessioncontent.S
|
||||
}
|
||||
}
|
||||
switch event.Kind {
|
||||
case "message/retract":
|
||||
ids, err := retractedMessageIDs(event, payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range ids {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE documents SET current=0,valid_to=? WHERE message_id=? AND current=1`, event.Sequence, id); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(state.positions, id)
|
||||
}
|
||||
return nil
|
||||
case "message/complete", "message/upsert":
|
||||
var body struct {
|
||||
Message *provider.Message `json:"message"`
|
||||
@@ -357,18 +369,16 @@ func indexSearchEvent(ctx context.Context, tx *sql.Tx, content *sessioncontent.S
|
||||
}
|
||||
return indexSearchMessage(ctx, tx, state, *body.Message, event.Sequence, event.Kind == "message/upsert")
|
||||
case "history/replace", "legacy/import":
|
||||
var body struct {
|
||||
Messages []provider.Message `json:"messages"`
|
||||
}
|
||||
if err := strictPayload(payload, &body); err != nil || body.Messages == nil {
|
||||
return damagedPayload(event, err)
|
||||
messages, err := replacementEventMessages(event, payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE documents SET current=0,valid_to=? WHERE current=1`, event.Sequence); err != nil {
|
||||
return err
|
||||
}
|
||||
state.positions = map[string]int64{}
|
||||
state.nextPosition = 0
|
||||
for _, message := range body.Messages {
|
||||
for _, message := range messages {
|
||||
if err := indexSearchMessage(ctx, tx, state, message, event.Sequence, false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -156,6 +156,12 @@ func (s *Session) PrepareBatchContext(ctx context.Context, operationID string, b
|
||||
for i := range events {
|
||||
event := &events[i]
|
||||
event.Kind = strings.TrimSpace(event.Kind)
|
||||
if event.Kind == "message/retract" {
|
||||
if event.Optional {
|
||||
return PreparedBatch{}, fmt.Errorf("message/retract must be required")
|
||||
}
|
||||
event.Required = true
|
||||
}
|
||||
if event.Kind == "" {
|
||||
return PreparedBatch{}, fmt.Errorf("session: events[%d].kind is required", i)
|
||||
}
|
||||
@@ -422,9 +428,6 @@ func (s *Session) CatalogMetadata() catalogMetadata {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
metadata := metadataFromProjection(s.manifest, s.next-1, s.projection)
|
||||
if metadata.Preview == "" {
|
||||
metadata.Preview = s.catalogPreview
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
@@ -486,7 +489,7 @@ func (s *Session) RecentSnapshot() RecentSnapshot {
|
||||
Version: recoveryFormatVersion, SessionID: s.id, StorageGeneration: s.storageGeneration,
|
||||
DurableSequence: durable,
|
||||
Title: s.projection.Title, ModelRef: s.projection.ModelRef, ModelIdentity: s.projection.ModelIdentity,
|
||||
TotalTurns: len(s.projection.Turns),
|
||||
TotalTurns: visibleBoundaryCount(s.projection, false),
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if sessionDir != "" {
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Seed only revision-1 events, then label the fixture with its original
|
||||
// revision. Do not use this helper to downgrade real session data.
|
||||
func revisionOneFixture(t *testing.T) (string, string, string, []byte, []byte) {
|
||||
t.Helper()
|
||||
root, id := filepath.Join(t.TempDir(), "sessions"), "revision-one"
|
||||
persistence := NewFilesystemPersistence(root)
|
||||
writer, err := persistence.Create(CreateOptions{SessionID: id})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := writer.Append(t.Context(), Batch{OperationID: "old-message", Events: []Event{{Kind: "message/complete", Payload: []byte(`{"message":{"id":"old","role":"user","content":"preserve original bytes"}}`)}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := writer.Flush(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writer.Close(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := filepath.Join(root, id)
|
||||
manifestPath := filepath.Join(dir, "manifest.json")
|
||||
manifest, err := readStoredManifest(manifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest.StorageRevision = 1
|
||||
if err := writeManifestFile(manifestPath, manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifestBytes, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
logBytes, err := os.ReadFile(filepath.Join(dir, currentLogName))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return root, id, dir, manifestBytes, logBytes
|
||||
}
|
||||
|
||||
func assertRevisionFixtureBytes(t *testing.T, dir string, manifestBytes, logBytes []byte) {
|
||||
t.Helper()
|
||||
for name, want := range map[string][]byte{"manifest.json": manifestBytes, currentLogName: logBytes} {
|
||||
got, err := os.ReadFile(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("%s bytes changed", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageRevisionReadOnlyDoesNotUpgrade(t *testing.T) {
|
||||
root, id, dir, manifestBytes, logBytes := revisionOneFixture(t)
|
||||
persistence := NewFilesystemPersistence(root)
|
||||
reader, err := persistence.Open(id, ReadOnly)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := reader.Read(t.Context(), 0, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := reader.Close(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertRevisionFixtureBytes(t, dir, manifestBytes, logBytes)
|
||||
}
|
||||
|
||||
func TestStorageRevisionWriterUpgradesWithoutRewritingLog(t *testing.T) {
|
||||
root, id, dir, _, logBytes := revisionOneFixture(t)
|
||||
writer, err := NewFilesystemPersistence(root).Open(id, ReadWrite)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writer.Close(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest, err := readStoredManifest(filepath.Join(dir, "manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if manifest.StorageRevision != 2 {
|
||||
t.Fatalf("revision=%d, want=2", manifest.StorageRevision)
|
||||
}
|
||||
got, err := os.ReadFile(filepath.Join(dir, currentLogName))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, logBytes) {
|
||||
t.Fatal("upgrade rewrote original event bytes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageRevisionDamagedLogDoesNotUpgrade(t *testing.T) {
|
||||
root, id, dir, manifestBytes, logBytes := revisionOneFixture(t)
|
||||
logBytes[0] ^= 0xff
|
||||
if err := os.WriteFile(filepath.Join(dir, currentLogName), logBytes, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writer, err := NewFilesystemPersistence(root).Open(id, ReadWrite)
|
||||
if err == nil {
|
||||
_ = writer.Close(context.Background())
|
||||
t.Fatal("damaged log was opened for writing")
|
||||
}
|
||||
assertRevisionFixtureBytes(t, dir, manifestBytes, logBytes)
|
||||
}
|
||||
|
||||
func TestStorageRevisionExternalLeasePreventsUpgrade(t *testing.T) {
|
||||
root, id, dir, manifestBytes, logBytes := revisionOneFixture(t)
|
||||
release, err := acquireSessionWriter(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer release()
|
||||
writer, err := NewFilesystemPersistence(root).Open(id, ReadWrite)
|
||||
if writer != nil {
|
||||
_ = writer.Close(context.Background())
|
||||
}
|
||||
if !errors.Is(err, ErrWriterOwned) {
|
||||
t.Fatalf("open while owned: %v", err)
|
||||
}
|
||||
assertRevisionFixtureBytes(t, dir, manifestBytes, logBytes)
|
||||
}
|
||||
|
||||
func TestStorageRevisionUnknownRejectedByReadersAndWriters(t *testing.T) {
|
||||
for _, mode := range []AccessMode{ReadOnly, ReadWrite} {
|
||||
root, id, dir, _, logBytes := revisionOneFixture(t)
|
||||
manifestPath := filepath.Join(dir, "manifest.json")
|
||||
manifest, err := readStoredManifest(manifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest.StorageRevision = 999
|
||||
if err := writeManifestFile(manifestPath, manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifestBytes, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handle, err := NewFilesystemPersistence(root).Open(id, mode)
|
||||
if handle != nil {
|
||||
_ = handle.Close(context.Background())
|
||||
}
|
||||
if !errors.Is(err, ErrUnsupportedVersion) {
|
||||
t.Fatalf("mode=%v unknown revision: %v", mode, err)
|
||||
}
|
||||
assertRevisionFixtureBytes(t, dir, manifestBytes, logBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageRevisionManifestPublishFailureReleasesWriter(t *testing.T) {
|
||||
_, id, dir, manifestBytes, logBytes := revisionOneFixture(t)
|
||||
manifestPath := filepath.Join(dir, "manifest.json")
|
||||
backupPath := filepath.Join(dir, "manifest.before-failure.json")
|
||||
// ObserveRecovery is called after validation and before manifest publication.
|
||||
// A directory at the rename destination deterministically rejects the atomic
|
||||
// publish on every platform, without timing or permission assumptions.
|
||||
writer, err := OpenWithOptions(dir, id, OpenOptions{ObserveRecovery: func(RecoveryOpenStats) {
|
||||
if err := os.Rename(manifestPath, backupPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Mkdir(manifestPath, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}})
|
||||
if writer != nil {
|
||||
_ = writer.Close(context.Background())
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("manifest publish unexpectedly succeeded")
|
||||
}
|
||||
got, err := os.ReadFile(backupPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, manifestBytes) {
|
||||
t.Fatal("failed upgrade changed old manifest bytes")
|
||||
}
|
||||
got, err = os.ReadFile(filepath.Join(dir, currentLogName))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, logBytes) {
|
||||
t.Fatal("failed upgrade changed log bytes")
|
||||
}
|
||||
if err := os.Remove(manifestPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(backupPath, manifestPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The failed open must release its lease, so a retry can finish the upgrade.
|
||||
retry, err := Open(dir, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := retry.Close(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest, err := readStoredManifest(manifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if manifest.StorageRevision != 2 {
|
||||
t.Fatalf("retry revision=%d", manifest.StorageRevision)
|
||||
}
|
||||
}
|
||||
+15
-25
@@ -34,7 +34,7 @@ const (
|
||||
// StorageRevision distinguishes the final v4 layout from unpublished v4
|
||||
// drafts. Physical layout changes are migration boundaries even when the
|
||||
// logical codec remains v4.
|
||||
StorageRevision = 1
|
||||
StorageRevision = 2
|
||||
// Codec identifies the current framed linear session format. Earlier linear
|
||||
// and prototype stores are immutable migration inputs.
|
||||
Codec = V4Codec
|
||||
@@ -437,6 +437,9 @@ func openExistingHandle(dir, sessionID string, opts OpenOptions) (*Store, error)
|
||||
startup.operations = map[string]operationRecord{}
|
||||
}
|
||||
}
|
||||
// Upgrade only after the exclusive writer validated the complete log. Old
|
||||
// readers reject revision 2 before using caches or accepting new writes.
|
||||
manifest.StorageRevision = StorageRevision
|
||||
manifest.WriterGeneration++
|
||||
if err := writeManifestFile(manifestPath, manifest); err != nil {
|
||||
return failRecovery(err)
|
||||
@@ -563,13 +566,14 @@ func loadBoundedStartupSessionState(ctx context.Context, dir string, file *os.Fi
|
||||
recent.Events = nil
|
||||
for _, event := range commit.Events {
|
||||
switch event.Kind {
|
||||
case "message/complete", "message/upsert", "history/replace", "legacy/import":
|
||||
case "message/complete", "message/upsert", "message/retract", "history/replace", "legacy/import":
|
||||
resolved, err := resolveProjectionEvent(ctx, content, event)
|
||||
if err != nil {
|
||||
projectionErr = err
|
||||
return false
|
||||
}
|
||||
recent.Events = append(recent.Events, resolved)
|
||||
applyTranscriptMetadata(&state.projection, commit, resolved)
|
||||
}
|
||||
}
|
||||
if err := applyRecentCommit(&state.recentMessages, recent); err != nil {
|
||||
@@ -589,9 +593,11 @@ func loadBoundedStartupSessionState(ctx context.Context, dir string, file *os.Fi
|
||||
return nil, 0, false, projectionErr
|
||||
}
|
||||
if sawModelEvent {
|
||||
inputs, hidden := state.projection.TranscriptInputs, state.projection.HiddenTurns
|
||||
if err := loadCurrentModelProjection(ctx, file, content, state, modelOffset, modelSequence); err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
state.projection.TranscriptInputs, state.projection.HiddenTurns = inputs, hidden
|
||||
}
|
||||
state.projection.Messages = nil
|
||||
state.projection.CommittedSequence = state.durable
|
||||
@@ -640,7 +646,7 @@ func resolveProjectionEvent(ctx context.Context, content *sessioncontent.Store,
|
||||
|
||||
func modelProjectionEvent(kind string) bool {
|
||||
switch kind {
|
||||
case "message/complete", "message/upsert", "history/replace", "model/context-replace", "compaction", "legacy/import":
|
||||
case "message/complete", "message/upsert", "message/retract", "history/replace", "model/context-replace", "compaction", "legacy/import":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
@@ -683,27 +689,6 @@ func bindSession(handle *Store, opts OpenOptions) (*Session, error) {
|
||||
return session, nil
|
||||
}
|
||||
|
||||
// metadataForDurable rebuilds the list projection for the catalog cache once
|
||||
// the accepted prefix is fully durable.
|
||||
func (s *Session) metadataForDurable(durable uint64) (catalogMetadata, bool) {
|
||||
if s == nil {
|
||||
return catalogMetadata{}, false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if durable+1 != s.next {
|
||||
return catalogMetadata{}, false
|
||||
}
|
||||
metadata := metadataFromProjection(s.manifest, durable, s.projection)
|
||||
if s.catalogPreview == "" && metadata.Preview != "" {
|
||||
s.catalogPreview = metadata.Preview
|
||||
}
|
||||
if metadata.Preview == "" {
|
||||
metadata.Preview = s.catalogPreview
|
||||
}
|
||||
return metadata, true
|
||||
}
|
||||
|
||||
func writeManifestFile(path string, manifest Manifest) error {
|
||||
b, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
@@ -737,13 +722,18 @@ func logPathForManifest(dir string, manifest Manifest) string {
|
||||
}
|
||||
|
||||
func supportedStoredManifest(manifest Manifest) bool {
|
||||
if manifest.SchemaVersion == SchemaVersion && manifest.Codec == Codec && manifest.StorageRevision == StorageRevision {
|
||||
if currentStoredManifest(manifest) {
|
||||
return true
|
||||
}
|
||||
return manifest.SchemaVersion == 3 &&
|
||||
(manifest.Codec == FinalV31Codec || manifest.Codec == LegacyLinearCodec || manifest.Codec == PrototypeCodec)
|
||||
}
|
||||
|
||||
func currentStoredManifest(manifest Manifest) bool {
|
||||
return manifest.SchemaVersion == SchemaVersion && manifest.Codec == Codec &&
|
||||
(manifest.StorageRevision == 1 || manifest.StorageRevision == StorageRevision)
|
||||
}
|
||||
|
||||
func readStoredManifest(path string) (Manifest, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reasonix/internal/provider"
|
||||
)
|
||||
|
||||
// Transcript metadata retains only input identities and bounded previews, never
|
||||
// assistant/tool bodies. It remains available when display history is external.
|
||||
type transcriptInput struct {
|
||||
ID string
|
||||
TurnID string
|
||||
Preview string
|
||||
}
|
||||
|
||||
func applyTranscriptMetadata(p *Projection, commit Commit, ev Event) {
|
||||
switch ev.Kind {
|
||||
case "history/replace", "legacy/import":
|
||||
p.TranscriptInputs = nil
|
||||
p.HiddenTurns = nil
|
||||
messages, err := replacementEventMessages(ev, ev.Payload)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, m := range messages {
|
||||
noteTranscriptInput(p, "", m)
|
||||
}
|
||||
case "message/complete", "message/upsert":
|
||||
var body struct {
|
||||
Message provider.Message `json:"message"`
|
||||
}
|
||||
if json.Unmarshal(ev.Payload, &body) != nil {
|
||||
return
|
||||
}
|
||||
if body.Message.LocalOnly || body.Message.Role != provider.RoleAssistant {
|
||||
for i := range p.Turns {
|
||||
if p.Turns[i].MessageID == body.Message.ID {
|
||||
p.Turns[i].MessageID = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
noteTranscriptInput(p, commit.TurnID, body.Message)
|
||||
case "message/retract":
|
||||
ids, err := retractedMessageIDs(ev, ev.Payload)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, id := range ids {
|
||||
for i := 0; i < len(p.TranscriptInputs); i++ {
|
||||
input := p.TranscriptInputs[i]
|
||||
if input.ID != id {
|
||||
continue
|
||||
}
|
||||
if input.TurnID != "" {
|
||||
if p.HiddenTurns == nil {
|
||||
p.HiddenTurns = map[string]bool{}
|
||||
}
|
||||
p.HiddenTurns[input.TurnID] = true
|
||||
}
|
||||
p.TranscriptInputs = append(p.TranscriptInputs[:i], p.TranscriptInputs[i+1:]...)
|
||||
break
|
||||
}
|
||||
for i := range p.Turns {
|
||||
if p.Turns[i].MessageID == id {
|
||||
p.Turns[i].MessageID = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func noteTranscriptInput(p *Projection, turnID string, m provider.Message) {
|
||||
if m.Role != provider.RoleUser {
|
||||
return
|
||||
}
|
||||
for i := range p.TranscriptInputs {
|
||||
if p.TranscriptInputs[i].ID == m.ID {
|
||||
p.TranscriptInputs[i].Preview = messagePreview(m)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.TranscriptInputs = append(p.TranscriptInputs, transcriptInput{ID: m.ID, TurnID: turnID, Preview: messagePreview(m)})
|
||||
delete(p.HiddenTurns, turnID)
|
||||
}
|
||||
|
||||
func visibleBoundaryCount(p Projection, endedOnly bool) int {
|
||||
count := 0
|
||||
for _, t := range p.Turns {
|
||||
if !p.HiddenTurns[t.TurnID] && (!endedOnly || t.EndSequence != 0) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
Reference in new issue
Block a user