diff --git a/.github/workflows/sdk-java.yml b/.github/workflows/sdk-java.yml index d8fca711025..a05754ddbd9 100644 --- a/.github/workflows/sdk-java.yml +++ b/.github/workflows/sdk-java.yml @@ -742,13 +742,19 @@ jobs: # 150 s, moving the fork to 9261 s and Verify to 165 minutes; the FG13b # Hook dispatch store-rejection cell adds 150 s, moving the fork to # 9411 s and Verify to 167 minutes; the FG11f publication cell adds 82 s, so - # the fork moves to 9765 s and Verify to 173 minutes, wrapping fork + 600 s - # plus margin; + # the fork moves to 9493 s and Verify to 169 minutes; the FG12f delivery + # cell adds 30 s, moving the fork to 9523 s and Verify to 169 minutes; the + # FG11f refusing-arms cells add 242 s, moving the fork to 9765 s and + # Verify to 173 minutes; the FG13b Hook result store-rejection cell adds + # 150 s, moving the fork to 9915 s and Verify to 176 minutes; the FG13b + # Hook plan-result store-rejection cell adds 150 s, moving the fork to + # 10065 s and Verify to 178 minutes, wrapping + # fork + 600 s plus margin; # the Broker fault gates use 20, # and admission plus O4 keep per-fork ceilings) plus ~13 minutes for the # uncapped setup steps (Checkout.. cgroup delegation, the report - # checks): 263 + 13. The failover arms run in hosted-failover-e2e. - timeout-minutes: 276 + # checks): 268 + 13. The failover arms run in hosted-failover-e2e. + timeout-minutes: 281 services: mysql: image: 'mysql:8.4.6' @@ -848,8 +854,8 @@ jobs: # fork leaves no per-test failure lines, so the main-CI failure # analyzer can only file an undiagnosable per-commit issue (#13503). # hosted-process-ci.test.js pins this ordering; the fork budget is - # 9765 s, so the step wraps to fork + 600 s plus margin (173 min). - timeout-minutes: 173 + # 10065 s, so the step wraps to fork + 600 s plus margin (178 min). + timeout-minutes: 178 env: MAVEN_ARGS: '--settings ${{ runner.temp }}/setup-java-m2/settings.xml --toolchains ${{ runner.temp }}/setup-java-m2/toolchains.xml' MYSQL_PORT: "${{ job.services.mysql.ports['3306'] }}" diff --git a/docs/design/2026-10-20-hosted-hook-store-fault-gates-b.md b/docs/design/2026-10-20-hosted-hook-store-fault-gates-b.md index 95b2b33d315..0c3dc01866b 100644 --- a/docs/design/2026-10-20-hosted-hook-store-fault-gates-b.md +++ b/docs/design/2026-10-20-hosted-hook-store-fault-gates-b.md @@ -2,8 +2,10 @@ [English](2026-10-20-hosted-hook-store-fault-gates-b.md) | [简体中文](2026-10-20-hosted-hook-store-fault-gates-b.zh-CN.md) -> The injection seam and the first two cells ship in one PR; the remaining -> three cells are planned and are not claimed implemented until each lands. +> The seam and the first two cells shipped in #14817. The result and +> plan-result cells ship here. The registration settle needs a catalog phase +> the seam does not have, so it moves to the FG13a store-ack slice rather +> than growing a second seam here. ## Problem and scope @@ -23,13 +25,13 @@ commit still runs in the real transaction. ## Cells -| Cell | Inject | Expect | -| --------------------------------------------------------- | --------------------------------------- | ------------------------------------------------------------------------- | -| `fg13bHookIntentStoreRejectionDispatchesNothingOnMySql` | Reject the intent commit | The injection fires; zero physical calls. | -| `fg13bHookDispatchStoreRejectionDispatchesNothingOnMySql` | Reject the `dispatch_started` commit | The injection fires; zero physical calls. | -| result commit (planned) | Reject the settled commit | One physical call; a later status read settles from the worker's receipt. | -| plan result commit (planned) | Reject the plan marker's settled commit | The marker is rebuilt from the saved receipts. | -| registration settle (planned) | Reject the catalog settle | The retry keeps the original operation id. | +| Cell | Inject | Expect | +| ----------------------------------------------------------- | --------------------------------------- | ----------------------------------------------------------------------------- | +| `fg13bHookIntentStoreRejectionDispatchesNothingOnMySql` | Reject the intent commit | The injection fires; zero physical calls. | +| `fg13bHookDispatchStoreRejectionDispatchesNothingOnMySql` | Reject the `dispatch_started` commit | The injection fires; zero physical calls. | +| `fg13bHookResultStoreRejectionSettlesFromTheReceiptOnMySql` | Reject the settled commit | One physical call; a later status read settles from the worker's receipt. | +| `fg13bHookPlanResultStoreRejectionRebuildsTheMarkerOnMySql` | Reject the plan marker's settled commit | The marker is rebuilt from the saved receipts. | +| registration settle (moved to FG13a) | Reject the catalog settle | The retry keeps the original operation id; needs a catalog phase on the seam. | ## Witnesses diff --git a/docs/design/2026-10-20-hosted-hook-store-fault-gates-b.zh-CN.md b/docs/design/2026-10-20-hosted-hook-store-fault-gates-b.zh-CN.md index 7461b3f617e..712be5532c5 100644 --- a/docs/design/2026-10-20-hosted-hook-store-fault-gates-b.zh-CN.md +++ b/docs/design/2026-10-20-hosted-hook-store-fault-gates-b.zh-CN.md @@ -2,7 +2,9 @@ [English](2026-10-20-hosted-hook-store-fault-gates-b.md) | [简体中文](2026-10-20-hosted-hook-store-fault-gates-b.zh-CN.md) -> 注入缝与首两格在同一个 PR 内交付;其余三格是计划,落地前不宣称已实现。 +> 注入缝与首两格已在 #14817 合入。结果提交与 plan 结果提交两格在本 PR +> 交付;注册结算需要注入缝尚无的 catalog phase,因此归入 FG13a store acks +> 那一片,而不是在这里再长出一套接缝。 ## 问题与范围 @@ -20,13 +22,13 @@ FG13b 在真实 Session store 事务内逐一拒绝 Hook 的持久化提交: ## 用例 -| 用例 | 注入 | 期望 | -| --------------------------------------------------------- | ----------------------------- | -------------------------------------------------- | -| `fg13bHookIntentStoreRejectionDispatchesNothingOnMySql` | 拒绝 intent 提交 | 注入确实触发;零物理调用。 | -| `fg13bHookDispatchStoreRejectionDispatchesNothingOnMySql` | 拒绝 `dispatch_started` 提交 | 注入确实触发;零物理调用。 | -| 结果提交(计划) | 拒绝已结算提交 | 一次物理调用;后续 status 读取按 worker 回执结算。 | -| plan 结果提交(计划) | 拒绝 plan marker 的已结算提交 | 由已保存回执重建 marker。 | -| 注册结算(计划) | 拒绝 catalog 结算 | 重试沿用原 operation id。 | +| 用例 | 注入 | 期望 | +| ----------------------------------------------------------- | ----------------------------- | ------------------------------------------------------- | +| `fg13bHookIntentStoreRejectionDispatchesNothingOnMySql` | 拒绝 intent 提交 | 注入确实触发;零物理调用。 | +| `fg13bHookDispatchStoreRejectionDispatchesNothingOnMySql` | 拒绝 `dispatch_started` 提交 | 注入确实触发;零物理调用。 | +| `fg13bHookResultStoreRejectionSettlesFromTheReceiptOnMySql` | 拒绝已结算提交 | 一次物理调用;后续 status 读取按 worker 回执结算。 | +| `fg13bHookPlanResultStoreRejectionRebuildsTheMarkerOnMySql` | 拒绝 plan marker 的已结算提交 | 由已保存回执重建 marker。 | +| 注册结算(移交 FG13a) | 拒绝 catalog 结算 | 重试沿用原 operation id;需要注入缝新增 catalog phase。 | ## 变异见证 diff --git a/packages/managed-agent-server/pom.xml b/packages/managed-agent-server/pom.xml index 89936e82ca3..1841336a89b 100644 --- a/packages/managed-agent-server/pom.xml +++ b/packages/managed-agent-server/pom.xml @@ -520,8 +520,19 @@ to 9411 s and the Verify step to 167 minutes; the FG11f publication cell adds 82 s, so the fork moves to 9493 s and the Verify step to - 169 minutes. --> - 9765 + 169 minutes; the FG12f delivery cell adds + 30 s, so the fork moves to 9523 s and Verify + stays 169 minutes; the FG11f refusing-arms + cells add 242 s, so the fork moves to 9765 s + and the Verify step to 173 minutes. The + FG13b Hook result store-rejection cell adds + 150 s: the healthy sum becomes 9315 s, so + the fork moves to 9915 s and the Verify step + to 176 minutes. The FG13b Hook plan-result + store-rejection cell adds 150 s: the healthy + sum becomes 9465 s, so the fork moves to + 10065 s and the Verify step to 178 minutes. --> + 10065 diff --git a/packages/managed-agent-server/src/test/java/com/alibaba/qwen/code/managedagent/HostedHookReplyLossFaultGatesIT.java b/packages/managed-agent-server/src/test/java/com/alibaba/qwen/code/managedagent/HostedHookReplyLossFaultGatesIT.java index d87f399243b..24ac17dd517 100644 --- a/packages/managed-agent-server/src/test/java/com/alibaba/qwen/code/managedagent/HostedHookReplyLossFaultGatesIT.java +++ b/packages/managed-agent-server/src/test/java/com/alibaba/qwen/code/managedagent/HostedHookReplyLossFaultGatesIT.java @@ -291,6 +291,16 @@ class HostedHookReplyLossFaultGatesIT { "input", List.of(Map.of("type", "input_text", "text", prompt))), prompt, 202).path("turn_id").asText(); } + private HttpResponse harnessStatus(String session, HarnessAttach attach) throws Exception { + var request = HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + harnessPort + + "/session/" + session + "/status")) + .timeout(Duration.ofSeconds(10)).header("Authorization", "Bearer " + TOKEN) + .header("X-Qwen-Client-Id", attach.clientId()) + .header("X-Qwen-Harness-Protocol-Version", "1") + .header("X-Qwen-Harness-Boot-Id", attach.bootId()).GET().build(); + return http.send(request, HttpResponse.BodyHandlers.ofString()); + } + private JsonNode hookExecution(String session) { List records = hookExecutions(session); assertThat(records).as("hook execution records").hasSize(1); @@ -567,6 +577,90 @@ class HostedHookReplyLossFaultGatesIT { StoreFaultGate.INSTANCE.clear(); } + @Test + @Timeout(300) + void fg13bHookResultStoreRejectionSettlesFromTheReceiptOnMySql() throws Exception { + boot("60000", "1000", "hook-solo"); + StoreFaultGate.INSTANCE.clear(); + StoreFaultGate.INSTANCE.arm(StoreFaultGate.Phase.HOOK_SETTLED, null); + JsonNode definition = api("POST", "/v1/agents", definition(), "definition", 202); + String session = createSession(definition.path("id").asText(), "initial"); + submit(session, "FG13_STORE_SETTLE"); + // The worker already ran the Hook. An exhausted append leaves the + // Turn recovery-blocked rather than failed, so quiescence is the + // end-of-attempt barrier: the read below then has to recover from + // the receipt rather than watch the original attempt land late. + waitFor(() -> assertThat(StoreFaultGate.INSTANCE.attempts()).isGreaterThanOrEqualTo(1)); + // submit() returns 202 before the connector's background attach, so + // the lookup waits for the dispatch that the fired fault proves ran. + HarnessAttach attach = attachment(spring.getBean(HarnessConnector.class), session); + assertThat(attach).as("Session attachment").isNotNull(); + waitFor(() -> { + var probe = harnessStatus(session, attach); + assertThat(probe.statusCode()).as("harness status: %s", probe.body()).isEqualTo(200); + JsonNode body = json.readTree(probe.body()); + assertThat(body.path("hasActivePrompt").asBoolean()).isFalse(); + assertThat(body.path("recoveryBlocked").asBoolean()).isTrue(); + }); + assertThat(events("hook.call")).as("physical calls").hasSize(1); + JsonNode parked = hookExecution(session); + assertThat(parked.path("resultRef").isNull()).as("unsettled record").isTrue(); + String executionId = parked.path("hookExecutionId").asText(); + StoreFaultGate.INSTANCE.clear(); + // A later read takes the receipt the worker still holds and commits + // the result without a second physical call. + waitFor(() -> { + var status = hookStatus(session, executionId, attach); + assertThat(status.statusCode()).as("hook status: %s", status.body()).isEqualTo(200); + assertThat(json.readTree(status.body()).path("state").asText()).isEqualTo("settled"); + }); + assertThat(hookExecution(session).at("/run/state").asText()).isEqualTo("settled"); + assertThat(events("hook.call")).as("physical calls").hasSize(1); + StoreFaultGate.INSTANCE.clear(); + } + + @Test + @Timeout(300) + void fg13bHookPlanResultStoreRejectionRebuildsTheMarkerOnMySql() throws Exception { + boot("60000", "1000", "hook-solo"); + StoreFaultGate.INSTANCE.clear(); + StoreFaultGate.INSTANCE.arm(StoreFaultGate.Phase.HOOK_PLAN_SETTLED, null); + JsonNode definition = api("POST", "/v1/agents", definition(), "definition", 202); + String session = createSession(definition.path("id").asText(), "initial"); + submit(session, "FG13_STORE_PLAN_SETTLE"); + // The child settled; only the PreToolUse marker's own commit is + // refused. Quiescence is the end-of-attempt barrier, after which the + // marker is rebuilt from the saved receipts. + waitFor(() -> assertThat(StoreFaultGate.INSTANCE.attempts()).isGreaterThanOrEqualTo(1)); + // submit() returns 202 before the connector's background attach, so + // the lookup waits for the dispatch that the fired fault proves ran. + HarnessAttach attach = attachment(spring.getBean(HarnessConnector.class), session); + assertThat(attach).as("Session attachment").isNotNull(); + waitFor(() -> { + var probe = harnessStatus(session, attach); + assertThat(probe.statusCode()).as("harness status: %s", probe.body()).isEqualTo(200); + JsonNode body = json.readTree(probe.body()); + assertThat(body.path("hasActivePrompt").asBoolean()).isFalse(); + assertThat(body.path("recoveryBlocked").asBoolean()).isTrue(); + }); + assertThat(events("hook.call")).as("physical calls").hasSize(1); + JsonNode marker = HostedGateTestSupport.extensionRecords(jdbc, tenant, session, "hook_execution").stream() + .filter(record -> "__plan__".equals(record.path("hookId").asText())) + .filter(record -> "PreToolUse".equals(record.path("eventName").asText())) + .findFirst() + .orElseThrow(); + assertThat(marker.path("resultRef").isNull()).as("unsettled marker").isTrue(); + String occurrenceId = marker.path("hookExecutionId").asText(); + StoreFaultGate.INSTANCE.clear(); + waitFor(() -> { + var status = hookStatus(session, occurrenceId, attach); + assertThat(status.statusCode()).as("hook status: %s", status.body()).isEqualTo(200); + assertThat(json.readTree(status.body()).path("state").asText()).isEqualTo("settled"); + }); + assertThat(events("hook.call")).as("physical calls").hasSize(1); + StoreFaultGate.INSTANCE.clear(); + } + // The Harness status route identifies the caller by the client id its // attach minted and the process generation it attached to; the connector // keeps both on the private attachment ref. @@ -652,7 +746,9 @@ class HostedHookReplyLossFaultGatesIT { && List.of("FG13_EXECUTE_LOSS", "FG13_STATUS_LOSS", "FG13_CATALOG_LOSS", "FG13_HTTP_LOSS", "FG13_DENY_LOSS", "FG13_FAILCLOSED_LOSS", - "FG13_STORE_INTENT", "FG13_STORE_DISPATCH").contains(prompt); + "FG13_STORE_INTENT", "FG13_STORE_DISPATCH", + "FG13_STORE_SETTLE", + "FG13_STORE_PLAN_SETTLE").contains(prompt); var chunk = json.createObjectNode().put("id", "hook-fault").put("object", "chat.completion.chunk") .put("created", 0).put("model", "hook-fault-fixture"); var choice = chunk.putArray("choices").addObject().put("index", 0); diff --git a/packages/managed-agent-server/src/test/java/com/alibaba/qwen/code/managedagent/persistence/StoreFaultGate.java b/packages/managed-agent-server/src/test/java/com/alibaba/qwen/code/managedagent/persistence/StoreFaultGate.java index 59239fe2bfd..b153d6012a4 100644 --- a/packages/managed-agent-server/src/test/java/com/alibaba/qwen/code/managedagent/persistence/StoreFaultGate.java +++ b/packages/managed-agent-server/src/test/java/com/alibaba/qwen/code/managedagent/persistence/StoreFaultGate.java @@ -121,7 +121,9 @@ public final class StoreFaultGate { && !"__plan__".equals(record.path("hookId").asText()); case HOOK_PLAN_SETTLED -> "hook_execution".equals(domain) && "settled".equals(execution) - && "__plan__".equals(record.path("hookId").asText()); + && "__plan__".equals(record.path("hookId").asText()) + && "PreToolUse".equals( + record.path("eventName").asText()); }; } diff --git a/packages/managed-agent-server/src/test/java/com/alibaba/qwen/code/managedagent/persistence/StoreFaultGateTest.java b/packages/managed-agent-server/src/test/java/com/alibaba/qwen/code/managedagent/persistence/StoreFaultGateTest.java index 5d5557bb878..b295a4f5c21 100644 --- a/packages/managed-agent-server/src/test/java/com/alibaba/qwen/code/managedagent/persistence/StoreFaultGateTest.java +++ b/packages/managed-agent-server/src/test/java/com/alibaba/qwen/code/managedagent/persistence/StoreFaultGateTest.java @@ -55,10 +55,35 @@ class StoreFaultGateTest { assertThat(StoreFaultGate.INSTANCE.attempts()).isEqualTo(1); } + @Test + void hookPlanSettledPhaseSkipsAnotherEventsMarker() { + StoreFaultGate.INSTANCE.arm( + StoreFaultGate.Phase.HOOK_PLAN_SETTLED, null); + StoreFaultGate.INSTANCE.inspect( + EVENT, record("__plan__", "settled", "SessionStart")); + assertThat(StoreFaultGate.INSTANCE.attempts()).isZero(); + } + + @Test + void hookPlanSettledPhaseRejectsThePreToolUseMarker() { + StoreFaultGate.INSTANCE.arm( + StoreFaultGate.Phase.HOOK_PLAN_SETTLED, null); + assertThatThrownBy(() -> StoreFaultGate.INSTANCE.inspect( + EVENT, record("__plan__", "settled", "PreToolUse"))) + .isInstanceOf(IllegalStateException.class); + assertThat(StoreFaultGate.INSTANCE.attempts()).isEqualTo(1); + } + private static Function record( String hookId, String execution) { + return record(hookId, execution, "PreToolUse"); + } + + private static Function record( + String hookId, String execution, String eventName) { byte[] bytes = ("{\"hookExecutionId\":\"occ-1\",\"hookId\":\"" - + hookId + "\",\"run\":{\"execution\":\"" + execution + "\"}}") + + hookId + "\",\"eventName\":\"" + eventName + + "\",\"run\":{\"execution\":\"" + execution + "\"}}") .getBytes(StandardCharsets.UTF_8); return id -> new StoredResource(id, "managed-hook-execution", 1, bytes.length, "sha256:x", bytes); diff --git a/scripts/tests/hosted-process-ci.test.js b/scripts/tests/hosted-process-ci.test.js index b9167e7824e..259a1f2a810 100644 --- a/scripts/tests/hosted-process-ci.test.js +++ b/scripts/tests/hosted-process-ci.test.js @@ -475,9 +475,9 @@ describe('Hosted real-process gates', () => { }); it.each([ - // Step ceilings today (173 + 20 + 50 + 20): Verify, the Broker fault + // Step ceilings today (178 + 20 + 50 + 20): Verify, the Broker fault // gates, admission and O4. The failover arms moved to their own job. - ['hosted-harness-mysql', 263, 276], + ['hosted-harness-mysql', 268, 281], // Twenty failover arms at 10 each. ['hosted-failover-e2e', 200, 215], ])( @@ -699,7 +699,7 @@ describe('Hosted real-process gates', () => { // anything up to step*60, so the value is pinned literally here — // an unnoticed raise has no other red to land on. if (profile === 'hosted-harness-mysql') { - expect(forkSeconds).toBe(9765); + expect(forkSeconds).toBe(10065); } }, ); @@ -768,12 +768,17 @@ describe('Hosted real-process gates', () => { // measures 22.2s on JDK 21 + MySQL 8.4.11, budgeted at 60s, so it becomes // 8511s; the FG13b Hook intent store-rejection cell adds 150s, so it // becomes 8661s; the FG13b Hook dispatch store-rejection cell adds 150s, - // so it becomes 8811s. The largest + // so it becomes 8811s; the FG11f publication cell adds 82s, so it becomes + // 8893s; the FG12f delivery cell adds 30s, so it becomes 8923s; the FG11f + // refusing-arms cells add 242s, so it becomes 9165s; the FG13b Hook result + // store-rejection cell adds 150s, so it becomes 9315s; the FG13b Hook + // plan-result store-rejection cell adds 150s, so it becomes 9465s. The + // largest // ceiling (600s) plus // that healthy sum still fits the fork. A larger ceiling lets failsafe // kill the fork mid-run, and the classes left unrun never report, failing // the gate undiagnosably (#13780). - const hostedForkHealthySeconds = 9165; + const hostedForkHealthySeconds = 9465; const hostedForkSeconds = () => Number( read('packages/managed-agent-server/pom.xml')