mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-07 06:16:06 -10:00
test(runner-e2e): allow controller outages up to ten minutes (#19797)
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The restart chaos harness and the runner-e2e recover-controller cells stop the control plane with `replaceController` (`tests/runner-e2e/controller-outage.ts`), hold it down, then start it again > - `parseControllerOutage` refuses holds longer than 300000 ms > - Restart chaos needs longer holds. One example is proving that a Daytona sandbox survives an outage longer than its old 5-minute auto-stop but inside the 7-minute runner reconnect grace (RC-54) > - With the cap, such a fault errors before the controller stops, so the scenario cannot run > - This pull request raises the test-harness cap to 600000 ms > - The benefit is that chaos runs can now inject 5–10 minute outages; production code and qualification contracts are unchanged ## Linked Issues or Issue Description **Bug description** A restart chaos replay with a 360000 ms K4 hold failed with "Controller outage requires SIGTERM/SIGKILL and 0..300000 integer milliseconds". The controller was never stopped. **Expected behavior** The test harness accepts outages of up to 10 minutes. The recover-controller receipt check keeps its own readiness limit (`readyAfterMs < 420000`). Refs #19544 ## What Changed - `tests/runner-e2e/controller-outage.ts`: adds `MAX_CONTROLLER_OUTAGE_MS = 600_000` and uses it in `parseControllerOutage` and its error message. `validControllerOutageReceipt` is unchanged and still requires readiness inside the 7-minute grace. - `tests/runner-e2e/controller-outage.test.ts`: the rejection cases now use 600001. A new test accepts 360000 (SIGKILL) and 600000 (SIGTERM). The existing five-minute contract test is unchanged. ## Verification - `vitest run tests/runner-e2e/controller-outage.test.ts tests/runner-e2e/server-restart-request.test.ts`: 52/52 pass. - `server-restart-request.test.ts` now uses 600001 as its out-of-range `controllerOutage`. The separate restart-request `controllerDowntimeMs` limit (five minutes) is unchanged. ## Risks - This change is test-only. No product or server code changes. - A suite that sets `PAPERCLIP_RUNNER_E2E_CONTROLLER_OUTAGE_MS` above 300000 now runs instead of being refused. The recover-controller receipt check still fails any receipt whose readiness is outside the grace. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, in Claude Code with tool use (shell, file edits, GitHub CLI). Default context window, extended thinking. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
35d41f11bb
commit
9cc3760552
3 files changed
+18
-4
No files matched your search
@@ -88,10 +88,14 @@ describe("controller outage qualification", () => {
|
||||
expect(f.input.start).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([-1, 300001, 0.5, NaN, Infinity, "300000", null])("rejects invalid duration %s before signaling", holdMs => {
|
||||
it.each([-1, 600001, 0.5, NaN, Infinity, "300000", null])("rejects invalid duration %s before signaling", holdMs => {
|
||||
expect(() => parseControllerOutage({ signal: "SIGKILL", holdMs })).toThrow();
|
||||
});
|
||||
it.each(["", "-1", "1e3", "NaN", " 1000", "300001"])("rejects invalid environment duration %s", value => {
|
||||
it("accepts chaos holds up to ten minutes", () => {
|
||||
expect(parseControllerOutage({ signal: "SIGKILL", holdMs: 360_000 })).toEqual({ signal: "SIGKILL", holdMs: 360_000 });
|
||||
expect(parseControllerOutage({ signal: "SIGTERM", holdMs: 600_000 })).toEqual({ signal: "SIGTERM", holdMs: 600_000 });
|
||||
});
|
||||
it.each(["", "-1", "1e3", "NaN", " 1000", "600001"])("rejects invalid environment duration %s", value => {
|
||||
expect(() => configuredControllerOutage({ PAPERCLIP_RUNNER_E2E_CONTROLLER_OUTAGE_MS: value })).toThrow();
|
||||
});
|
||||
it("accepts the five-minute contract without extending the runner grace", () => {
|
||||
|
||||
@@ -26,11 +26,21 @@ export interface ControllerOutageReceipt extends ControllerOutage {
|
||||
recoveryProgress?: Partial<ControllerRecoveryReceipt>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Longest controller outage a harness may inject. Ten minutes lets chaos
|
||||
* holds pass Daytona's sandbox auto-stop and the 7-minute runner reconnect
|
||||
* grace. The recover-controller receipt check below still requires readiness
|
||||
* inside that grace, so this does not loosen the qualification contract.
|
||||
*/
|
||||
export const MAX_CONTROLLER_OUTAGE_MS = 600_000;
|
||||
|
||||
export function parseControllerOutage(value: unknown): ControllerOutage {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid controller outage");
|
||||
const { signal, holdMs } = value as Record<string, unknown>;
|
||||
if ((signal !== "SIGTERM" && signal !== "SIGKILL") || !Number.isSafeInteger(holdMs)
|
||||
|| Number(holdMs) < 0 || Number(holdMs) > 300_000) throw new Error("Controller outage requires SIGTERM/SIGKILL and 0..300000 integer milliseconds");
|
||||
|| Number(holdMs) < 0 || Number(holdMs) > MAX_CONTROLLER_OUTAGE_MS) {
|
||||
throw new Error(`Controller outage requires SIGTERM/SIGKILL and 0..${MAX_CONTROLLER_OUTAGE_MS} integer milliseconds`);
|
||||
}
|
||||
return { signal, holdMs: Number(holdMs) };
|
||||
}
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ it("rejects cache replacement for unselected, local and legacy cells", async ()
|
||||
|
||||
it.each([
|
||||
{ controllerOutage: { signal: "SIGKILL", holdMs: 125_000 }, discardControllerCache: false },
|
||||
{ controllerOutage: { signal: "SIGKILL", holdMs: 300_001 }, discardControllerCache: true },
|
||||
{ controllerOutage: { signal: "SIGKILL", holdMs: 600_001 }, discardControllerCache: true },
|
||||
{ controllerOutage: { signal: "SIGKILL", holdMs: 125_000 }, discardControllerCache: true, preserveRunner: {} },
|
||||
{ controllerOutage: { signal: "SIGKILL", holdMs: 125_000 }, discardControllerCache: true, crashAfterResult: {} },
|
||||
{ recoveryBinding: {} },
|
||||
|
||||
Reference in new issue
Block a user