mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-13 18:49:34 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
515bff6ceb | ||
|
|
48c4800828 |
No files matched your search
@@ -347,9 +347,7 @@ async function withClient<T>(
|
||||
}
|
||||
|
||||
function printResult(json: boolean, body: unknown, status: number): void {
|
||||
if (status >= 400 && body && typeof body === "object"
|
||||
&& (body as { error?: unknown }).error === "remote_queue_write_failed"
|
||||
&& (body as { outcome?: unknown }).outcome === "indeterminate") {
|
||||
if (isIndeterminateRemoteWrite(status, body)) {
|
||||
console.error("The write outcome is INDETERMINATE if the request may have reached a daemon — reconcile by ID before any retry.");
|
||||
}
|
||||
if (json) {
|
||||
@@ -463,6 +461,34 @@ function writeReceipt(full: boolean | undefined, ctx: ReceiptContext): ReceiptCo
|
||||
return !full && compactOutputEnabled() ? ctx : null;
|
||||
}
|
||||
|
||||
/** A handoff whose outcome is unknown may have committed. This names the source row to read before any retry, on the
|
||||
* daemon the command used, and where a successor would be. It never says the handoff failed, names a successor id, or
|
||||
* asks for the handoff to be repeated. */
|
||||
function handoffRecoveryHint(qitemId: string, hostId: string | undefined): string {
|
||||
const remote = remoteHostOf(hostId);
|
||||
return `Before any retry, read source row ${qitemId} at the same endpoint and OPENRIG_HOME: `
|
||||
+ `rig queue show ${shellQuote(qitemId)} --full --json. Read its recorded outcome: a handed-off row names its successor`
|
||||
+ (remote ? ` on host ${shellQuote(remote)}` : "")
|
||||
+ ", or says why it can't. A row that still reads open doesn't show the handoff failed: it may still commit.";
|
||||
}
|
||||
|
||||
/** A handoff whose answer was lost (no answer in time, a lost connection, or an unreadable response) carries the hint in
|
||||
* its error. A pre-header disconnect can follow a committed handoff, as it can a create, so the shared renderer must not
|
||||
* call it undelivered. */
|
||||
function withHandoffRecovery(error: unknown, qitemId: string, hostId: string | undefined): unknown {
|
||||
if (!(error instanceof DaemonConnectionError || error instanceof DaemonResponseError)) return error;
|
||||
if (error instanceof DaemonConnectionError) error.writeOutcome = "unknown";
|
||||
error.message += ` ${handoffRecoveryHint(qitemId, hostId)}`;
|
||||
return error;
|
||||
}
|
||||
|
||||
/** The selected daemon's forward to another host lost the remote's answer: the write was sent and may still commit. */
|
||||
function isIndeterminateRemoteWrite(status: number, body: unknown): boolean {
|
||||
return status >= 400 && !!body && typeof body === "object"
|
||||
&& (body as { error?: unknown }).error === "remote_queue_write_failed"
|
||||
&& (body as { outcome?: unknown }).outcome === "indeterminate";
|
||||
}
|
||||
|
||||
/** A `--host` id that may name another daemon. Empty and `local` never do; one that
|
||||
* equals the selected daemon's own host id is settled by {@link settleReceiptHost}. */
|
||||
function remoteHostOf(hostId: string | undefined): string | undefined {
|
||||
@@ -1687,12 +1713,13 @@ workspace home that is deferred/not-imminent belongs in its mission/slice.`)
|
||||
targetRepo: opts.targetRepo,
|
||||
nudge: opts.nudge,
|
||||
...(hostResolved.hostId !== undefined ? { hostId: hostResolved.hostId } : {}),
|
||||
}, crossHostWriteOptions(hostResolved.hostId));
|
||||
}, crossHostWriteOptions(hostResolved.hostId)).catch((error: unknown) => { throw withHandoffRecovery(error, qitemId, hostResolved.hostId); });
|
||||
const hint = hostSubstitutionHint(res.status, res.data, opts.to, opts.host, hostResolved);
|
||||
if (hint) console.error(hint);
|
||||
await printQueueItemResult(client, qitemId, opts.json ?? false, res.data, res.status, writeReceipt(opts.full, {
|
||||
verb: "handoff", nudgeSuppressed: opts.nudge === false, remoteHost: remoteHostOf(hostResolved.hostId),
|
||||
}));
|
||||
if (isIndeterminateRemoteWrite(res.status, res.data)) console.error(handoffRecoveryHint(qitemId, hostResolved.hostId));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1789,12 +1816,13 @@ workspace home that is deferred/not-imminent belongs in its mission/slice.`)
|
||||
targetRepo: opts.targetRepo,
|
||||
nudge: opts.nudge,
|
||||
...(hostResolved.hostId !== undefined ? { hostId: hostResolved.hostId } : {}),
|
||||
}, crossHostWriteOptions(hostResolved.hostId));
|
||||
}, crossHostWriteOptions(hostResolved.hostId)).catch((error: unknown) => { throw withHandoffRecovery(error, qitemId, hostResolved.hostId); });
|
||||
const hint = hostSubstitutionHint(res.status, res.data, opts.to, opts.host, hostResolved);
|
||||
if (hint) console.error(hint);
|
||||
await printQueueItemResult(client, qitemId, opts.json ?? false, res.data, res.status, writeReceipt(opts.full, {
|
||||
verb: "handoff-and-complete", nudgeSuppressed: opts.nudge === false, remoteHost: remoteHostOf(hostResolved.hostId),
|
||||
}));
|
||||
if (isIndeterminateRemoteWrite(res.status, res.data)) console.error(handoffRecoveryHint(qitemId, hostResolved.hostId));
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -6,7 +6,8 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { QueueDeps } from "../src/commands/queue.js";
|
||||
import { createProgram } from "../src/index.js";
|
||||
import { DaemonTimeoutError } from "../src/client.js";
|
||||
import { DaemonConnectionError, DaemonResponseError, DaemonTimeoutError } from "../src/client.js";
|
||||
import { runProgram } from "../src/cli-error.js";
|
||||
|
||||
vi.mock("../src/daemon-lifecycle.js", async () => {
|
||||
const actual = await vi.importActual<Record<string, unknown>>("../src/daemon-lifecycle.js");
|
||||
@@ -44,13 +45,15 @@ function makeDeps(replies: Reply[]): { deps: QueueDeps; requests: Request[] } {
|
||||
describe("cross-host queue writes wait for the daemon's answer", () => {
|
||||
let home: string;
|
||||
let stderr: string[];
|
||||
let stdout: string[];
|
||||
|
||||
beforeEach(() => {
|
||||
home = fs.mkdtempSync(path.join(os.tmpdir(), "queue-xhost-wait-"));
|
||||
vi.stubEnv("OPENRIG_HOME", home);
|
||||
vi.stubEnv("OPENRIG_SESSION_NAME", "lead@rig-a");
|
||||
stderr = [];
|
||||
vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
stdout = [];
|
||||
vi.spyOn(console, "log").mockImplementation((...a: unknown[]) => { stdout.push(a.join(" ")); });
|
||||
vi.spyOn(console, "error").mockImplementation((...a: unknown[]) => { stderr.push(a.join(" ")); });
|
||||
vi.spyOn(process.stderr, "write").mockImplementation((chunk: string | Uint8Array) => { stderr.push(String(chunk)); return true; });
|
||||
process.exitCode = undefined;
|
||||
@@ -100,4 +103,78 @@ describe("cross-host queue writes wait for the daemon's answer", () => {
|
||||
expect(all).toContain("The write outcome is INDETERMINATE");
|
||||
expect(all).toMatch(/repeat the same create with --id 'qitem-\d{14}-[0-9a-f]{16}'/);
|
||||
});
|
||||
|
||||
// A handoff's lost answer leaves the source row, on the daemon the command used, as the thing to read: its successor is
|
||||
// named there once recorded. The error never names a successor id, says the handoff failed, or asks to repeat it.
|
||||
for (const verb of ["handoff", "handoff-and-complete"] as const) {
|
||||
for (const [where, to, successor] of [
|
||||
["to another host", "dev@rig-b@vps-b", "Read its recorded outcome: a handed-off row names its successor on host 'vps-b', or says why it can't."],
|
||||
["on this daemon", "dev@rig-b", "Read its recorded outcome: a handed-off row names its successor, or says why it can't."],
|
||||
] as const) {
|
||||
it(`a ${verb} ${where} that gets no answer in time names the source row to read before any retry`, async () => {
|
||||
let thrown: unknown;
|
||||
await run([verb, "qitem-src-1", "--to", to, "--summary", "s"],
|
||||
[new DaemonTimeoutError(`Request to http://localhost:7433/api/queue/qitem-src-1/${verb} timed out after 15000ms`)])
|
||||
.catch((e: unknown) => { thrown = e; });
|
||||
expect(thrown).toBeInstanceOf(DaemonTimeoutError);
|
||||
const message = (thrown as Error).message;
|
||||
expect(message).toContain("Before any retry, read source row qitem-src-1 at the same endpoint and OPENRIG_HOME: rig queue show 'qitem-src-1' --full --json.");
|
||||
expect(message).toContain(successor);
|
||||
expect(message).toContain("doesn't show the handoff failed");
|
||||
expect(message).not.toMatch(/--id|repeat the/);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Rendered as a person sees it, through the shared program renderer: a lost connection or an unreadable response after
|
||||
// a handoff was sent is an unknown outcome, never "not delivered", and keeps the source row to read.
|
||||
async function runRendered(argv: string[], replies: Reply[], json: boolean): Promise<{ text: string; code: number }> {
|
||||
const { deps } = makeDeps(replies);
|
||||
const lines: string[] = [];
|
||||
const code = await runProgram(createProgram({ queueDeps: deps }), ["node", "rig", "queue", ...argv, ...(json ? ["--json"] : [])],
|
||||
{ out: (l) => lines.push(l), err: (l) => lines.push(l), exit: () => {} });
|
||||
return { text: [...stderr, ...lines].join("\n"), code };
|
||||
}
|
||||
|
||||
for (const verb of ["handoff", "handoff-and-complete"] as const) {
|
||||
for (const [kind, error] of [
|
||||
["a lost connection", () => new DaemonConnectionError("Cannot connect to the daemon at http://localhost:7433: fetch failed", "ECONNRESET")],
|
||||
["an unreadable response", () => new DaemonResponseError(200, "")],
|
||||
] as const) {
|
||||
for (const json of [false, true]) {
|
||||
it(`a ${verb} that meets ${kind} reads as an unknown outcome, never undelivered (${json ? "--json" : "human"})`, async () => {
|
||||
const { text, code } = await runRendered([verb, "qitem-src-1", "--to", "dev@rig-b", "--summary", "s"], [error()], json);
|
||||
expect(code).toBe(1);
|
||||
expect(text).toContain("outcome is UNKNOWN");
|
||||
expect(text).not.toMatch(/not delivered/i);
|
||||
expect(text).toContain("Before any retry, read source row qitem-src-1");
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The selected daemon's forward can answer with a structured reply instead of throwing. An explicit indeterminate one
|
||||
// gets the same source-row hint, with the reply and its exit code unchanged; a refusal before any forward implies no
|
||||
// write that may still commit.
|
||||
const INDETERMINATE = { status: 502, data: { error: "remote_queue_write_failed", hostId: "vps-b", failureClass: "unreachable",
|
||||
outcome: "indeterminate", detail: "remote queue write timed out after 10000ms; the remote may still commit it" } };
|
||||
const REFUSED = { status: 502, data: { error: "remote_queue_write_failed", hostId: "vps-b", failureClass: "unsupported-transport",
|
||||
detail: "host 'vps-b' is SSH-declared" } };
|
||||
for (const verb of ["handoff", "handoff-and-complete"] as const) {
|
||||
it(`a ${verb} whose forward answers indeterminate names the source row to read, keeping the reply and exit code`, async () => {
|
||||
await run([verb, "qitem-src-1", "--to", "dev@rig-b@vps-b", "--summary", "s"], [INDETERMINATE]);
|
||||
const all = stderr.join("\n");
|
||||
expect(all).toContain("Before any retry, read source row qitem-src-1 at the same endpoint and OPENRIG_HOME: rig queue show 'qitem-src-1' --full --json.");
|
||||
expect(all).toContain("a handed-off row names its successor on host 'vps-b', or says why it can't.");
|
||||
expect(stdout).toEqual([JSON.stringify(INDETERMINATE.data)]);
|
||||
expect(process.exitCode).toBe(2);
|
||||
});
|
||||
|
||||
it(`a ${verb} the daemon refused to forward says nothing may still commit`, async () => {
|
||||
await run([verb, "qitem-src-1", "--to", "dev@rig-b@vps-b", "--summary", "s"], [REFUSED]);
|
||||
expect(stderr.join("\n")).not.toMatch(/may still commit|INDETERMINATE|Before any retry/);
|
||||
expect(stdout).toEqual([JSON.stringify(REFUSED.data)]);
|
||||
expect(process.exitCode).toBe(2);
|
||||
});
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user