mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-08 08:11:17 -10:00
fix(daemon): latch acknowledged Worker write failure so drain is non-clean
Guard PRE-EDIT CLEAR (4a04c1f7) — bounded recorder-local correction of a parent-carried defect (QA signal bbd28852). A Worker reply ok=false marked health degraded but resolved the pending write without latching lost durability, so a later flush marker resolved, close resolved, and drainSlowOpRecorderOnShutdown returned exit 0 despite a known-lost record. slow-op-recorder.ts only (index.ts byte-identical): - sticky acknowledgedWriteFailure latch set in the ok=false message branch BEFORE resolving the waiter (one-shot degraded signal + waiter resolution preserved; no hang, no duplicate emission, no unhandled fire-and-forget rejection); - flush() completes its marker round-trip then rejects the new named SlowOpRecorderWriteError when latched — distinct from the worker-terminal SlowOpRecorderTerminatedError; close() still terminates finitely + rethrows; - the production drain (unchanged) maps that rejection to exit 1. RED-first (compiled Node v22.22.1): ENOTDIR-class real Worker ok=false -> flush/ close reject, snapshot recorder_write_failed, one emission, drain exit 1 + loss log; clean recorder still exit 0. 40/40 focused (default + maxWorkers=1), 73/73 adjacent, typecheck + build. Prior lifecycle/non-interference legs preserved. Author-excluded from later gates.
This commit is contained in:
1 parent
f2ac5ca5e0
commit
ceb9a3da32
3 files changed
+81
-3
No files matched your search
@@ -27,6 +27,19 @@ export class SlowOpRecorderTerminatedError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* OPR.0.4.3.21 (51elv2) — the named acknowledged-write-failure signal. The
|
||||
* Worker is still alive (distinct from {@link SlowOpRecorderTerminatedError}),
|
||||
* but it acknowledged that a record could not be durably written; flush()/close()
|
||||
* reject with this so a known-lost record can never read as a clean durable drain.
|
||||
*/
|
||||
export class SlowOpRecorderWriteError extends Error {
|
||||
constructor(reason: string) {
|
||||
super(`slow-operation recorder write failed: ${reason}`);
|
||||
this.name = "SlowOpRecorderWriteError";
|
||||
}
|
||||
}
|
||||
|
||||
export interface SlowOperationInstrumentation {
|
||||
runSync?<T>(site: string, fn: () => T): T;
|
||||
runStage?<T>(
|
||||
@@ -127,6 +140,10 @@ export class SlowOpRecorder implements SlowOperationInstrumentation {
|
||||
// rejects every future post/flush so nothing waits forever or reports a
|
||||
// false durable drain.
|
||||
private terminalReason: string | null = null;
|
||||
// OPR.0.4.3.21 (51elv2) — sticky latch set once the Worker acknowledges a
|
||||
// record it could not durably write (ok:false). The Worker stays alive, but
|
||||
// durability is lost, so flush()/close() must refuse to report a clean drain.
|
||||
private acknowledgedWriteFailure = false;
|
||||
|
||||
constructor(options: SlowOpRecorderOptions) {
|
||||
this.logPath = options.logPath;
|
||||
@@ -138,7 +155,13 @@ export class SlowOpRecorder implements SlowOperationInstrumentation {
|
||||
this.worker.unref();
|
||||
this.worker.on("message", (message: { id?: string; ok?: boolean }) => {
|
||||
if (!message.id) return;
|
||||
if (message.ok === false) this.markDegraded("recorder_write_failed", "recorder.worker");
|
||||
if (message.ok === false) {
|
||||
// Latch the lost durability BEFORE resolving this waiter (the write
|
||||
// attempt is done, just failed): keep the one-shot degraded signal and
|
||||
// waiter resolution, but flush()/close() will now refuse a clean drain.
|
||||
this.acknowledgedWriteFailure = true;
|
||||
this.markDegraded("recorder_write_failed", "recorder.worker");
|
||||
}
|
||||
this.pending.get(message.id)?.resolve();
|
||||
this.pending.delete(message.id);
|
||||
});
|
||||
@@ -236,6 +259,12 @@ export class SlowOpRecorder implements SlowOperationInstrumentation {
|
||||
// Rejects with SlowOpRecorderTerminatedError when the Worker is lost —
|
||||
// finite, and never a false durable-drain success.
|
||||
await this.postAndWait({ type: "flush" });
|
||||
// The marker round-tripped, but if the Worker earlier acknowledged a write
|
||||
// it could not persist, durability is unproven — reject so flush()/close()
|
||||
// (and the production drain) never report a clean drain over a lost record.
|
||||
if (this.acknowledgedWriteFailure) {
|
||||
throw new SlowOpRecorderWriteError("recorder_write_failed");
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
|
||||
@@ -44,7 +44,7 @@ describe("drainSlowOpRecorderOnShutdown — bounded shutdown drain", () => {
|
||||
expect(code).toBe(0);
|
||||
const records = await readRecords(logPath);
|
||||
expect(records.map((r) => r.site)).toEqual(["test.drain.a", "test.drain.b"]);
|
||||
});
|
||||
}, 30_000);
|
||||
|
||||
it("returns 0 when no recorder is wired", async () => {
|
||||
const { drainSlowOpRecorderOnShutdown } = await loadIndex();
|
||||
@@ -60,6 +60,21 @@ describe("drainSlowOpRecorderOnShutdown — bounded shutdown drain", () => {
|
||||
expect(logs.join(" ")).toMatch(/slow-operation/);
|
||||
});
|
||||
|
||||
it("returns nonzero and logs when the recorder acknowledged a lost write (durability not clean)", async () => {
|
||||
const { drainSlowOpRecorderOnShutdown } = await loadIndex();
|
||||
const { SlowOpRecorder } = await loadRecorder();
|
||||
// Deterministic real Worker ok:false — log parent is a regular file (ENOTDIR).
|
||||
const dir = tempDir();
|
||||
const notADir = path.join(dir, "regular-file");
|
||||
fs.writeFileSync(notADir, "x");
|
||||
const recorder = new SlowOpRecorder({ logPath: path.join(notADir, "slow-operations.jsonl") });
|
||||
recorder.recordMeasurement("test.write.fail", 300);
|
||||
const logs: string[] = [];
|
||||
const code = await drainSlowOpRecorderOnShutdown(recorder, { timeoutMs: 20_000, log: (m: string) => logs.push(m) });
|
||||
expect(code).toBe(1);
|
||||
expect(logs.join(" ")).toMatch(/slow-operation/);
|
||||
}, 30_000);
|
||||
|
||||
it("treats a FALSEY promise rejection as a failure (logs + nonzero), not a silent success", async () => {
|
||||
const { drainSlowOpRecorderOnShutdown } = await loadIndex();
|
||||
for (const value of [undefined, null, false, 0, ""]) {
|
||||
|
||||
@@ -181,7 +181,10 @@ describe("SlowOpRecorder locked instrumentation contract", () => {
|
||||
site: "test.sync.observer_failure",
|
||||
});
|
||||
} finally {
|
||||
await recorder.close();
|
||||
// This recorder's log path is intentionally unwritable (to force the
|
||||
// begin-barrier failure above), so close() now correctly rejects on the
|
||||
// latched lost-write durability — tolerate it in teardown.
|
||||
await recorder.close().catch(() => {});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -417,6 +420,37 @@ describe("SlowOpRecorder locked instrumentation contract", () => {
|
||||
expect(emissions).toBe(0);
|
||||
});
|
||||
|
||||
it("latches an acknowledged Worker write failure so flush/close cannot report a clean durable drain", async () => {
|
||||
const mod = await loadRecorderModule();
|
||||
expect(mod, "slow-op-recorder production module is missing").not.toBeNull();
|
||||
if (!mod) return;
|
||||
|
||||
// Deterministic real Worker append rejection: the log's parent is a regular
|
||||
// file, so the Worker's mkdirSync(dirname) fails ENOTDIR-class -> ok:false.
|
||||
const dir = tempDir();
|
||||
const notADir = path.join(dir, "regular-file");
|
||||
fs.writeFileSync(notADir, "x");
|
||||
const recorder = new mod.SlowOpRecorder({ logPath: path.join(notADir, "slow-operations.jsonl") });
|
||||
let emissions = 0;
|
||||
recorder.setDegradedHandler(() => { emissions += 1; });
|
||||
recorder.recordMeasurement("test.write.fail", 300);
|
||||
|
||||
// A known-lost write must make the drain non-clean: flush rejects with the
|
||||
// write-failure error (distinct from the worker-terminal error).
|
||||
await expect(recorder.flush()).rejects.toBeInstanceOf(mod.SlowOpRecorderWriteError);
|
||||
expect(recorder.snapshot()).toMatchObject({
|
||||
healthy: false,
|
||||
reason: "recorder_write_failed",
|
||||
site: "recorder.worker",
|
||||
});
|
||||
expect(emissions).toBe(1);
|
||||
// Wrapped operation identity is still authoritative after a write failure.
|
||||
expect(recorder.runSync("test.after.write", () => 7)).toBe(7);
|
||||
// close() preserves the drain failure (rejects) but still tears down finitely.
|
||||
await expect(recorder.close()).rejects.toBeInstanceOf(mod.SlowOpRecorderWriteError);
|
||||
expect(emissions).toBe(1);
|
||||
});
|
||||
|
||||
it("isolates a throwing request observer so the route keeps its exact successful status and body", async () => {
|
||||
const recorder = {
|
||||
recordRequest(): void { throw new Error("request observer sink failed"); },
|
||||
|
||||
Reference in new issue
Block a user