fix(daemon): latch acknowledged Worker write failure so drain is non-clean

Guard PRE-EDIT CLEAR (4a04c1f7) — bounded recorder-local correction of a
parent-carried defect (QA signal bbd28852). A Worker reply ok=false marked
health degraded but resolved the pending write without latching lost durability,
so a later flush marker resolved, close resolved, and drainSlowOpRecorderOnShutdown
returned exit 0 despite a known-lost record.

slow-op-recorder.ts only (index.ts byte-identical):
- sticky acknowledgedWriteFailure latch set in the ok=false message branch BEFORE
  resolving the waiter (one-shot degraded signal + waiter resolution preserved;
  no hang, no duplicate emission, no unhandled fire-and-forget rejection);
- flush() completes its marker round-trip then rejects the new named
  SlowOpRecorderWriteError when latched — distinct from the worker-terminal
  SlowOpRecorderTerminatedError; close() still terminates finitely + rethrows;
- the production drain (unchanged) maps that rejection to exit 1.

RED-first (compiled Node v22.22.1): ENOTDIR-class real Worker ok=false -> flush/
close reject, snapshot recorder_write_failed, one emission, drain exit 1 + loss
log; clean recorder still exit 0. 40/40 focused (default + maxWorkers=1), 73/73
adjacent, typecheck + build. Prior lifecycle/non-interference legs preserved.
Author-excluded from later gates.
This commit is contained in:
v-openrig-build committed 2026-08-05 17:51:53 +00:00
1 parent f2ac5ca5e0
commit ceb9a3da32
3 files changed
+81 -3

No files matched your search

+30 -1
View File
@@ -27,6 +27,19 @@ export class SlowOpRecorderTerminatedError extends Error {
}
}
/**
* OPR.0.4.3.21 (51elv2) — the named acknowledged-write-failure signal. The
* Worker is still alive (distinct from {@link SlowOpRecorderTerminatedError}),
* but it acknowledged that a record could not be durably written; flush()/close()
* reject with this so a known-lost record can never read as a clean durable drain.
*/
export class SlowOpRecorderWriteError extends Error {
constructor(reason: string) {
super(`slow-operation recorder write failed: ${reason}`);
this.name = "SlowOpRecorderWriteError";
}
}
export interface SlowOperationInstrumentation {
runSync?<T>(site: string, fn: () => T): T;
runStage?<T>(
@@ -127,6 +140,10 @@ export class SlowOpRecorder implements SlowOperationInstrumentation {
// rejects every future post/flush so nothing waits forever or reports a
// false durable drain.
private terminalReason: string | null = null;
// OPR.0.4.3.21 (51elv2) — sticky latch set once the Worker acknowledges a
// record it could not durably write (ok:false). The Worker stays alive, but
// durability is lost, so flush()/close() must refuse to report a clean drain.
private acknowledgedWriteFailure = false;
constructor(options: SlowOpRecorderOptions) {
this.logPath = options.logPath;
@@ -138,7 +155,13 @@ export class SlowOpRecorder implements SlowOperationInstrumentation {
this.worker.unref();
this.worker.on("message", (message: { id?: string; ok?: boolean }) => {
if (!message.id) return;
if (message.ok === false) this.markDegraded("recorder_write_failed", "recorder.worker");
if (message.ok === false) {
// Latch the lost durability BEFORE resolving this waiter (the write
// attempt is done, just failed): keep the one-shot degraded signal and
// waiter resolution, but flush()/close() will now refuse a clean drain.
this.acknowledgedWriteFailure = true;
this.markDegraded("recorder_write_failed", "recorder.worker");
}
this.pending.get(message.id)?.resolve();
this.pending.delete(message.id);
});
@@ -236,6 +259,12 @@ export class SlowOpRecorder implements SlowOperationInstrumentation {
// Rejects with SlowOpRecorderTerminatedError when the Worker is lost —
// finite, and never a false durable-drain success.
await this.postAndWait({ type: "flush" });
// The marker round-tripped, but if the Worker earlier acknowledged a write
// it could not persist, durability is unproven — reject so flush()/close()
// (and the production drain) never report a clean drain over a lost record.
if (this.acknowledgedWriteFailure) {
throw new SlowOpRecorderWriteError("recorder_write_failed");
}
}
async close(): Promise<void> {
@@ -44,7 +44,7 @@ describe("drainSlowOpRecorderOnShutdown — bounded shutdown drain", () => {
expect(code).toBe(0);
const records = await readRecords(logPath);
expect(records.map((r) => r.site)).toEqual(["test.drain.a", "test.drain.b"]);
});
}, 30_000);
it("returns 0 when no recorder is wired", async () => {
const { drainSlowOpRecorderOnShutdown } = await loadIndex();
@@ -60,6 +60,21 @@ describe("drainSlowOpRecorderOnShutdown — bounded shutdown drain", () => {
expect(logs.join(" ")).toMatch(/slow-operation/);
});
it("returns nonzero and logs when the recorder acknowledged a lost write (durability not clean)", async () => {
const { drainSlowOpRecorderOnShutdown } = await loadIndex();
const { SlowOpRecorder } = await loadRecorder();
// Deterministic real Worker ok:false — log parent is a regular file (ENOTDIR).
const dir = tempDir();
const notADir = path.join(dir, "regular-file");
fs.writeFileSync(notADir, "x");
const recorder = new SlowOpRecorder({ logPath: path.join(notADir, "slow-operations.jsonl") });
recorder.recordMeasurement("test.write.fail", 300);
const logs: string[] = [];
const code = await drainSlowOpRecorderOnShutdown(recorder, { timeoutMs: 20_000, log: (m: string) => logs.push(m) });
expect(code).toBe(1);
expect(logs.join(" ")).toMatch(/slow-operation/);
}, 30_000);
it("treats a FALSEY promise rejection as a failure (logs + nonzero), not a silent success", async () => {
const { drainSlowOpRecorderOnShutdown } = await loadIndex();
for (const value of [undefined, null, false, 0, ""]) {
+35 -1
View File
@@ -181,7 +181,10 @@ describe("SlowOpRecorder locked instrumentation contract", () => {
site: "test.sync.observer_failure",
});
} finally {
await recorder.close();
// This recorder's log path is intentionally unwritable (to force the
// begin-barrier failure above), so close() now correctly rejects on the
// latched lost-write durability — tolerate it in teardown.
await recorder.close().catch(() => {});
}
});
@@ -417,6 +420,37 @@ describe("SlowOpRecorder locked instrumentation contract", () => {
expect(emissions).toBe(0);
});
it("latches an acknowledged Worker write failure so flush/close cannot report a clean durable drain", async () => {
const mod = await loadRecorderModule();
expect(mod, "slow-op-recorder production module is missing").not.toBeNull();
if (!mod) return;
// Deterministic real Worker append rejection: the log's parent is a regular
// file, so the Worker's mkdirSync(dirname) fails ENOTDIR-class -> ok:false.
const dir = tempDir();
const notADir = path.join(dir, "regular-file");
fs.writeFileSync(notADir, "x");
const recorder = new mod.SlowOpRecorder({ logPath: path.join(notADir, "slow-operations.jsonl") });
let emissions = 0;
recorder.setDegradedHandler(() => { emissions += 1; });
recorder.recordMeasurement("test.write.fail", 300);
// A known-lost write must make the drain non-clean: flush rejects with the
// write-failure error (distinct from the worker-terminal error).
await expect(recorder.flush()).rejects.toBeInstanceOf(mod.SlowOpRecorderWriteError);
expect(recorder.snapshot()).toMatchObject({
healthy: false,
reason: "recorder_write_failed",
site: "recorder.worker",
});
expect(emissions).toBe(1);
// Wrapped operation identity is still authoritative after a write failure.
expect(recorder.runSync("test.after.write", () => 7)).toBe(7);
// close() preserves the drain failure (rejects) but still tears down finitely.
await expect(recorder.close()).rejects.toBeInstanceOf(mod.SlowOpRecorderWriteError);
expect(emissions).toBe(1);
});
it("isolates a throwing request observer so the route keeps its exact successful status and body", async () => {
const recorder = {
recordRequest(): void { throw new Error("request observer sink failed"); },