Compare commits

...
Author SHA1 Message Date
review50-r3andClaude Opus 5.5 1a52040b6b perf(daemon): name the queue's synchronous reads in the slow-operation log
An event-loop gap in the slow-operation log carries no site, so a stall during
one of the queue's synchronous reads couldn't be told from any other. Name
three of them:

- queue.handoff: a whole handoff, which runs synchronously (runSyncSite).
- queue.linked_successors.build: the stuck sweep's linked-successor read
  (runSyncSite).
- queue.decision_revision.replies: decisionRevision's read of a row's replies,
  which runs on every row read and on each row of the attention read. A begin
  and an end record per call would crowd the bounded log, so it goes through a
  new runSyncSiteWhenSlow, which records only a call at or above the existing
  slow threshold and writes no begin record. The recorder gains
  recordSlowSync for it; a failing record never replaces the call's own result.

Tests pin each site through a fake recorder, the real recorder's threshold and
outcome for recordSlowSync, and the new labels in the accepted-site list.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-11-12 20:53:59 +00:00
5 changed files with 125 additions and 4 deletions

No files matched your search

+11 -4
View File
@@ -105,6 +105,7 @@ import { armQueueWait, backOffQueueWait, refreshQueueWaits, evaluateQueueWait, r
import { parseHumanQuestions, unansweredQuestions, describeTypedReplyPlacement, formatHumanAnswers, humanAnswerText, type HumanQuestion, type HumanAnswers, type TypedHumanReply, type RecordHumanAnswerResult } from "./human-questions.js";
import { slackMicros } from "./gateway/slack/state-store.js";
import { askBackRoute, RUNNER_CLOSE_WAITS, terminalAnswerMeaning } from "./receiver-context.js";
import { runSyncSite, runSyncSiteWhenSlow } from "./sync-site-wrap.js";
/** A direct ask's answer row ID: from the ask, or from the ask and the park episode its answer belongs to. */
function humanAnswerId(askId: string, episodeKey?: string): string {
@@ -821,14 +822,14 @@ export function createLinkedSuccessorLookup(db: Database.Database, sourceQitemId
const startedAt = performance.now();
const map = new Map<string, Array<{ qitemId: string; destination: string | null }>>();
const ids = JSON.stringify([...sources]);
const rows = db.prepare(
const rows = runSyncSite("queue.linked_successors.build", () => db.prepare(
`SELECT s.qitem_id, s.destination_session, s.handed_off_from AS source FROM queue_items s
WHERE s.handed_off_from IN (SELECT value FROM json_each(?))
UNION
SELECT s.qitem_id, s.destination_session, j.value AS source FROM queue_items s, json_each(s.chain_of_record) j
WHERE s.chain_of_record IS NOT NULL AND json_valid(s.chain_of_record)
AND j.value IN (SELECT value FROM json_each(?))`,
).all(ids, ids) as Array<{ qitem_id: string; destination_session: string | null; source: string }>;
).all(ids, ids)) as Array<{ qitem_id: string; destination_session: string | null; source: string }>;
for (const r of rows) {
const list = map.get(r.source);
const entry = { qitemId: r.qitem_id, destination: r.destination_session };
@@ -2634,8 +2635,13 @@ export class QueueRepository {
}
/** {@link handoff}'s whole act, synchronously: its transaction commits before this returns, and the wake is delivered
* after commit without being awaited, so a synchronous caller (an answer's close path) can hand a row on. */
* after commit without being awaited, so a synchronous caller (an answer's close path) can hand a row on. Named in
* the slow-operation log, so a stall during a handoff is attributed to it. */
private handoffNow(input: QueueHandoffInput): { closed: QueueItem; created: QueueItem } {
return runSyncSite("queue.handoff", () => this.handoffWithin(input));
}
private handoffWithin(input: QueueHandoffInput): { closed: QueueItem; created: QueueItem } {
const source = this.getById(input.qitemId);
if (!source) {
throw new QueueRepositoryError(
@@ -4563,8 +4569,9 @@ export class QueueRepository {
typeof opts?.excludeOperationReply === "string" && opts.excludeOperationReply.length > 0
? mcOperationTag(opts.excludeOperationReply)
: null;
// Named only when slow: it runs on every row read and on each row of the attention read.
const replies = this.hasReplyToColumn
? (excludeOp
? runSyncSiteWhenSlow("queue.decision_revision.replies", () => excludeOp
? (this.db
.prepare(
"SELECT qitem_id FROM queue_items WHERE reply_to = ? AND NOT (tags IS NOT NULL AND json_valid(tags) AND EXISTS (SELECT 1 FROM json_each(tags) WHERE value = ?)) ORDER BY ts_created, rowid",
@@ -48,6 +48,8 @@ export interface SlowOperationInstrumentation {
classify?: (value: T) => "ok" | "failed",
): Promise<T>;
recordRequest?(site: string, durationMs: number): void;
/** A synchronous site's duration, recorded only at or above the slow threshold (runSyncSiteWhenSlow). */
recordSlowSync?(site: string, durationMs: number, outcome: "ok" | "failed"): void;
/** Bounded nonblocking diagnostic append; false means observation was dropped. */
recordDiagnostic?(record: Record<string, unknown>): boolean;
snapshot?(): SlowOperationSnapshot;
@@ -262,6 +264,11 @@ export class SlowOpRecorder implements SlowOperationInstrumentation {
this.recordMeasurement(`request:${site}`, durationMs);
}
recordSlowSync(site: string, durationMs: number, outcome: "ok" | "failed" = "ok"): void {
if (!Number.isFinite(durationMs) || durationMs < this.slowThresholdMs) return;
this.recordMeasurement(site, durationMs, outcome);
}
recordDiagnostic(record: Record<string, unknown>): boolean {
this.diagnostic.offered++;
const coverage = { ...this.diagnostic, pending: this.diagnosticIds.size,
@@ -10,6 +10,26 @@ export function runSyncSite<T>(site: string, fn: () => T): T {
return recorder?.runSync ? recorder.runSync(site, fn) : fn();
}
/**
* runSyncSite for a site on a hot path (per request, per row), where a begin and an end record per call would crowd
* the bounded log: it records only a call that took at least the recorder's slow threshold, and writes no begin
* record, so a call that wedges the loop is named once it ends rather than while it runs.
*/
export function runSyncSiteWhenSlow<T>(site: string, fn: () => T): T {
const record = recorder?.recordSlowSync?.bind(recorder);
if (!record) return fn();
const startedAt = performance.now();
let outcome: "ok" | "failed" = "failed";
try {
const value = fn();
outcome = "ok";
return value;
} finally {
// Measurement only: a failing record never replaces fn's own result or error.
try { record(site, performance.now() - startedAt, outcome); } catch { /* the recorder reports its own health */ }
}
}
/**
* Async twin of runSyncSite: instruments a NON-blocking site through the
* recorder's async `runStage` seam (falls back to a bare call when no recorder
@@ -0,0 +1,69 @@
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import type Database from "better-sqlite3";
import { createDb } from "../src/db/connection.js";
import { migrate } from "../src/db/migrate.js";
import { ALL_MIGRATIONS } from "../src/db/all-migrations.js";
import { EventBus } from "../src/domain/event-bus.js";
import { QueueRepository, createLinkedSuccessorLookup } from "../src/domain/queue-repository.js";
import { configureSyncSiteRecorder, runSyncSiteWhenSlow } from "../src/domain/sync-site-wrap.js";
// The queue's synchronous reads named in the slow-operation log, so an event-loop stall during one is attributed to it.
// A fake recorder takes every call; the real recorder's slow threshold is pinned in slow-op-recorder.test.ts.
describe("queue reads named in the slow-operation log", () => {
let db: Database.Database;
let repo: QueueRepository;
const sync: string[] = [];
const slow: Array<{ site: string; outcome: string }> = [];
beforeEach(() => {
db = createDb();
migrate(db, ALL_MIGRATIONS);
repo = new QueueRepository(db, new EventBus(db));
configureSyncSiteRecorder({
runSync<T>(site: string, fn: () => T): T { sync.push(site); return fn(); },
recordSlowSync(site: string, _durationMs: number, outcome: "ok" | "failed"): void { slow.push({ site, outcome }); },
});
});
afterEach(() => { configureSyncSiteRecorder(undefined); sync.length = 0; slow.length = 0; db.close(); });
it("names a handoff", async () => {
const source = await repo.create({ sourceSession: "planner@r", destinationSession: "relay@r", body: "work", nudge: false });
sync.length = 0;
const { created } = await repo.handoff({ qitemId: source.qitemId, fromSession: "relay@r", toSession: "worker@r", nudge: false });
expect(created.destinationSession).toBe("worker@r");
expect(sync).toContain("queue.handoff");
});
it("names decisionRevision's reply read through the slow-only site, once per revision", async () => {
const ask = await repo.create({ sourceSession: "planner@r", destinationSession: "relay@r", body: "work", nudge: false });
slow.length = 0;
const revision = repo.decisionRevision(repo.getById(ask.qitemId)!);
expect(typeof revision).toBe("string");
expect(slow).toEqual([{ site: "queue.decision_revision.replies", outcome: "ok" }]);
});
it("names the stuck sweep's linked-successor build when it is built", () => {
const lookup = createLinkedSuccessorLookup(db, ["qitem-source"]);
expect(sync).toEqual([]);
expect(lookup.has("qitem-source", "worker@r")).toBe(false);
expect(lookup.has("qitem-source", "other@r")).toBe(false);
expect(sync).toEqual(["queue.linked_successors.build"]);
});
it("keeps the wrapped call's result and error, and records each outcome", () => {
expect(runSyncSiteWhenSlow("test.site", () => 7)).toBe(7);
expect(() => runSyncSiteWhenSlow("test.site", () => { throw new Error("read failed"); })).toThrow("read failed");
expect(slow).toEqual([{ site: "test.site", outcome: "ok" }, { site: "test.site", outcome: "failed" }]);
});
it("never lets a failing record replace the wrapped call's result", () => {
configureSyncSiteRecorder({ recordSlowSync(): void { throw new Error("recorder gone"); } });
expect(runSyncSiteWhenSlow("test.site", () => "value")).toBe("value");
expect(() => runSyncSiteWhenSlow("test.site", () => { throw new Error("read failed"); })).toThrow("read failed");
});
it("runs the call bare when no recorder is configured", () => {
configureSyncSiteRecorder(undefined);
expect(runSyncSiteWhenSlow("test.site", () => "bare")).toBe("bare");
expect(slow).toEqual([]);
});
});
@@ -21,6 +21,7 @@ const expectedSites = new Map<string, string[]>([
["domain/codex-thread-id.ts", ["codex_thread_id.resolve_home"]],
["domain/review/gather.ts", ["review.gather.git"]],
["domain/tmux-option-defaults.ts", ["tmux_options.command_v"]],
["domain/queue-repository.ts", ["queue.decision_revision.replies", "queue.linked_successors.build", "queue.handoff"]],
]);
const pluginVendorUrl = "https://github.com/mvschwarz/openrig-plugins/releases/latest/download/openrig-core.tar.gz";
@@ -290,6 +291,23 @@ describe("SlowOpRecorder locked instrumentation contract", () => {
await recorder.close();
});
it("records a slow synchronous site only at or above the slow threshold, with its outcome", async () => {
const mod = await loadRecorderModule();
expect(mod, "slow-op-recorder production module is missing").not.toBeNull();
if (!mod) return;
const dir = tempDir();
const logPath = path.join(dir, "slow-operations.jsonl");
const recorder = new mod.SlowOpRecorder({ logPath, slowThresholdMs: 250 });
recorder.recordSlowSync("queue.decision_revision.replies", 249, "ok");
recorder.recordSlowSync("queue.decision_revision.replies", 250, "ok");
recorder.recordSlowSync("queue.decision_revision.replies", 400, "failed");
await recorder.flush();
const records = (await readRecords(logPath)).filter((record) => record.site === "queue.decision_revision.replies");
expect(records.map((record) => [record.phase, record.durationMs, record.outcome])).toEqual([["end", 250, "ok"], ["end", 400, "failed"]]);
await recorder.close();
});
// The request-timing MIDDLEWARE CONTRACT, hermetic + deterministic. The prior "real server" form
// ran the full daemon (event-loop monitor + a globalThis.fetch monkey-patch), which is structurally
// ambient-fragile under the concurrent-workspace gate: it intermittently returned 200 for an unmatched