fix(providers): bound launcher diagnostics without losing byte counts

Give Ghidra and Hopper an 8 MiB aggregate retained-output budget while continuing to drain their process pipes. Expose total observed bytes at the supervisor boundary and retained bytes plus explicit truncation in provider diagnostics.

Redact known transport tokens even when the retained prefix ends inside one, preserving adjacent caller path/query evidence. Existing partial-launch and cleanup ownership remain intact.

Validation: 285 tests across 34 focused suites passed; the final cutoff-fixture refinement passed all 31 Ghidra/Hopper diagnostic tests. check:fast and architecture guards passed. Provider fixtures use real child processes; real Ghidra/Hopper applications were not verified.
This commit is contained in:
morluto
2026-10-11 02:50:06 +08:00
parent cb13a27bfa
commit ba056e08a4
14 changed files with 302 additions and 33 deletions
+22 -6
View File
@@ -20,8 +20,12 @@ it.each([
const stderr =
"FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory\r\n";
const cause = new OwnedCommandFailure("process", "Decoder failed", {
stdout: { text: "", bytes: 0 },
stderr: { text: stderr, bytes: Buffer.byteLength(stderr) },
stdout: { text: "", bytes: 0, observedBytes: 0 },
stderr: {
text: stderr,
bytes: Buffer.byteLength(stderr),
observedBytes: Buffer.byteLength(stderr),
},
exitCode,
signal,
});
@@ -70,8 +74,12 @@ it.each([
format: "har",
});
const cause = new OwnedCommandFailure("process", "Decoder failed", {
stdout: { text: "", bytes: 0 },
stderr: { text: stderr, bytes: Buffer.byteLength(stderr) },
stdout: { text: "", bytes: 0, observedBytes: 0 },
stderr: {
text: stderr,
bytes: Buffer.byteLength(stderr),
observedBytes: Buffer.byteLength(stderr),
},
exitCode,
signal,
});
@@ -128,8 +136,16 @@ it("retains actual command status and both diagnostic streams when cleanup also
"process",
"Observed command failure",
{
stdout: { text: "original provider output", bytes: 24 },
stderr: { text: "private-marker provider reason", bytes: 30 },
stdout: {
text: "original provider output",
bytes: 24,
observedBytes: 24,
},
stderr: {
text: "private-marker provider reason",
bytes: 30,
observedBytes: 30,
},
exitCode: 2,
signal: null,
},
+11 -2
View File
@@ -16,8 +16,17 @@ export abstract class HopperError extends AnalysisError {}
export type HopperLauncherOutcome = {
readonly exit_code: number | null;
readonly signal: string | null;
readonly stdout: { readonly text: string; readonly bytes: number };
readonly stderr: { readonly text: string; readonly bytes: number };
/** Total bytes drained from the stream; retained_bytes reports captured bytes. */
readonly stdout: {
readonly text: string;
readonly bytes: number;
readonly retained_bytes: number;
};
readonly stderr: {
readonly text: string;
readonly bytes: number;
readonly retained_bytes: number;
};
readonly output_closed: boolean;
readonly diagnostic_truncated: boolean;
};
+6 -1
View File
@@ -24,7 +24,10 @@ import type {
} from "./GhidraClientTypes.js";
import type { GhidraInventoryOperation } from "./GhidraInventoryValues.js";
import type { GhidraFunctionOperation } from "./GhidraFunctionValues.js";
import { createGhidraDiagnostics } from "./GhidraDiagnostics.js";
import {
createGhidraDiagnostics,
GHIDRA_PROCESS_DIAGNOSTIC_BYTES,
} from "./GhidraDiagnostics.js";
import { GhidraLaunchError, type GhidraLaunch } from "./GhidraLauncher.js";
import { GhidraResponseBuffer } from "./GhidraResponseBuffer.js";
import { GhidraResponseRouter } from "./GhidraResponseRouter.js";
@@ -432,6 +435,7 @@ export class GhidraClient {
if (partialLaunch !== undefined) {
this.#launch = partialLaunch;
this.#process = new ProviderProcessSupervisor(partialLaunch, {
maxDiagnosticBytes: GHIDRA_PROCESS_DIAGNOSTIC_BYTES,
onDiagnostic: (event) => this.#onProcessDiagnostic(event),
});
}
@@ -444,6 +448,7 @@ export class GhidraClient {
}
this.#launch = launched.value;
this.#process = new ProviderProcessSupervisor(launched.value, {
maxDiagnosticBytes: GHIDRA_PROCESS_DIAGNOSTIC_BYTES,
onDiagnostic: (event) => this.#onProcessDiagnostic(event),
});
const connected = await this.#connect(endpoint, deadline);
+12 -9
View File
@@ -1,8 +1,12 @@
import type { JsonValue } from "../domain/jsonValue.js";
import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js";
import type { ProviderProcessSnapshot } from "../process/ProviderProcess.js";
import type { GhidraLaunch } from "./GhidraLauncher.js";
import type { GhidraTransportKind } from "./GhidraTransport.js";
/** Aggregate in-memory retention budget for this provider's launcher output. */
export const GHIDRA_PROCESS_DIAGNOSTIC_BYTES = 8 * 1024 * 1024;
/** Inputs needed to project bounded diagnostics for one Ghidra runtime. */
export interface GhidraDiagnosticsOptions {
readonly targetPath: string;
@@ -31,10 +35,6 @@ export const createGhidraDiagnostics = (
Object.keys(options.previous).length > 0
)
return options.previous;
const redacted = (value: string): string =>
options.token === undefined
? value
: value.replaceAll(options.token, "[REDACTED]");
return {
target_path: options.targetPath,
target_sha256: options.targetSha256,
@@ -67,16 +67,19 @@ export const createGhidraDiagnostics = (
: {
exit_code: options.snapshot.exitCode ?? null,
exit_signal: options.snapshot.signal ?? null,
stdout: boundedStream(options.snapshot.stdout, redacted),
stderr: boundedStream(options.snapshot.stderr, redacted),
diagnostic_truncated: options.snapshot.diagnosticTruncated === true,
stdout: boundedStream(options.snapshot.stdout, options.token),
stderr: boundedStream(options.snapshot.stderr, options.token),
}),
};
};
const boundedStream = (
stream: ProviderProcessSnapshot["stdout"],
redact: (value: string) => string,
token: string | undefined,
): JsonValue => ({
text: redact(stream.text),
bytes: stream.bytes,
text: redactCapturedTransportCredential(stream, token, "[REDACTED]"),
// Preserve the historical total-byte counter; text is now bounded separately.
bytes: stream.observedBytes,
retained_bytes: stream.bytes,
});
+23 -6
View File
@@ -35,6 +35,7 @@ import { silentLogger } from "../logger.js";
import type { Logger } from "pino";
import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js";
import { ProviderStartupDeadline } from "../process/ProviderDeadline.js";
import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js";
import { ProviderRunLineage } from "../process/ProviderRunLineage.js";
import {
type ProviderProcessDiagnostic,
@@ -64,6 +65,9 @@ import {
responseResult,
} from "./protocol.js";
/** Aggregate in-memory retention budget for this provider's launcher output. */
export const HOPPER_PROCESS_DIAGNOSTIC_BYTES = 8 * 1024 * 1024;
const SHUTDOWN_TIMEOUT_MS = 30_000;
const SESSION_ROOT = process.platform === "darwin" ? "/tmp" : tmpdir();
@@ -609,15 +613,27 @@ export class HopperClient {
const snapshot = this.#resources.processSupervisor?.snapshot();
const token = this.#token;
if (snapshot === undefined) return undefined;
const redact = (text: string): string =>
token === undefined
? text
: text.replaceAll(token, "[redacted transport credential]");
return {
exit_code: snapshot.exitCode ?? null,
signal: snapshot.signal ?? null,
stdout: { ...snapshot.stdout, text: redact(snapshot.stdout.text) },
stderr: { ...snapshot.stderr, text: redact(snapshot.stderr.text) },
stdout: {
text: redactCapturedTransportCredential(
snapshot.stdout,
token,
"[redacted transport credential]",
),
bytes: snapshot.stdout.observedBytes,
retained_bytes: snapshot.stdout.bytes,
},
stderr: {
text: redactCapturedTransportCredential(
snapshot.stderr,
token,
"[redacted transport credential]",
),
bytes: snapshot.stderr.observedBytes,
retained_bytes: snapshot.stderr.bytes,
},
output_closed: outputClosed,
diagnostic_truncated: snapshot.diagnosticTruncated === true,
};
@@ -743,6 +759,7 @@ export class HopperClient {
#attachLauncher(launch: BridgeLaunch): void {
this.#resources.processSupervisor = new ProviderProcessSupervisor(launch, {
maxDiagnosticBytes: HOPPER_PROCESS_DIAGNOSTIC_BYTES,
onDiagnostic: (event) => this.#onLauncherDiagnostic(launch, event),
});
}
@@ -0,0 +1,31 @@
import type { ProviderProcessSnapshot } from "./ProviderProcess.js";
/** Redact explicit transport credentials from complete and budget-cut output. */
export const redactCapturedTransportCredential = (
stream: Pick<
ProviderProcessSnapshot["stdout"],
"text" | "bytes" | "observedBytes"
>,
token: string | undefined,
replacement: string,
): string => {
if (token === undefined || token.length === 0) return stream.text;
const { text } = stream;
const cutOff = stream.observedBytes > stream.bytes;
if (cutOff && !text.endsWith(token)) {
const maximumPrefixLength = Math.min(text.length, token.length - 1);
for (
let prefixLength = maximumPrefixLength;
prefixLength > 0;
prefixLength -= 1
) {
if (text.endsWith(token.slice(0, prefixLength)))
return `${text
.slice(0, text.length - prefixLength)
.replaceAll(token, replacement)}${replacement}`;
}
}
return text.replaceAll(token, replacement);
};
+9 -3
View File
@@ -76,11 +76,16 @@ export interface ProviderProcessSnapshot {
readonly diagnosticTruncated?: boolean;
readonly stdout: {
readonly text: string;
/** Bytes retained in text, limited by maxDiagnosticBytes when configured. */
readonly bytes: number;
/** All bytes drained from this stream since construction or resetOutput. */
readonly observedBytes: number;
};
readonly stderr: {
readonly text: string;
/** Bytes retained in text, limited by maxDiagnosticBytes when configured. */
readonly bytes: number;
readonly observedBytes: number;
};
readonly exitCode: number | null | undefined;
readonly signal: NodeJS.Signals | null | undefined;
@@ -268,8 +273,8 @@ export class ProviderProcessSupervisor {
snapshot(): ProviderProcessSnapshot {
return {
...(this.#diagnosticTruncated ? { diagnosticTruncated: true } : {}),
stdout: this.#stdout.snapshot(),
stderr: this.#stderr.snapshot(),
stdout: this.#stdout.snapshot(this.#stdoutObservedBytes),
stderr: this.#stderr.snapshot(this.#stderrObservedBytes),
exitCode: this.#exitObservation?.code,
signal: this.#exitObservation?.signal,
};
@@ -488,11 +493,12 @@ class ProcessOutputCapture {
this.#chunks.push(Buffer.from(chunk));
}
snapshot(): ProviderProcessSnapshot["stdout"] {
snapshot(observedBytes: number): ProviderProcessSnapshot["stdout"] {
const text = Buffer.concat(this.#chunks, this.#bytes).toString("utf8");
return {
text,
bytes: this.#bytes,
observedBytes,
};
}
}
@@ -202,9 +202,11 @@ describe("provider process output and cleanup primitives", () => {
expect(snapshot).toMatchObject({
stdout: {
bytes: outputBytes,
observedBytes: outputBytes,
},
stderr: {
bytes: outputBytes,
observedBytes: outputBytes,
},
exitCode: 23,
signal: null,
@@ -29,6 +29,12 @@ it("bounds capture before diagnostic callbacks while draining both output stream
expect(
diagnostics.some((event) => event.type === "output" && event.truncated),
).toBe(true);
const snapshot = supervisor.snapshot();
expect(snapshot.stdout.observedBytes).toBe(70_000);
expect(snapshot.stderr.observedBytes).toBe(70_000);
expect(snapshot.stdout.bytes + snapshot.stderr.bytes).toBeLessThanOrEqual(
1024,
);
for (const stream of ["stdout", "stderr"])
expect(
diagnostics
@@ -11,6 +11,7 @@ import type { Logger } from "pino";
import { ok } from "../../../../src/domain/result.js";
import { GhidraClient } from "../../../../src/ghidra/GhidraClient.js";
import { GHIDRA_PROCESS_DIAGNOSTIC_BYTES } from "../../../../src/ghidra/GhidraDiagnostics.js";
import { GhidraHeadlessLauncher } from "../../../../src/ghidra/GhidraLauncher.js";
import type { GhidraDiagnostic } from "../../../../src/ghidra/GhidraClientTypes.js";
import type {
@@ -49,7 +50,8 @@ type FixtureMode =
| "hang_tools"
| "exit_tools"
| "silent"
| "exit";
| "exit"
| "diagnostic_output";
class FixtureLauncher implements GhidraLauncher {
#notifyStarted: ((child: ChildProcess) => void) | undefined;
@@ -458,6 +460,53 @@ describe("GhidraClient cleanup and diagnostics", () => {
});
});
it("bounds long-lived launcher diagnostics without interrupting the session", async () => {
const launcher = new FixtureLauncher("diagnostic_output");
const client = clientFor(launcher);
await expect(client.start()).resolves.toMatchObject({ ok: true });
await expect(
client.callTool("list_strings", { document: null }),
).resolves.toMatchObject({ ok: true });
await vi.waitFor(() => {
expect(client.diagnostics()).toMatchObject({
diagnostic_truncated: true,
});
});
const diagnostics = client.diagnostics();
const stdout = diagnostics.stdout as {
bytes: number;
retained_bytes: number;
text: string;
};
const stderr = diagnostics.stderr as {
bytes: number;
retained_bytes: number;
text: string;
};
expect(stdout.bytes + stderr.bytes).toBeGreaterThan(
GHIDRA_PROCESS_DIAGNOSTIC_BYTES,
);
expect(stdout.retained_bytes + stderr.retained_bytes).toBeLessThanOrEqual(
GHIDRA_PROCESS_DIAGNOSTIC_BYTES,
);
expect(stdout.bytes).toBeGreaterThanOrEqual(stdout.retained_bytes);
expect(stderr.bytes).toBeGreaterThanOrEqual(stderr.retained_bytes);
expect(JSON.stringify(diagnostics)).not.toContain(launcher.tokens[0] ?? "");
expect(`${stdout.text}${stderr.text}`).toContain("[REDACTED]");
expect(
stderr.text.endsWith(
"source=/tmp/local-evidence.bin?cursor=keep&[REDACTED]",
),
).toBe(true);
await client.close();
expect(exited(launcher.processes[0])).toBe(true);
await expect(access(launcher.runtimeRoots[0] ?? "")).rejects.toMatchObject({
code: "ENOENT",
});
});
it("retains actionable diagnostics without retaining its token", async () => {
const launcher = new FixtureLauncher("exit");
const events: GhidraDiagnostic[] = [];
@@ -0,0 +1,88 @@
import { access } from "node:fs/promises";
import { expect, it, onTestFinished, vi } from "vitest";
import { HopperTimeoutError } from "../../../../src/domain/hopperErrors.js";
import {
HopperClient,
HOPPER_PROCESS_DIAGNOSTIC_BYTES,
} from "../../../../src/hopper/HopperClient.js";
import { HopperFixtureLauncher } from "./hopperClient.fixture.js";
it("bounds long-lived launcher output while keeping the bridge usable", async () => {
let observedStderrBytes = 0;
const diagnosticLauncher = new HopperFixtureLauncher(undefined, "noisy");
const client = new HopperClient({
launcher: diagnosticLauncher,
startupTimeoutMs: 10_000,
onDiagnostic(event) {
if (event.type === "launcher-stderr") observedStderrBytes += event.bytes;
},
});
onTestFinished(async () => {
await client.close();
});
await expect(client.start()).resolves.toMatchObject({ ok: true });
const session = await client.callTool("echo", { value: "still usable" });
expect(session).toEqual({ ok: true, value: { value: "still usable" } });
await vi.waitFor(() => {
expect(observedStderrBytes).toBeGreaterThan(
HOPPER_PROCESS_DIAGNOSTIC_BYTES,
);
});
await client.close();
const directory = diagnosticLauncher.directories[0] ?? "";
await expect(access(directory)).rejects.toMatchObject({ code: "ENOENT" });
expect(diagnosticLauncher.processes[0]?.exitCode).not.toBeNull();
});
it("reports observed and retained output when capture cuts a token", async () => {
const diagnosticLauncher = new HopperFixtureLauncher(
undefined,
"noisy_no_socket",
);
const client = new HopperClient({
launcher: diagnosticLauncher,
startupTimeoutMs: 1_000,
});
onTestFinished(async () => {
await client.close();
});
const started = await client.start();
if (started.ok || !(started.error instanceof HopperTimeoutError))
throw new Error("Expected the no-socket fixture to time out");
const outcome = started.error.launcherOutcome;
expect(outcome).toMatchObject({ diagnostic_truncated: true });
if (outcome === undefined) throw new Error("Launcher outcome was omitted");
expect(outcome.stdout.bytes + outcome.stderr.bytes).toBeGreaterThan(
HOPPER_PROCESS_DIAGNOSTIC_BYTES,
);
expect(
outcome.stdout.retained_bytes + outcome.stderr.retained_bytes,
).toBeLessThanOrEqual(HOPPER_PROCESS_DIAGNOSTIC_BYTES);
expect(outcome.stdout.bytes).toBeGreaterThanOrEqual(
outcome.stdout.retained_bytes,
);
expect(outcome.stderr.bytes).toBeGreaterThanOrEqual(
outcome.stderr.retained_bytes,
);
expect(outcome.stderr.retained_bytes).toBe(HOPPER_PROCESS_DIAGNOSTIC_BYTES);
const token = diagnosticLauncher.tokens[0] ?? "";
expect(JSON.stringify(outcome)).not.toContain(token);
expect(`${outcome.stdout.text}${outcome.stderr.text}`).toContain(
"[redacted transport credential]",
);
expect(
outcome.stderr.text.endsWith(
"source=/tmp/local-evidence.bin?cursor=keep&[redacted transport credential]",
),
).toBe(true);
await client.close();
const directory = diagnosticLauncher.directories[0] ?? "";
await expect(access(directory)).rejects.toMatchObject({ code: "ENOENT" });
expect(diagnosticLauncher.processes[0]?.exitCode).not.toBeNull();
});
@@ -28,15 +28,20 @@ export class HopperFixtureLauncher implements BridgeLauncher {
readonly socketPaths: string[] = [];
readonly directories: string[] = [];
readonly runIds: string[] = [];
readonly tokens: string[] = [];
readonly processes: ChildProcess[] = [];
readonly requests: FixtureRequest[] = [];
constructor(readonly tokenOverride?: string) {}
constructor(
readonly tokenOverride?: string,
readonly diagnosticMode: "quiet" | "noisy" | "noisy_no_socket" = "quiet",
) {}
launch(session: BridgeSession) {
this.socketPaths.push(session.socketPath);
this.directories.push(session.directory);
this.runIds.push(session.runId);
this.tokens.push(session.token);
const child = spawn(
process.execPath,
[
@@ -44,8 +49,10 @@ export class HopperFixtureLauncher implements BridgeLauncher {
session.socketPath,
this.tokenOverride ?? session.token,
session.runId,
"acknowledge",
this.diagnosticMode,
],
{ stdio: ["ignore", "ignore", "pipe", "ipc"] },
{ stdio: ["ignore", "pipe", "pipe", "ipc"] },
);
child.on("message", (value: unknown) => {
const request = requestSchema.safeParse(value);
+10
View File
@@ -153,6 +153,16 @@ const handleRequest = (socket, server, request, state) => {
const handlePing = (socket, request, state) => {
state.pings.value += 1;
if (state.mode === "diagnostic_output" && state.pings.value === 1) {
const retainedTokenPrefixBytes = Math.floor(state.token.length / 2);
const preservedMarker = "source=/tmp/local-evidence.bin?cursor=keep&";
const fillLength = 8 * 1024 * 1024 - retainedTokenPrefixBytes;
const fill = "x".repeat(fillLength - Buffer.byteLength(preservedMarker));
process.stderr.write(`${fill}${preservedMarker}`);
setImmediate(() => {
process.stderr.write(state.token);
});
}
if (state.mode === "malformed") {
socket.write("{invalid\n");
return;
+23 -3
View File
@@ -1,7 +1,13 @@
import { createServer } from "node:net";
const [socketPath, token, runId, shutdownMode = "acknowledge"] =
process.argv.slice(2);
const [
socketPath,
token,
runId,
shutdownMode = "acknowledge",
diagnosticMode = "quiet",
] = process.argv.slice(2);
let diagnosticOutputEmitted = false;
const heldReplies = new Map();
if (process.send !== undefined)
process.on("message", (message) => {
@@ -16,6 +22,18 @@ const shutdownResult = () => ({
document_closed: shutdownMode !== "unconfirmed",
...(shutdownMode === "cleanup-required" ? { cleanup_required: true } : {}),
});
const emitDiagnosticOutput = () => {
if (diagnosticMode === "quiet" || diagnosticOutputEmitted) return;
diagnosticOutputEmitted = true;
const retainedTokenPrefixBytes = Math.floor(token.length / 2);
const preservedMarker = "source=/tmp/local-evidence.bin?cursor=keep&";
const fillLength = 8 * 1024 * 1024 - retainedTokenPrefixBytes;
const fill = "x".repeat(fillLength - Buffer.byteLength(preservedMarker));
process.stderr.write(`${fill}${preservedMarker}`);
setImmediate(() => {
process.stderr.write(token);
});
};
const capabilityUnavailableResult = (id) => ({
id,
error: {
@@ -182,6 +200,7 @@ const server = createServer((socket) => {
if (request.token !== token) {
send({ id: request.id, error: { code: -32001, message: "bad token" } });
} else if (request.method === "health") {
emitDiagnosticOutput();
send(
{
id: request.id,
@@ -249,5 +268,6 @@ const closeServer = () =>
if (process.connected) process.disconnect();
});
server.listen(socketPath);
if (diagnosticMode === "noisy_no_socket") emitDiagnosticOutput();
else server.listen(socketPath);
process.on("SIGTERM", closeServer);