mirror of
https://github.com/openclaw/lobster.git
synced 2026-08-25 12:50:33 -04:00
2d951036ee
* fix(runtime): keep process execution shell-free * chore: leave release notes to release automation
5836 lines
191 KiB
TypeScript
5836 lines
191 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { promises as fsp, readFileSync } from "node:fs";
|
|
import {
|
|
access,
|
|
chmod,
|
|
copyFile,
|
|
mkdtemp,
|
|
readFile,
|
|
readdir,
|
|
rm,
|
|
writeFile,
|
|
} from "node:fs/promises";
|
|
import { spawn } from "node:child_process";
|
|
import { createServer } from "node:http";
|
|
import { tmpdir } from "node:os";
|
|
import { dirname, join } from "node:path";
|
|
import { PassThrough } from "node:stream";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
import { createDefaultRegistry } from "../src/commands/registry.js";
|
|
import { runAbortableProcess } from "../src/abortable_process.js";
|
|
import { resumeToolRequest, runToolRequest } from "../src/core/tool_runtime.js";
|
|
import { decodeResumeToken } from "../src/resume.js";
|
|
import { runPipeline } from "../src/runtime.js";
|
|
import { keyToPath } from "../src/state/store.js";
|
|
import { encodeToken } from "../src/token.js";
|
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const __dirname = dirname(__filename);
|
|
|
|
function streamOf(items: unknown[]) {
|
|
return (async function* () {
|
|
for (const item of items) yield item;
|
|
})();
|
|
}
|
|
|
|
test("explicit cancellation accepts legacy embedded workflow tokens", async () => {
|
|
const token = encodeToken({
|
|
protocolVersion: 1,
|
|
v: 1,
|
|
kind: "workflow-file",
|
|
filePath: "/legacy/workflow.lobster",
|
|
resumeAtIndex: 1,
|
|
steps: { gate: { id: "gate", approved: false } },
|
|
args: {},
|
|
});
|
|
|
|
const result = await resumeToolRequest({ token, cancel: true });
|
|
assert.equal(result.ok, true);
|
|
assert.equal(result.status, "cancelled");
|
|
});
|
|
|
|
test("runAbortableProcess preserves UTF-8 characters split across pipe chunks", async () => {
|
|
const result = await runAbortableProcess({
|
|
command: process.execPath,
|
|
argv: [
|
|
"-e",
|
|
"process.stdout.write(Buffer.from([0xe2])); setTimeout(() => process.stdout.write(Buffer.from([0x82, 0xac])), 20)",
|
|
],
|
|
env: process.env,
|
|
notFoundMessage: "node missing",
|
|
});
|
|
assert.equal(result.stdout, "€");
|
|
});
|
|
|
|
test(
|
|
"runAbortableProcess does not shell-interpret an executable path",
|
|
{ skip: process.platform === "win32" },
|
|
async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-process-path-"));
|
|
try {
|
|
const marker = join(dir, "injected");
|
|
const executable = join(dir, "node;touch injected");
|
|
await fsp.symlink(process.execPath, executable);
|
|
|
|
const result = await runAbortableProcess({
|
|
command: executable,
|
|
argv: ["-e", "process.stdout.write('ok')"],
|
|
cwd: dir,
|
|
env: process.env,
|
|
notFoundMessage: "node missing",
|
|
});
|
|
|
|
assert.equal(result.stdout, "ok");
|
|
assert.equal(await fileExists(marker), false);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
);
|
|
|
|
test("runAbortableProcess does not spawn when its signal is already aborted", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-pre-aborted-process-"));
|
|
try {
|
|
const marker = join(dir, "spawned");
|
|
const controller = new AbortController();
|
|
controller.abort(new Error("abort before spawn"));
|
|
|
|
await assert.rejects(
|
|
runAbortableProcess({
|
|
command: process.execPath,
|
|
argv: ["-e", `require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'spawned')`],
|
|
env: process.env,
|
|
signal: controller.signal,
|
|
notFoundMessage: "node missing",
|
|
}),
|
|
/abort before spawn/,
|
|
);
|
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
assert.equal(await fileExists(marker), false);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test(
|
|
"runAbortableProcess without a signal keeps direct children in the terminal process group",
|
|
{ skip: process.platform !== "linux" },
|
|
async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-terminal-direct-process-"));
|
|
let terminal: ReturnType<typeof spawn> | undefined;
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const runner = join(repoRoot, "dist", "src", "abortable_process.js");
|
|
const wrapper = join(repoRoot, "test", "fixtures", "abortable-process-terminal-direct.mjs");
|
|
const groupFile = join(dir, "process-group");
|
|
const startedFile = join(dir, "started");
|
|
const completedFile = join(dir, "completed");
|
|
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
|
|
const command = `exec ${[process.execPath, wrapper, runner].map(quote).join(" ")}`;
|
|
terminal = spawn("script", ["-qefc", command, "/dev/null"], {
|
|
env: {
|
|
...process.env,
|
|
LOBSTER_TERMINAL_DIRECT_GROUP_FILE: groupFile,
|
|
LOBSTER_TERMINAL_DIRECT_STARTED_FILE: startedFile,
|
|
LOBSTER_TERMINAL_DIRECT_COMPLETED_FILE: completedFile,
|
|
},
|
|
stdio: ["ignore", "ignore", "ignore"],
|
|
});
|
|
await waitForFile(groupFile, 5000);
|
|
await waitForFile(startedFile, 5000);
|
|
const processGroup = Number(await readFile(groupFile, "utf8"));
|
|
assert.ok(Number.isInteger(processGroup) && processGroup > 0);
|
|
process.kill(-processGroup, "SIGINT");
|
|
await new Promise((resolve) => setTimeout(resolve, 900));
|
|
assert.equal(
|
|
await fileExists(completedFile),
|
|
false,
|
|
"a direct child must not complete after terminal interruption",
|
|
);
|
|
} finally {
|
|
terminal?.kill("SIGKILL");
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
);
|
|
|
|
async function fileExists(path: string) {
|
|
try {
|
|
await access(path);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function listStateFiles(stateDir: string) {
|
|
try {
|
|
return await readdir(stateDir);
|
|
} catch (err: any) {
|
|
if (err?.code === "ENOENT") return [];
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
async function waitForFile(path: string, timeoutMs = 2000) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (Date.now() < deadline) {
|
|
if (await fileExists(path)) return;
|
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
}
|
|
throw new Error(`Timed out waiting for ${path}`);
|
|
}
|
|
|
|
async function waitForChildProcess(parentPid: number, timeoutMs = 5000) {
|
|
const childrenPath = `/proc/${parentPid}/task/${parentPid}/children`;
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
const children = (await readFile(childrenPath, "utf8"))
|
|
.trim()
|
|
.split(/\s+/)
|
|
.map(Number)
|
|
.filter(Number.isInteger);
|
|
if (children.length === 1) return children[0];
|
|
} catch {
|
|
// The pseudo-terminal process may not have spawned the CLI yet.
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
}
|
|
throw new Error(`Timed out waiting for a child process of ${parentPid}`);
|
|
}
|
|
|
|
function startLobsterCli({
|
|
args,
|
|
cwd,
|
|
env,
|
|
interactive = false,
|
|
}: {
|
|
args: string[];
|
|
cwd: string;
|
|
env: NodeJS.ProcessEnv;
|
|
interactive?: boolean;
|
|
}) {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const entry = join(repoRoot, "bin", "lobster.js");
|
|
const shellQuote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
|
|
const command = `exec ${[process.execPath, entry, ...args].map(shellQuote).join(" ")}`;
|
|
const child = spawn(
|
|
interactive ? "script" : process.execPath,
|
|
interactive ? ["-qefc", command, "/dev/null"] : [entry, ...args],
|
|
{
|
|
cwd,
|
|
env,
|
|
stdio: [interactive ? "pipe" : "ignore", "pipe", "pipe"],
|
|
},
|
|
);
|
|
let stdout = "";
|
|
let stderr = "";
|
|
child.stdout?.setEncoding("utf8");
|
|
child.stdout?.on("data", (chunk) => {
|
|
stdout += chunk;
|
|
});
|
|
child.stderr?.setEncoding("utf8");
|
|
child.stderr?.on("data", (chunk) => {
|
|
stderr += chunk;
|
|
});
|
|
const result = new Promise<{
|
|
code: number | null;
|
|
signal: NodeJS.Signals | null;
|
|
stdout: string;
|
|
stderr: string;
|
|
}>((resolve, reject) => {
|
|
child.once("error", reject);
|
|
child.once("close", (code, signal) => resolve({ code, signal, stdout, stderr }));
|
|
});
|
|
return { child, result, getStdout: () => stdout };
|
|
}
|
|
|
|
async function waitForCliOutput(getStdout: () => string, pattern: RegExp, timeoutMs = 5000) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (Date.now() < deadline) {
|
|
if (pattern.test(getStdout())) return;
|
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
}
|
|
throw new Error(`Timed out waiting for CLI output matching ${pattern}`);
|
|
}
|
|
|
|
async function observeSettlement<T>(promise: Promise<T>, timeoutMs: number) {
|
|
return Promise.race([
|
|
promise.then((value) => ({ settled: true as const, value, settledAt: Date.now() })),
|
|
new Promise<{ settled: false }>((resolve) =>
|
|
setTimeout(() => resolve({ settled: false }), timeoutMs),
|
|
),
|
|
]);
|
|
}
|
|
|
|
function processIsRunning(pid: number) {
|
|
try {
|
|
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
const state = stat.slice(stat.lastIndexOf(")") + 2, stat.lastIndexOf(")") + 3);
|
|
if (state === "Z") return false;
|
|
} catch {
|
|
// /proc is unavailable on Windows and the process may already be reaped.
|
|
}
|
|
try {
|
|
process.kill(pid, 0);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function withCommands(
|
|
defaultRegistry: ReturnType<typeof createDefaultRegistry>,
|
|
...commands: any[]
|
|
) {
|
|
return {
|
|
get(name: string) {
|
|
return commands.find((command) => command.name === name) ?? defaultRegistry.get(name);
|
|
},
|
|
};
|
|
}
|
|
|
|
function assertCancellationEnvelope(envelope: Awaited<ReturnType<typeof runToolRequest>>) {
|
|
assert.equal(envelope.ok, false);
|
|
assert.equal(envelope.status, undefined);
|
|
assert.equal(envelope.error?.type, "runtime_error");
|
|
assert.match(envelope.error?.message ?? "", /abort/i);
|
|
}
|
|
|
|
function createCustomAbortSideEffect() {
|
|
let invocations = 0;
|
|
let markStarted!: () => void;
|
|
const started = new Promise<void>((resolve) => {
|
|
markStarted = resolve;
|
|
});
|
|
return {
|
|
command: {
|
|
name: "test.custom-abort-side-effect",
|
|
async run({ input, ctx }: { input: AsyncIterable<unknown>; ctx: { signal?: AbortSignal } }) {
|
|
const items = [];
|
|
for await (const item of input) items.push(item);
|
|
invocations += 1;
|
|
if (invocations === 1) {
|
|
assert.ok(ctx.signal);
|
|
markStarted();
|
|
await new Promise<void>((resolve) =>
|
|
ctx.signal!.addEventListener("abort", () => resolve(), { once: true }),
|
|
);
|
|
ctx.signal.throwIfAborted();
|
|
}
|
|
return { output: streamOf(items) };
|
|
},
|
|
},
|
|
started,
|
|
get invocations() {
|
|
return invocations;
|
|
},
|
|
};
|
|
}
|
|
|
|
function createCountingSideEffect(name: string) {
|
|
let invocations = 0;
|
|
return {
|
|
command: {
|
|
name,
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
invocations += 1;
|
|
const items = [];
|
|
for await (const item of input) items.push(item);
|
|
return { output: streamOf(items.length > 0 ? items : [{ ok: true }]) };
|
|
},
|
|
},
|
|
get invocations() {
|
|
return invocations;
|
|
},
|
|
};
|
|
}
|
|
|
|
test("pre-aborted tool pipeline does not start its first stage", async () => {
|
|
const controller = new AbortController();
|
|
controller.abort();
|
|
let sideEffectStarted = false;
|
|
const sideEffect = {
|
|
name: "test.side-effect",
|
|
async run() {
|
|
sideEffectStarted = true;
|
|
return { output: streamOf([{ ok: true }]) };
|
|
},
|
|
};
|
|
|
|
const envelope = await runToolRequest({
|
|
pipeline: "test.side-effect",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), sideEffect),
|
|
signal: controller.signal,
|
|
},
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(sideEffectStarted, false, "pre-aborted requests must not start side effects");
|
|
});
|
|
|
|
test("pre-aborted workflow pipeline does not start its first stage", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-pre-abort-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [{ id: "side-effect", pipeline: "test.side-effect" }],
|
|
}),
|
|
"utf8",
|
|
);
|
|
let sideEffectStarted = false;
|
|
const sideEffect = {
|
|
name: "test.side-effect",
|
|
async run() {
|
|
sideEffectStarted = true;
|
|
return { output: streamOf([{ ok: true }]) };
|
|
},
|
|
};
|
|
const controller = new AbortController();
|
|
controller.abort();
|
|
|
|
const envelope = await runToolRequest({
|
|
filePath,
|
|
ctx: {
|
|
cwd: dir,
|
|
registry: withCommands(createDefaultRegistry(), sideEffect),
|
|
signal: controller.signal,
|
|
},
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(sideEffectStarted, false, "pre-aborted workflows must not start side effects");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pre-aborted workflow shell does not start its process", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-shell-pre-abort-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const started = join(dir, "shell-started");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [{ id: "shell", run: `printf started > ${started}` }],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const controller = new AbortController();
|
|
controller.abort(new Error("pre-aborted workflow shell"));
|
|
|
|
const envelope = await runToolRequest({
|
|
filePath,
|
|
ctx: { cwd: dir, signal: controller.signal },
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(envelope.error?.message, "pre-aborted workflow shell");
|
|
assert.equal(await fileExists(started), false, "pre-aborted workflow must not spawn a shell");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pipeline input cancellation after state publication does not return a live resume token", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-pipeline-gate-cancel-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const controller = new AbortController();
|
|
const signal = controller.signal;
|
|
const throwIfAborted = signal.throwIfAborted.bind(signal);
|
|
let signalChecks = 0;
|
|
Object.defineProperty(signal, "throwIfAborted", {
|
|
value() {
|
|
signalChecks += 1;
|
|
if (signalChecks === 6) {
|
|
controller.abort(new Error("abort after pipeline input state publication"));
|
|
}
|
|
throwIfAborted();
|
|
},
|
|
});
|
|
|
|
const envelope = await runToolRequest({
|
|
pipeline: `ask --emit --prompt Continue? --schema '${JSON.stringify({ type: "object" })}'`,
|
|
ctx: { env: { ...process.env, LOBSTER_STATE_DIR: stateDir }, signal },
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(signalChecks, 6);
|
|
assert.deepEqual(await listStateFiles(stateDir), [], "cancelled input state must be removed");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pipeline approval cancellation after index publication removes the state and index", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-pipeline-approval-cancel-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const controller = new AbortController();
|
|
const signal = controller.signal;
|
|
const throwIfAborted = signal.throwIfAborted.bind(signal);
|
|
let signalChecks = 0;
|
|
Object.defineProperty(signal, "throwIfAborted", {
|
|
value() {
|
|
signalChecks += 1;
|
|
if (signalChecks === 9) {
|
|
controller.abort(new Error("abort after pipeline approval index publication"));
|
|
}
|
|
throwIfAborted();
|
|
},
|
|
});
|
|
const source = {
|
|
name: "test.gate-source",
|
|
async run() {
|
|
return { output: streamOf([{ value: 1 }]) };
|
|
},
|
|
};
|
|
|
|
const envelope = await runToolRequest({
|
|
pipeline: "test.gate-source | approve --prompt Continue?",
|
|
ctx: {
|
|
env: { ...process.env, LOBSTER_STATE_DIR: stateDir },
|
|
signal,
|
|
registry: withCommands(createDefaultRegistry(), source),
|
|
},
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(signalChecks, 9);
|
|
assert.deepEqual(
|
|
await listStateFiles(stateDir),
|
|
[],
|
|
"cancelled approval state must be removed",
|
|
);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow input cancellation after state publication does not return a live resume token", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-gate-cancel-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const filePath = join(dir, "workflow.lobster");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "answer", input: { prompt: "Continue?", responseSchema: { type: "object" } } },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const controller = new AbortController();
|
|
const signal = controller.signal;
|
|
const throwIfAborted = signal.throwIfAborted.bind(signal);
|
|
let signalChecks = 0;
|
|
Object.defineProperty(signal, "throwIfAborted", {
|
|
value() {
|
|
signalChecks += 1;
|
|
if (signalChecks === 4) {
|
|
controller.abort(new Error("abort after workflow input state publication"));
|
|
}
|
|
throwIfAborted();
|
|
},
|
|
});
|
|
|
|
const envelope = await runToolRequest({
|
|
filePath,
|
|
ctx: { env: { ...process.env, LOBSTER_STATE_DIR: stateDir }, signal },
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(signalChecks, 4);
|
|
assert.deepEqual(await listStateFiles(stateDir), [], "cancelled input state must be removed");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow approval cancellation after index publication removes the state and index", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-approval-cancel-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const filePath = join(dir, "workflow.lobster");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({ steps: [{ id: "gate", approval: "Continue?" }] }),
|
|
"utf8",
|
|
);
|
|
const controller = new AbortController();
|
|
const signal = controller.signal;
|
|
const throwIfAborted = signal.throwIfAborted.bind(signal);
|
|
let signalChecks = 0;
|
|
Object.defineProperty(signal, "throwIfAborted", {
|
|
value() {
|
|
signalChecks += 1;
|
|
if (signalChecks === 5) {
|
|
controller.abort(new Error("abort after workflow approval index publication"));
|
|
}
|
|
throwIfAborted();
|
|
},
|
|
});
|
|
|
|
const envelope = await runToolRequest({
|
|
filePath,
|
|
ctx: { env: { ...process.env, LOBSTER_STATE_DIR: stateDir }, signal },
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(signalChecks, 5);
|
|
assert.deepEqual(
|
|
await listStateFiles(stateDir),
|
|
[],
|
|
"cancelled approval state must be removed",
|
|
);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pre-aborted pipeline approval resume remains retryable", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-pre-abort-approval-resume-"));
|
|
try {
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const sideEffect = createCountingSideEffect("test.pre-abort-approval-side-effect");
|
|
const source = {
|
|
name: "test.pre-abort-source",
|
|
async run() {
|
|
return { output: streamOf([{ value: 1 }]) };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), source, sideEffect.command);
|
|
const first = await runToolRequest({
|
|
pipeline:
|
|
"test.pre-abort-source | approve --prompt Continue? | test.pre-abort-approval-side-effect",
|
|
ctx: { registry, env },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
|
|
const controller = new AbortController();
|
|
controller.abort(new Error("pre-aborted approval resume"));
|
|
const aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { registry, signal: controller.signal, env },
|
|
});
|
|
assertCancellationEnvelope(aborted);
|
|
assert.equal(sideEffect.invocations, 0);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { registry, env },
|
|
});
|
|
assert.equal(retried.ok, true);
|
|
assert.deepEqual(retried.output, [{ value: 1 }]);
|
|
assert.equal(sideEffect.invocations, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pre-aborted pipeline resume does not wait for a live state lock", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-pre-abort-resume-live-lock-"));
|
|
try {
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const source = {
|
|
name: "test.pre-abort-live-lock-source",
|
|
async run() {
|
|
return { output: streamOf([{ value: 1 }]) };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), source);
|
|
const first = await runToolRequest({
|
|
pipeline: "test.pre-abort-live-lock-source | approve --prompt Continue?",
|
|
ctx: { registry, env },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const lockPath = `${keyToPath(env.LOBSTER_STATE_DIR!, payload.stateKey)}.lock`;
|
|
await fsp.mkdir(lockPath);
|
|
await fsp.writeFile(join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
|
|
|
const controller = new AbortController();
|
|
controller.abort(new Error("pre-aborted state lock resume"));
|
|
const pending = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { registry, signal: controller.signal, env },
|
|
});
|
|
const early = await observeSettlement(pending, 75);
|
|
if (!early.settled) await fsp.rm(lockPath, { recursive: true, force: true });
|
|
const envelope = early.settled ? early.value : await pending;
|
|
|
|
assert.equal(early.settled, true, "a pre-aborted resume must not wait for a live state lock");
|
|
assertCancellationEnvelope(envelope);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pre-aborted pipeline input resume remains retryable", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-pre-abort-input-resume-"));
|
|
try {
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const sideEffect = createCountingSideEffect("test.pre-abort-input-side-effect");
|
|
const registry = withCommands(createDefaultRegistry(), sideEffect.command);
|
|
const responseSchema = JSON.stringify({
|
|
type: "object",
|
|
properties: { value: { type: "number" } },
|
|
required: ["value"],
|
|
});
|
|
const first = await runToolRequest({
|
|
pipeline: `ask --emit --prompt Continue? --schema '${responseSchema}' | test.pre-abort-input-side-effect`,
|
|
ctx: { registry, env },
|
|
});
|
|
assert.equal(first.status, "needs_input");
|
|
assert.ok(first.requiresInput?.resumeToken);
|
|
|
|
const controller = new AbortController();
|
|
controller.abort(new Error("pre-aborted input resume"));
|
|
const aborted = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response: { value: 1 },
|
|
ctx: { registry, signal: controller.signal, env },
|
|
});
|
|
assertCancellationEnvelope(aborted);
|
|
assert.equal(sideEffect.invocations, 0);
|
|
|
|
const retried = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response: { value: 1 },
|
|
ctx: { registry, env },
|
|
});
|
|
assert.equal(retried.ok, true);
|
|
assert.deepEqual(retried.output, [{ value: 1 }]);
|
|
assert.equal(sideEffect.invocations, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pipeline approval resume remains retryable when a resume-safe stage aborts before input", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-safe-input-setup-abort-"));
|
|
try {
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const controller = new AbortController();
|
|
let abortBeforeInput = true;
|
|
const safeInput = {
|
|
name: "test.resume-safe-before-input",
|
|
meta: { resumeSafeBeforeInput: true },
|
|
async run({ ctx }: any) {
|
|
if (abortBeforeInput) {
|
|
abortBeforeInput = false;
|
|
controller.abort(new Error("abort before resume-safe input"));
|
|
}
|
|
await ctx.requestInput({
|
|
prompt: "Second?",
|
|
responseSchema: { type: "object" },
|
|
});
|
|
return { output: [] };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), safeInput);
|
|
const first = await runToolRequest({
|
|
pipeline: "approve --prompt First? | test.resume-safe-before-input",
|
|
ctx: { env, registry },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
|
|
const aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { env, registry, signal: controller.signal },
|
|
});
|
|
assertCancellationEnvelope(aborted);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { env, registry },
|
|
});
|
|
assert.equal(retried.status, "needs_input");
|
|
assert.equal(retried.requiresInput?.prompt, "Second?");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("same-stage ask resume restores its token when cancellation wins before execution", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-same-stage-ask-cancel-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const sideEffect = createCountingSideEffect("test.same-stage-ask-side-effect");
|
|
let activeController: AbortController | undefined;
|
|
const inputGate = {
|
|
name: "test.same-stage-ask-gate",
|
|
meta: { resumeSafeBeforeInput: true, resumeSafeAfterInput: true },
|
|
async run({ ctx }: any) {
|
|
await ctx.requestInput({ prompt: "First?", responseSchema: { type: "object" } });
|
|
activeController?.abort(new Error("abort after same-stage input delivery"));
|
|
return { output: streamOf([]) };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), inputGate, sideEffect.command);
|
|
const first = await runToolRequest({
|
|
pipeline: "test.same-stage-ask-gate | test.same-stage-ask-side-effect",
|
|
ctx: { env, registry },
|
|
});
|
|
assert.equal(first.status, "needs_input");
|
|
assert.ok(first.requiresInput?.resumeToken);
|
|
|
|
const controller = new AbortController();
|
|
activeController = controller;
|
|
const aborted = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response: { decision: "approve" },
|
|
ctx: { env, registry, signal: controller.signal },
|
|
});
|
|
activeController = undefined;
|
|
assertCancellationEnvelope(aborted);
|
|
assert.equal(sideEffect.invocations, 0);
|
|
|
|
const retried = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response: { decision: "approve" },
|
|
ctx: { env, registry },
|
|
});
|
|
assert.equal(retried.ok, true);
|
|
assert.equal(sideEffect.invocations, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("same-stage input resumes stop waiting on a held state lock", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-same-stage-input-lock-abort-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const workflowPath = join(dir, "workflow.lobster");
|
|
await writeFile(
|
|
workflowPath,
|
|
JSON.stringify({ steps: [{ id: "review", pipeline: "ask --prompt Review?" }] }),
|
|
"utf8",
|
|
);
|
|
|
|
for (const run of [
|
|
() => runToolRequest({ pipeline: "ask --prompt Review?", ctx: { cwd: dir, env } }),
|
|
() => runToolRequest({ filePath: workflowPath, ctx: { cwd: dir, env } }),
|
|
]) {
|
|
const first = await run();
|
|
assert.equal(first.status, "needs_input");
|
|
assert.ok(first.requiresInput?.resumeToken);
|
|
const payload = decodeResumeToken(first.requiresInput.resumeToken);
|
|
const lockPath = `${keyToPath(stateDir, payload.stateKey)}.lock`;
|
|
await fsp.mkdir(lockPath);
|
|
await fsp.writeFile(join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
|
|
|
const controller = new AbortController();
|
|
const originalMkdir = fsp.mkdir;
|
|
let signalLockAttempt: (() => void) | undefined;
|
|
const lockAttempted = new Promise<void>((resolve) => {
|
|
signalLockAttempt = resolve;
|
|
});
|
|
Object.defineProperty(fsp, "mkdir", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(
|
|
filePath: Parameters<typeof fsp.mkdir>[0],
|
|
options?: Parameters<typeof fsp.mkdir>[1],
|
|
) {
|
|
if (String(filePath) === lockPath) signalLockAttempt?.();
|
|
return originalMkdir(filePath, options);
|
|
},
|
|
});
|
|
|
|
let resumed;
|
|
try {
|
|
const pending = resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response: { decision: "approve" },
|
|
ctx: { cwd: dir, env, signal: controller.signal },
|
|
});
|
|
await lockAttempted;
|
|
controller.abort(
|
|
new Error("same-stage input cleanup stopped while waiting for a state lock"),
|
|
);
|
|
resumed = await observeSettlement(pending, 1500);
|
|
} finally {
|
|
Object.defineProperty(fsp, "mkdir", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalMkdir,
|
|
});
|
|
}
|
|
|
|
assert.equal(
|
|
resumed.settled,
|
|
true,
|
|
"same-stage cleanup must not remain blocked on a state lock",
|
|
);
|
|
if (resumed.settled) {
|
|
assert.equal(resumed.value.ok, false);
|
|
assert.equal(resumed.value.error?.type, "runtime_error");
|
|
}
|
|
assert.equal(await fileExists(keyToPath(stateDir, payload.stateKey)), true);
|
|
await rm(lockPath, { recursive: true, force: true });
|
|
|
|
const retried = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response: { decision: "approve" },
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(retried.status, "ok");
|
|
}
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow pipeline input resume restores its capability before execution starts", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-pipeline-safe-input-cancel-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const filePath = join(dir, "workflow.lobster");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({ steps: [{ id: "review", pipeline: "ask --prompt Review?" }] }),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_input");
|
|
assert.ok(first.requiresInput?.resumeToken);
|
|
const payload = decodeResumeToken(first.requiresInput.resumeToken);
|
|
const statePath = keyToPath(stateDir, payload.stateKey);
|
|
|
|
const controller = new AbortController();
|
|
const originalUnlink = fsp.unlink;
|
|
Object.defineProperty(fsp, "unlink", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(filePath: Parameters<typeof fsp.unlink>[0]) {
|
|
const result = await originalUnlink(filePath);
|
|
if (String(filePath) === statePath && !controller.signal.aborted) {
|
|
controller.abort(new Error("abort after safe workflow pipeline input cleanup"));
|
|
}
|
|
return result;
|
|
},
|
|
});
|
|
let aborted;
|
|
try {
|
|
aborted = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response: { decision: "approve" },
|
|
ctx: { cwd: dir, env, signal: controller.signal },
|
|
});
|
|
} finally {
|
|
Object.defineProperty(fsp, "unlink", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalUnlink,
|
|
});
|
|
}
|
|
assertCancellationEnvelope(aborted!);
|
|
assert.equal(await fileExists(statePath), true);
|
|
|
|
const retried = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response: { decision: "approve" },
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(retried.status, "ok");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pre-aborted workflow approval resume remains retryable", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-pre-abort-workflow-resume-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const sideEffect = createCountingSideEffect("test.pre-abort-workflow-side-effect");
|
|
const registry = withCommands(createDefaultRegistry(), sideEffect.command);
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "confirm", approval: "Continue?" },
|
|
{
|
|
id: "effect",
|
|
pipeline: "test.pre-abort-workflow-side-effect",
|
|
when: "$confirm.approved",
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, registry, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
|
|
const controller = new AbortController();
|
|
controller.abort(new Error("pre-aborted workflow resume"));
|
|
const aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, registry, signal: controller.signal, env },
|
|
});
|
|
assertCancellationEnvelope(aborted);
|
|
assert.equal(sideEffect.invocations, 0);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, registry, env },
|
|
});
|
|
assert.equal(retried.ok, true);
|
|
assert.equal(sideEffect.invocations, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow approval resume remains retryable when cancellation occurs during setup", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-setup-abort-workflow-resume-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const sideEffect = createCountingSideEffect("test.setup-abort-workflow-side-effect");
|
|
const registry = withCommands(createDefaultRegistry(), sideEffect.command);
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "confirm", approval: "Continue?" },
|
|
{
|
|
id: "effect",
|
|
pipeline: "test.setup-abort-workflow-side-effect",
|
|
when: "$confirm.approved",
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, registry, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
|
|
const controller = new AbortController();
|
|
const signal = controller.signal;
|
|
const throwIfAborted = signal.throwIfAborted.bind(signal);
|
|
let signalChecks = 0;
|
|
Object.defineProperty(signal, "throwIfAborted", {
|
|
value() {
|
|
signalChecks += 1;
|
|
if (signalChecks === 1) {
|
|
controller.abort(new Error("abort during workflow resume setup"));
|
|
}
|
|
throwIfAborted();
|
|
},
|
|
});
|
|
const aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, registry, signal, env },
|
|
});
|
|
assertCancellationEnvelope(aborted);
|
|
assert.equal(sideEffect.invocations, 0);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, registry, env },
|
|
});
|
|
assert.equal(retried.ok, true);
|
|
assert.equal(sideEffect.invocations, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pre-aborted terminal workflow approval resume remains retryable", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-pre-abort-terminal-workflow-resume-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({ steps: [{ id: "confirm", approval: "Continue?" }] }),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
|
|
const controller = new AbortController();
|
|
controller.abort(new Error("pre-aborted terminal workflow resume"));
|
|
const aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, signal: controller.signal, env },
|
|
});
|
|
assertCancellationEnvelope(aborted);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(retried.ok, true);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow approval resume remains retryable when cancellation creates its next input gate", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-input-gate-abort-workflow-resume-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "confirm", approval: "Continue?" },
|
|
{ id: "answer", input: { prompt: "Continue?", responseSchema: { type: "object" } } },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
|
|
const controller = new AbortController();
|
|
const signal = controller.signal;
|
|
const throwIfAborted = signal.throwIfAborted.bind(signal);
|
|
let signalChecks = 0;
|
|
Object.defineProperty(signal, "throwIfAborted", {
|
|
value() {
|
|
signalChecks += 1;
|
|
if (signalChecks === 3) {
|
|
controller.abort(new Error("abort while creating next workflow input gate"));
|
|
}
|
|
throwIfAborted();
|
|
},
|
|
});
|
|
const aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, signal, env },
|
|
});
|
|
assertCancellationEnvelope(aborted);
|
|
assert.equal(signalChecks, 3);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(retried.status, "needs_input");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow approval resume remains retryable when child workflow setup is cancelled", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-child-setup-abort-workflow-resume-"));
|
|
try {
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const parentPath = join(dir, "parent.lobster");
|
|
const childPath = join(dir, "child.lobster");
|
|
await writeFile(
|
|
parentPath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "confirm", approval: "Continue?" },
|
|
{ id: "child", workflow: "child.lobster" },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
await writeFile(
|
|
childPath,
|
|
JSON.stringify({ steps: [{ id: "child-step", run: "true" }] }),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath: parentPath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
|
|
const controller = new AbortController();
|
|
const originalRealpath = fsp.realpath;
|
|
let childSetupObserved = false;
|
|
let childRealpathCalls = 0;
|
|
Object.defineProperty(fsp, "realpath", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(filePath: Parameters<typeof fsp.realpath>[0]) {
|
|
const resolved = await originalRealpath(filePath);
|
|
if (String(filePath) === childPath) childRealpathCalls += 1;
|
|
if (childRealpathCalls === 2 && !controller.signal.aborted) {
|
|
childSetupObserved = true;
|
|
controller.abort(new Error("abort during child workflow setup"));
|
|
}
|
|
return resolved;
|
|
},
|
|
});
|
|
let aborted;
|
|
try {
|
|
aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, signal: controller.signal, env },
|
|
});
|
|
} finally {
|
|
Object.defineProperty(fsp, "realpath", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalRealpath,
|
|
});
|
|
}
|
|
assertCancellationEnvelope(aborted!);
|
|
assert.equal(childSetupObserved, true);
|
|
assert.equal(childRealpathCalls, 2);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(retried.ok, true);
|
|
assert.equal(retried.status, "ok");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("terminal pipeline approval resume restores its original approval ID when cleanup is cancelled", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-terminal-pipeline-cleanup-abort-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const first = await runToolRequest({
|
|
pipeline: "approve --prompt First?",
|
|
ctx: { env },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const previousStatePath = keyToPath(stateDir, payload.stateKey);
|
|
|
|
const controller = new AbortController();
|
|
const originalUnlink = fsp.unlink;
|
|
let previousStateDeleted = false;
|
|
Object.defineProperty(fsp, "unlink", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(filePath: Parameters<typeof fsp.unlink>[0]) {
|
|
const result = await originalUnlink(filePath);
|
|
if (String(filePath) === previousStatePath && !controller.signal.aborted) {
|
|
previousStateDeleted = true;
|
|
controller.abort(new Error("abort during terminal pipeline approval cleanup"));
|
|
}
|
|
return result;
|
|
},
|
|
});
|
|
let aborted;
|
|
try {
|
|
aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { env, signal: controller.signal },
|
|
});
|
|
} finally {
|
|
Object.defineProperty(fsp, "unlink", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalUnlink,
|
|
});
|
|
}
|
|
assertCancellationEnvelope(aborted!);
|
|
assert.equal(previousStateDeleted, true);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { env },
|
|
});
|
|
assert.equal(retried.ok, true);
|
|
assert.equal(retried.status, "ok");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pipeline approval resume restores its original capability after next-approval cleanup is cancelled", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-pipeline-approval-cleanup-abort-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const first = await runToolRequest({
|
|
pipeline: "approve --prompt First? | approve --prompt Second?",
|
|
ctx: { env },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const previousStatePath = keyToPath(stateDir, payload.stateKey);
|
|
|
|
const controller = new AbortController();
|
|
const originalRename = fsp.rename;
|
|
let previousStateClaimed = false;
|
|
Object.defineProperty(fsp, "rename", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(from: Parameters<typeof fsp.rename>[0], to: Parameters<typeof fsp.rename>[1]) {
|
|
const result = await originalRename(from, to);
|
|
if (String(to) === previousStatePath && !controller.signal.aborted) {
|
|
previousStateClaimed = true;
|
|
controller.abort(new Error("abort after previous pipeline approval state claim"));
|
|
}
|
|
return result;
|
|
},
|
|
});
|
|
let aborted;
|
|
try {
|
|
aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { env, signal: controller.signal },
|
|
});
|
|
} finally {
|
|
Object.defineProperty(fsp, "rename", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalRename,
|
|
});
|
|
}
|
|
assertCancellationEnvelope(aborted!);
|
|
assert.equal(previousStateClaimed, true);
|
|
const stateFilesAfterAbort = await listStateFiles(stateDir);
|
|
assert.equal(
|
|
stateFilesAfterAbort.filter((name) => name.startsWith("pipeline_resume_")).length,
|
|
1,
|
|
"cancellation must discard the unpublished next approval state",
|
|
);
|
|
assert.equal(
|
|
stateFilesAfterAbort.filter((name) => name.startsWith("approval_")).length,
|
|
1,
|
|
"cancellation must discard the unpublished next approval index",
|
|
);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { env },
|
|
});
|
|
assert.equal(retried.status, "needs_approval");
|
|
assert.equal(retried.requiresApproval?.prompt, "Second?");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pipeline approval resume restores its original capability after next-input cleanup is cancelled", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-pipeline-input-cleanup-abort-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const first = await runToolRequest({
|
|
pipeline: `approve --prompt First? | ask --emit --prompt Second? --schema '${JSON.stringify({ type: "object" })}'`,
|
|
ctx: { env },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const previousStatePath = keyToPath(stateDir, payload.stateKey);
|
|
|
|
const controller = new AbortController();
|
|
const originalRename = fsp.rename;
|
|
let previousStateClaimed = false;
|
|
Object.defineProperty(fsp, "rename", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(from: Parameters<typeof fsp.rename>[0], to: Parameters<typeof fsp.rename>[1]) {
|
|
const result = await originalRename(from, to);
|
|
if (String(to) === previousStatePath && !controller.signal.aborted) {
|
|
previousStateClaimed = true;
|
|
controller.abort(new Error("abort after previous pipeline input state claim"));
|
|
}
|
|
return result;
|
|
},
|
|
});
|
|
let aborted;
|
|
try {
|
|
aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { env, signal: controller.signal },
|
|
});
|
|
} finally {
|
|
Object.defineProperty(fsp, "rename", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalRename,
|
|
});
|
|
}
|
|
assertCancellationEnvelope(aborted!);
|
|
assert.equal(previousStateClaimed, true);
|
|
const stateFilesAfterAbort = await listStateFiles(stateDir);
|
|
assert.equal(
|
|
stateFilesAfterAbort.filter((name) => name.startsWith("pipeline_resume_")).length,
|
|
1,
|
|
"cancellation must discard the unpublished next input state",
|
|
);
|
|
assert.equal(
|
|
stateFilesAfterAbort.filter((name) => name.startsWith("approval_")).length,
|
|
1,
|
|
"cancellation must discard the unpublished next input index",
|
|
);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { env },
|
|
});
|
|
assert.equal(retried.status, "needs_input");
|
|
assert.equal(retried.requiresInput?.prompt, "Second?");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pipeline cancellation after side effects does not restore the consumed approval capability", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-pipeline-side-effect-gate-cancel-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
let sideEffects = 0;
|
|
const sideEffectThenInput = {
|
|
name: "test.side-effect-then-input",
|
|
async run({ ctx }: any) {
|
|
sideEffects += 1;
|
|
try {
|
|
await ctx.requestInput({
|
|
prompt: "Second?",
|
|
responseSchema: { type: "object" },
|
|
});
|
|
return { output: streamOf([]) };
|
|
} catch (err) {
|
|
controller.abort(new Error("abort after side-effecting pipeline state claim"));
|
|
throw err;
|
|
}
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), sideEffectThenInput);
|
|
const first = await runToolRequest({
|
|
pipeline: "approve --prompt First? | test.side-effect-then-input",
|
|
ctx: { env, registry },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
const controller = new AbortController();
|
|
const aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { env, registry, signal: controller.signal },
|
|
});
|
|
assertCancellationEnvelope(aborted!);
|
|
assert.equal(sideEffects, 1);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { env, registry },
|
|
});
|
|
assert.equal(retried.ok, false);
|
|
assert.equal(retried.error?.type, "parse_error");
|
|
assert.equal(sideEffects, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("same-stage resumed input consumes its capability before a custom command acts on it", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-resumed-input-side-effect-cancel-"));
|
|
try {
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const controller = new AbortController();
|
|
let sideEffects = 0;
|
|
const sideEffectAfterInput = {
|
|
name: "test.side-effect-after-input",
|
|
meta: { resumeSafeBeforeInput: true },
|
|
async run({ ctx }: any) {
|
|
await ctx.requestInput({ prompt: "Continue?", responseSchema: { type: "object" } });
|
|
sideEffects += 1;
|
|
controller.abort(new Error("abort after resumed input side effect"));
|
|
return { output: streamOf([]) };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), sideEffectAfterInput);
|
|
const first = await runToolRequest({
|
|
pipeline: "test.side-effect-after-input",
|
|
ctx: { env, registry },
|
|
});
|
|
assert.equal(first.status, "needs_input");
|
|
assert.ok(first.requiresInput?.resumeToken);
|
|
|
|
const aborted = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response: {},
|
|
ctx: { env, registry, signal: controller.signal },
|
|
});
|
|
assertCancellationEnvelope(aborted);
|
|
assert.equal(sideEffects, 1);
|
|
|
|
const replay = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response: {},
|
|
ctx: { env, registry },
|
|
});
|
|
assert.equal(replay.ok, false);
|
|
assert.notEqual(replay.status, "needs_input");
|
|
assert.equal(sideEffects, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow approval resume restores its original capability after next-input cleanup is cancelled", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-input-gate-cleanup-abort-workflow-resume-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "confirm", approval: "Continue?" },
|
|
{ id: "answer", input: { prompt: "Continue?", responseSchema: { type: "object" } } },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
|
|
const controller = new AbortController();
|
|
const signal = controller.signal;
|
|
const throwIfAborted = signal.throwIfAborted.bind(signal);
|
|
let signalChecks = 0;
|
|
Object.defineProperty(signal, "throwIfAborted", {
|
|
value() {
|
|
signalChecks += 1;
|
|
if (signalChecks === 5) {
|
|
controller.abort(new Error("abort after next-input state replaced the approval state"));
|
|
}
|
|
throwIfAborted();
|
|
},
|
|
});
|
|
const aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, signal, env },
|
|
});
|
|
assertCancellationEnvelope(aborted);
|
|
assert.equal(signalChecks, 5);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(retried.status, "needs_input");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow gate replacement retires the superseded approval index", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-gate-index-retirement-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "confirm", approval: "Continue?" },
|
|
{ id: "answer", input: { prompt: "Second?", responseSchema: { type: "object" } } },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
const oldIndexPath = join(stateDir, `approval_${first.requiresApproval.approvalId}.json`);
|
|
assert.equal(await fileExists(oldIndexPath), true);
|
|
|
|
const resumed = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(resumed.status, "needs_input");
|
|
assert.equal(await fileExists(oldIndexPath), false);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow approval resume restores its original capability after next-approval cleanup is cancelled", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-approval-gate-cleanup-abort-workflow-resume-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "first", approval: "Continue?" },
|
|
{ id: "second", approval: "Confirm again?" },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
|
|
const controller = new AbortController();
|
|
const signal = controller.signal;
|
|
const throwIfAborted = signal.throwIfAborted.bind(signal);
|
|
let signalChecks = 0;
|
|
Object.defineProperty(signal, "throwIfAborted", {
|
|
value() {
|
|
signalChecks += 1;
|
|
if (signalChecks === 6) {
|
|
controller.abort(
|
|
new Error("abort after next-approval state replaced the approval state"),
|
|
);
|
|
}
|
|
throwIfAborted();
|
|
},
|
|
});
|
|
const aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, signal, env },
|
|
});
|
|
assertCancellationEnvelope(aborted);
|
|
assert.equal(signalChecks, 6);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(retried.status, "needs_approval");
|
|
assert.equal(retried.requiresApproval?.prompt, "Confirm again?");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("terminal workflow approval resume restores its original capability when cleanup is cancelled", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-terminal-cleanup-abort-workflow-resume-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({ steps: [{ id: "confirm", approval: "Continue?" }] }),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
|
|
const controller = new AbortController();
|
|
const signal = controller.signal;
|
|
const throwIfAborted = signal.throwIfAborted.bind(signal);
|
|
let signalChecks = 0;
|
|
Object.defineProperty(signal, "throwIfAborted", {
|
|
value() {
|
|
signalChecks += 1;
|
|
if (signalChecks === 2) {
|
|
controller.abort(new Error("abort during terminal workflow resume cleanup"));
|
|
}
|
|
throwIfAborted();
|
|
},
|
|
});
|
|
const aborted = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, signal, env },
|
|
});
|
|
assertCancellationEnvelope(aborted);
|
|
assert.equal(signalChecks, 2);
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(retried.ok, true);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("a cancelled stale pipeline resume cannot restore a capability settled by another resume", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-stale-pipeline-resume-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
let delayedStageEntered!: () => void;
|
|
const delayedStageReady = new Promise<void>((resolve) => {
|
|
delayedStageEntered = resolve;
|
|
});
|
|
let releaseDelayedStage!: () => void;
|
|
const delayedStageReleased = new Promise<void>((resolve) => {
|
|
releaseDelayedStage = resolve;
|
|
});
|
|
let runs = 0;
|
|
let effects = 0;
|
|
const terminal = {
|
|
name: "test.stale-safe-terminal",
|
|
meta: { resumeSafeBeforeInput: true },
|
|
async run() {
|
|
runs += 1;
|
|
if (runs === 1) {
|
|
delayedStageEntered();
|
|
await delayedStageReleased;
|
|
} else {
|
|
effects += 1;
|
|
}
|
|
return { output: [] };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), terminal);
|
|
const first = await runToolRequest({
|
|
pipeline: "approve --prompt First? | test.stale-safe-terminal",
|
|
ctx: { env, registry },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const statePath = keyToPath(stateDir, payload.stateKey);
|
|
|
|
const controller = new AbortController();
|
|
const delayed = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { env, registry, signal: controller.signal },
|
|
});
|
|
await delayedStageReady;
|
|
|
|
const winner = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { env, registry },
|
|
});
|
|
assert.equal(winner.status, "ok");
|
|
assert.equal(effects, 1);
|
|
assert.equal(await fileExists(statePath), false);
|
|
|
|
const originalUnlink = fsp.unlink;
|
|
Object.defineProperty(fsp, "unlink", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(filePath: Parameters<typeof fsp.unlink>[0]) {
|
|
try {
|
|
return await originalUnlink(filePath);
|
|
} finally {
|
|
if (String(filePath) === statePath && !controller.signal.aborted) {
|
|
controller.abort(new Error("cancel stale pipeline terminal cleanup"));
|
|
}
|
|
}
|
|
},
|
|
});
|
|
let stale;
|
|
try {
|
|
releaseDelayedStage();
|
|
stale = await delayed;
|
|
} finally {
|
|
Object.defineProperty(fsp, "unlink", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalUnlink,
|
|
});
|
|
}
|
|
assert.equal(stale.ok, false);
|
|
assert.equal(await fileExists(statePath), false);
|
|
|
|
const replay = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { env, registry },
|
|
});
|
|
assert.equal(replay.ok, false);
|
|
assert.equal(effects, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("a cancelled stale workflow resume cannot restore a capability settled by another resume", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-stale-workflow-resume-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const effectPath = join(dir, "effects.log");
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "confirm", approval: "Continue?" },
|
|
{
|
|
id: "effect",
|
|
run: `node -e "require('fs').appendFileSync(process.argv[1], 'run\\n')" ${JSON.stringify(effectPath)}`,
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const statePath = keyToPath(stateDir, payload.stateKey);
|
|
|
|
let delayedPathEntered!: () => void;
|
|
const delayedPathReady = new Promise<void>((resolve) => {
|
|
delayedPathEntered = resolve;
|
|
});
|
|
let releaseDelayedPath!: () => void;
|
|
const delayedPathReleased = new Promise<void>((resolve) => {
|
|
releaseDelayedPath = resolve;
|
|
});
|
|
const controller = new AbortController();
|
|
const originalRealpath = fsp.realpath;
|
|
let delayedPathPaused = false;
|
|
Object.defineProperty(fsp, "realpath", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(...args: Parameters<typeof fsp.realpath>) {
|
|
if (!delayedPathPaused && String(args[0]) === filePath) {
|
|
delayedPathPaused = true;
|
|
delayedPathEntered();
|
|
await delayedPathReleased;
|
|
}
|
|
return originalRealpath(...args);
|
|
},
|
|
});
|
|
let stale;
|
|
try {
|
|
const delayed = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, signal: controller.signal },
|
|
});
|
|
await delayedPathReady;
|
|
Object.defineProperty(fsp, "realpath", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalRealpath,
|
|
});
|
|
|
|
const winner = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(winner.status, "ok");
|
|
assert.equal(await fileExists(statePath), false);
|
|
controller.abort(new Error("cancel stale workflow setup after winner settled"));
|
|
releaseDelayedPath();
|
|
stale = await delayed;
|
|
} finally {
|
|
Object.defineProperty(fsp, "realpath", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalRealpath,
|
|
});
|
|
}
|
|
assert.equal(stale.ok, false);
|
|
assert.equal(await fileExists(statePath), false);
|
|
|
|
const replay = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(replay.ok, false);
|
|
assert.equal((await readFile(effectPath, "utf8")).trim().split(/\r?\n/).length, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("cancellation stops terminal lazy output before reading its next item", async () => {
|
|
const controller = new AbortController();
|
|
let outputDrained = false;
|
|
const terminal = {
|
|
name: "test.terminal-output",
|
|
async run() {
|
|
return {
|
|
output: (async function* () {
|
|
yield { before: "abort" };
|
|
controller.abort();
|
|
yield { after: "abort" };
|
|
outputDrained = true;
|
|
})(),
|
|
};
|
|
},
|
|
};
|
|
|
|
const envelope = await runToolRequest({
|
|
pipeline: "test.terminal-output",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), terminal),
|
|
signal: controller.signal,
|
|
},
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(outputDrained, false, "terminal cancellation must not read another output item");
|
|
});
|
|
|
|
test("cancellation observed before terminal output does not start its iterator", async () => {
|
|
const controller = new AbortController();
|
|
let outputDrained = false;
|
|
const terminal = {
|
|
name: "test.pre-drain-abort-output",
|
|
async run() {
|
|
controller.abort(new Error("abort before terminal drain"));
|
|
return {
|
|
halt: true,
|
|
output: (async function* () {
|
|
yield { cleanup: true };
|
|
outputDrained = true;
|
|
})(),
|
|
};
|
|
},
|
|
};
|
|
|
|
const envelope = await runToolRequest({
|
|
pipeline: "test.pre-drain-abort-output",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), terminal),
|
|
signal: controller.signal,
|
|
},
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(envelope.error?.message, "abort before terminal drain");
|
|
assert.equal(outputDrained, false, "pre-aborted terminal output must not be read");
|
|
});
|
|
|
|
test("cancellation before halted lazy output skips its iterator and later pipeline stages", async () => {
|
|
const controller = new AbortController();
|
|
let outputDrained = false;
|
|
let downstreamRan = false;
|
|
const terminal = {
|
|
name: "test.halted-output-before-later-stage",
|
|
async run() {
|
|
controller.abort(new Error("abort before halted output drain"));
|
|
return {
|
|
halt: true,
|
|
output: (async function* () {
|
|
yield { cleanup: true };
|
|
outputDrained = true;
|
|
})(),
|
|
};
|
|
},
|
|
};
|
|
const downstream = {
|
|
name: "test.halted-output-unreached",
|
|
async run() {
|
|
downstreamRan = true;
|
|
return { output: [] };
|
|
},
|
|
};
|
|
|
|
const envelope = await runToolRequest({
|
|
pipeline: "test.halted-output-before-later-stage | test.halted-output-unreached",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), terminal, downstream),
|
|
signal: controller.signal,
|
|
},
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(envelope.error?.message, "abort before halted output drain");
|
|
assert.equal(outputDrained, false, "halted output must not be read after cancellation");
|
|
assert.equal(downstreamRan, false, "a halted stage must not dispatch later pipeline stages");
|
|
});
|
|
|
|
test("cancellation before a non-terminal lazy handoff does not wait for its next item", async () => {
|
|
const controller = new AbortController();
|
|
const source = {
|
|
name: "test.abort-before-lazy-handoff",
|
|
async run() {
|
|
controller.abort(new Error("abort before lazy handoff"));
|
|
return {
|
|
output: {
|
|
[Symbol.asyncIterator]() {
|
|
return {
|
|
next: () => new Promise<IteratorResult<unknown>>(() => {}),
|
|
async return() {
|
|
return { done: true, value: undefined };
|
|
},
|
|
};
|
|
},
|
|
},
|
|
};
|
|
},
|
|
};
|
|
|
|
const settled = await observeSettlement(
|
|
runToolRequest({
|
|
pipeline: "test.abort-before-lazy-handoff | test.unreached",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), source),
|
|
signal: controller.signal,
|
|
},
|
|
}),
|
|
500,
|
|
);
|
|
assert.equal(settled.settled, true, "cancellation must not wait for the stalled lazy handoff");
|
|
if (settled.settled) assertCancellationEnvelope(settled.value);
|
|
});
|
|
|
|
test("cancellation before a terminal lazy read invokes its iterator abort hook", async () => {
|
|
const controller = new AbortController();
|
|
let readStarted!: () => void;
|
|
const pendingReadStarted = new Promise<void>((resolve) => {
|
|
readStarted = resolve;
|
|
});
|
|
let resolveNext!: (value: IteratorResult<unknown>) => void;
|
|
let abortCalled = false;
|
|
const terminal = {
|
|
name: "test.cancellable-terminal-lazy-output",
|
|
async run() {
|
|
return {
|
|
halt: true,
|
|
output: {
|
|
[Symbol.asyncIterator]() {
|
|
return {
|
|
abort() {
|
|
abortCalled = true;
|
|
resolveNext({ done: true, value: undefined });
|
|
},
|
|
next() {
|
|
readStarted();
|
|
return new Promise<IteratorResult<unknown>>((resolve) => {
|
|
resolveNext = resolve;
|
|
});
|
|
},
|
|
async return() {
|
|
return { done: true, value: undefined };
|
|
},
|
|
};
|
|
},
|
|
},
|
|
};
|
|
},
|
|
};
|
|
|
|
const run = runToolRequest({
|
|
pipeline: "test.cancellable-terminal-lazy-output",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), terminal),
|
|
signal: controller.signal,
|
|
},
|
|
});
|
|
await pendingReadStarted;
|
|
controller.abort(new Error("abort cancellable terminal lazy output"));
|
|
const settled = await observeSettlement(run, 500);
|
|
assert.equal(settled.settled, true, "cancellation must release the terminal lazy read");
|
|
if (settled.settled) assertCancellationEnvelope(settled.value);
|
|
assert.equal(abortCalled, true);
|
|
});
|
|
|
|
test("cancellation falls back to an iterable-owned lazy abort hook", async () => {
|
|
const controller = new AbortController();
|
|
let readStarted!: () => void;
|
|
const pendingReadStarted = new Promise<void>((resolve) => {
|
|
readStarted = resolve;
|
|
});
|
|
let resolveNext!: (value: IteratorResult<unknown>) => void;
|
|
let abortCalled = false;
|
|
const source = {
|
|
name: "test.iterable-owned-lazy-abort",
|
|
async run() {
|
|
return {
|
|
output: {
|
|
abort() {
|
|
abortCalled = true;
|
|
resolveNext({ done: true, value: undefined });
|
|
},
|
|
[Symbol.asyncIterator]() {
|
|
return {
|
|
next() {
|
|
readStarted();
|
|
return new Promise<IteratorResult<unknown>>((resolve) => {
|
|
resolveNext = resolve;
|
|
});
|
|
},
|
|
async return() {
|
|
return { done: true, value: undefined };
|
|
},
|
|
};
|
|
},
|
|
},
|
|
};
|
|
},
|
|
};
|
|
|
|
const pending = runToolRequest({
|
|
pipeline: "test.iterable-owned-lazy-abort",
|
|
ctx: { registry: withCommands(createDefaultRegistry(), source), signal: controller.signal },
|
|
});
|
|
await pendingReadStarted;
|
|
controller.abort(new Error("cancel iterable-owned lazy output"));
|
|
const settled = await observeSettlement(pending, 500);
|
|
assert.equal(settled.settled, true);
|
|
if (settled.settled) {
|
|
assert.equal(settled.value.ok, false);
|
|
assert.equal(settled.value.error?.type, "runtime_error");
|
|
}
|
|
assert.equal(abortCalled, true);
|
|
});
|
|
|
|
test("cancellation does not pull another lazy output item after an in-flight read aborts", async () => {
|
|
const controller = new AbortController();
|
|
let afterAbortEffects = 0;
|
|
const source = {
|
|
name: "test.abort-does-not-pull-another-output",
|
|
async run() {
|
|
return {
|
|
output: (async function* () {
|
|
yield { value: 1 };
|
|
controller.abort(new Error("abort before next lazy output"));
|
|
afterAbortEffects += 1;
|
|
yield { value: 2 };
|
|
afterAbortEffects += 1;
|
|
yield { value: 3 };
|
|
})(),
|
|
};
|
|
},
|
|
};
|
|
|
|
const envelope = await runToolRequest({
|
|
pipeline: "test.abort-does-not-pull-another-output",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), source),
|
|
signal: controller.signal,
|
|
},
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(
|
|
afterAbortEffects,
|
|
1,
|
|
"cancellation must close the in-flight iterator instead of pulling a later item",
|
|
);
|
|
});
|
|
|
|
test("cancellable lazy output releases a pending read before cancellation returns", async () => {
|
|
const controller = new AbortController();
|
|
let readStarted!: () => void;
|
|
const pendingReadStarted = new Promise<void>((resolve) => {
|
|
readStarted = resolve;
|
|
});
|
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
let resolveNext!: (value: IteratorResult<unknown>) => void;
|
|
let abortCalled = false;
|
|
const source = {
|
|
name: "test.cancellable-lazy-output",
|
|
async run() {
|
|
return {
|
|
output: {
|
|
[Symbol.asyncIterator]() {
|
|
return {
|
|
abort() {
|
|
abortCalled = true;
|
|
if (timer) clearTimeout(timer);
|
|
resolveNext({ done: true, value: undefined });
|
|
},
|
|
next() {
|
|
readStarted();
|
|
return new Promise<IteratorResult<unknown>>((resolve) => {
|
|
resolveNext = resolve;
|
|
timer = setTimeout(() => resolve({ done: true, value: undefined }), 2000);
|
|
});
|
|
},
|
|
async return() {
|
|
return { done: true, value: undefined };
|
|
},
|
|
};
|
|
},
|
|
},
|
|
};
|
|
},
|
|
};
|
|
const downstream = {
|
|
name: "test.cancellable-lazy-downstream",
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
for await (const _item of input) {
|
|
// Draining begins the source's pending read.
|
|
}
|
|
return { output: streamOf([]) };
|
|
},
|
|
};
|
|
|
|
try {
|
|
const run = runToolRequest({
|
|
pipeline: "test.cancellable-lazy-output | test.cancellable-lazy-downstream",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), source, downstream),
|
|
signal: controller.signal,
|
|
},
|
|
});
|
|
await pendingReadStarted;
|
|
controller.abort(new Error("abort cancellable lazy output"));
|
|
const settled = await observeSettlement(run, 500);
|
|
assert.equal(settled.settled, true, "cancellation must release the pending lazy read");
|
|
if (settled.settled) assertCancellationEnvelope(settled.value);
|
|
assert.equal(abortCalled, true);
|
|
} finally {
|
|
if (timer) clearTimeout(timer);
|
|
}
|
|
});
|
|
|
|
test("a permanently stalled native lazy generator does not block cancellation", async () => {
|
|
const controller = new AbortController();
|
|
let readStarted!: () => void;
|
|
const pendingReadStarted = new Promise<void>((resolve) => {
|
|
readStarted = resolve;
|
|
});
|
|
let sourceClosed = false;
|
|
const source = {
|
|
name: "test.native-lazy-output",
|
|
async run() {
|
|
return {
|
|
output: (async function* () {
|
|
try {
|
|
readStarted();
|
|
await new Promise<void>(() => {});
|
|
} finally {
|
|
sourceClosed = true;
|
|
}
|
|
})(),
|
|
};
|
|
},
|
|
};
|
|
const downstream = {
|
|
name: "test.native-lazy-downstream",
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
for await (const _item of input) {
|
|
// Draining begins the source's pending read.
|
|
}
|
|
return { output: streamOf([]) };
|
|
},
|
|
};
|
|
|
|
const run = runToolRequest({
|
|
pipeline: "test.native-lazy-output | test.native-lazy-downstream",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), source, downstream),
|
|
signal: controller.signal,
|
|
},
|
|
});
|
|
await pendingReadStarted;
|
|
controller.abort(new Error("abort native lazy output"));
|
|
const settled = await observeSettlement(run, 500);
|
|
assert.equal(settled.settled, true, "cancellation must not wait for native generator cleanup");
|
|
if (settled.settled) assertCancellationEnvelope(settled.value);
|
|
assert.equal(
|
|
sourceClosed,
|
|
false,
|
|
"a native generator without an abort hook cannot guarantee prompt cleanup",
|
|
);
|
|
});
|
|
|
|
test("cancellation during lazy handoff stops yielding items to the downstream stage", async () => {
|
|
const controller = new AbortController();
|
|
const delivered: number[] = [];
|
|
let sourceClosed = false;
|
|
const source = {
|
|
name: "test.lazy-item-source",
|
|
async run() {
|
|
return {
|
|
output: (async function* () {
|
|
try {
|
|
yield 1;
|
|
yield 2;
|
|
yield 3;
|
|
} finally {
|
|
sourceClosed = true;
|
|
}
|
|
})(),
|
|
};
|
|
},
|
|
};
|
|
const downstream = {
|
|
name: "test.per-item-side-effect",
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
return {
|
|
output: (async function* () {
|
|
for await (const item of input) {
|
|
delivered.push(item as number);
|
|
if (delivered.length === 1) {
|
|
controller.abort(new Error("abort after first lazy item"));
|
|
}
|
|
yield item;
|
|
}
|
|
})(),
|
|
};
|
|
},
|
|
};
|
|
|
|
const envelope = await runToolRequest({
|
|
pipeline: "test.lazy-item-source | test.per-item-side-effect",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), source, downstream),
|
|
signal: controller.signal,
|
|
},
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(envelope.error?.message, "abort after first lazy item");
|
|
assert.deepEqual(delivered, [1], "cancelled handoff must not deliver later items");
|
|
await new Promise((resolve) => setImmediate(resolve));
|
|
assert.equal(sourceClosed, true, "cancelling the handoff must close the source iterator");
|
|
});
|
|
|
|
test("cancellation while draining lazy input prevents a downstream OpenClaw call", async () => {
|
|
const controller = new AbortController();
|
|
let requests = 0;
|
|
const server = createServer((_req, res) => {
|
|
requests += 1;
|
|
res.writeHead(200, { "content-type": "application/json" });
|
|
res.end(JSON.stringify({ ok: true, result: { sent: true } }));
|
|
});
|
|
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
|
|
try {
|
|
const address = server.address();
|
|
if (!address || typeof address === "string") throw new Error("Missing server address");
|
|
const source = {
|
|
name: "test.lazy-abort-source",
|
|
async run() {
|
|
return {
|
|
output: (async function* () {
|
|
yield { value: 1 };
|
|
controller.abort(new Error("aborted during lazy stage handoff"));
|
|
})(),
|
|
};
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), source);
|
|
const envelope = await runToolRequest({
|
|
pipeline: `test.lazy-abort-source | openclaw.invoke --url http://127.0.0.1:${address.port} --tool messages --action send`,
|
|
ctx: { registry, signal: controller.signal },
|
|
});
|
|
|
|
assert.equal(requests, 0, "cancellation must prevent the downstream POST");
|
|
assertCancellationEnvelope(envelope);
|
|
} finally {
|
|
await new Promise<void>((resolve, reject) => {
|
|
server.close((err) => (err ? reject(err) : resolve()));
|
|
});
|
|
}
|
|
});
|
|
|
|
test("parent cancellation terminates the gog process tree and skips Gmail sends", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cancel-search-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const searchStarted = join(dir, "search-started");
|
|
const searchTerminated = join(dir, "search-terminated");
|
|
const searchCompleted = join(dir, "search-completed");
|
|
const descendantStarted = join(dir, "descendant-started");
|
|
const descendantCompleted = join(dir, "descendant-completed");
|
|
const sendStarted = join(dir, "send-started");
|
|
const sendCompleted = join(dir, "send-completed");
|
|
const controller = new AbortController();
|
|
const run = runToolRequest({
|
|
pipeline: "gog.gmail.search --query newer_than:1d | gog.gmail.send",
|
|
ctx: {
|
|
signal: controller.signal,
|
|
env: {
|
|
...process.env,
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: searchStarted,
|
|
MOCK_GOG_SEARCH_TERMINATED_FILE: searchTerminated,
|
|
MOCK_GOG_SEARCH_COMPLETED_FILE: searchCompleted,
|
|
MOCK_GOG_DESCENDANT_STARTED_FILE: descendantStarted,
|
|
MOCK_GOG_DESCENDANT_COMPLETED_FILE: descendantCompleted,
|
|
MOCK_GOG_SEND_STARTED_FILE: sendStarted,
|
|
MOCK_GOG_SEND_COMPLETED_FILE: sendCompleted,
|
|
MOCK_GOG_TERMINATION_DELAY_MS: "1000",
|
|
},
|
|
},
|
|
});
|
|
|
|
await waitForFile(searchStarted);
|
|
await waitForFile(descendantStarted);
|
|
const childPid = Number(await readFile(searchStarted, "utf8"));
|
|
const descendantPid = Number(await readFile(descendantStarted, "utf8"));
|
|
const abortedAt = Date.now();
|
|
controller.abort();
|
|
const immediate = await observeSettlement(run, 50);
|
|
const observed = await observeSettlement(run, 500);
|
|
const envelope = observed.settled ? observed.value : await run;
|
|
|
|
assert.equal(immediate.settled, false, "workflow must wait for the child to exit");
|
|
assert.equal(observed.settled, true, "workflow should settle promptly after parent abort");
|
|
assertCancellationEnvelope(envelope);
|
|
if (process.platform !== "win32") {
|
|
await waitForFile(searchTerminated, 500);
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 700));
|
|
assert.equal(processIsRunning(childPid), false);
|
|
assert.equal(
|
|
processIsRunning(descendantPid),
|
|
false,
|
|
"cancellation must kill child processes too",
|
|
);
|
|
if (process.platform !== "win32") {
|
|
assert.equal(await fileExists(searchTerminated), true);
|
|
}
|
|
assert.equal(await fileExists(searchCompleted), false);
|
|
assert.equal(
|
|
await fileExists(descendantCompleted),
|
|
false,
|
|
"descendants must not continue after abort",
|
|
);
|
|
assert.equal(await fileExists(sendStarted), false);
|
|
assert.equal(await fileExists(sendCompleted), false);
|
|
if (observed.settled) assert.ok(observed.settledAt - abortedAt < 500);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow shell cancellation terminates descendant processes", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cancel-workflow-shell-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const searchStarted = join(dir, "search-started");
|
|
const descendantStarted = join(dir, "descendant-started");
|
|
const descendantCompleted = join(dir, "descendant-completed");
|
|
const controller = new AbortController();
|
|
const command = [process.execPath, mockGog, "gmail", "search"]
|
|
.map((arg) => JSON.stringify(arg))
|
|
.join(" ");
|
|
await writeFile(filePath, JSON.stringify({ steps: [{ id: "shell", run: command }] }), "utf8");
|
|
|
|
const run = runToolRequest({
|
|
filePath,
|
|
ctx: {
|
|
cwd: dir,
|
|
signal: controller.signal,
|
|
env: {
|
|
...process.env,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: searchStarted,
|
|
MOCK_GOG_DESCENDANT_STARTED_FILE: descendantStarted,
|
|
MOCK_GOG_DESCENDANT_COMPLETED_FILE: descendantCompleted,
|
|
MOCK_GOG_TERMINATION_DELAY_MS: "1000",
|
|
},
|
|
},
|
|
});
|
|
|
|
await waitForFile(searchStarted);
|
|
await waitForFile(descendantStarted);
|
|
const descendantPid = Number(await readFile(descendantStarted, "utf8"));
|
|
controller.abort();
|
|
const immediate = await observeSettlement(run, 50);
|
|
const envelope = await run;
|
|
|
|
assert.equal(immediate.settled, false, "workflow must wait for tree termination");
|
|
assertCancellationEnvelope(envelope);
|
|
await new Promise((resolve) => setTimeout(resolve, 900));
|
|
assert.equal(processIsRunning(descendantPid), false);
|
|
assert.equal(
|
|
await fileExists(descendantCompleted),
|
|
false,
|
|
"a workflow shell descendant must not complete after cancellation",
|
|
);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("exec cancellation terminates descendant processes", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cancel-exec-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const searchStarted = join(dir, "search-started");
|
|
const descendantStarted = join(dir, "descendant-started");
|
|
const descendantCompleted = join(dir, "descendant-completed");
|
|
const controller = new AbortController();
|
|
const run = runToolRequest({
|
|
pipeline: `exec ${process.execPath} ${mockGog} gmail search`,
|
|
ctx: {
|
|
signal: controller.signal,
|
|
env: {
|
|
...process.env,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: searchStarted,
|
|
MOCK_GOG_DESCENDANT_STARTED_FILE: descendantStarted,
|
|
MOCK_GOG_DESCENDANT_COMPLETED_FILE: descendantCompleted,
|
|
MOCK_GOG_TERMINATION_DELAY_MS: "1000",
|
|
},
|
|
},
|
|
});
|
|
|
|
await waitForFile(searchStarted);
|
|
await waitForFile(descendantStarted);
|
|
const descendantPid = Number(await readFile(descendantStarted, "utf8"));
|
|
controller.abort();
|
|
const envelope = await run;
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
await new Promise((resolve) => setTimeout(resolve, 900));
|
|
assert.equal(processIsRunning(descendantPid), false);
|
|
assert.equal(
|
|
await fileExists(descendantCompleted),
|
|
false,
|
|
"an exec descendant must not complete after cancellation",
|
|
);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("a configured workflow timeout preserves SIGTERM for external cancellation", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-timeout-external-cancel-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const workflow = join(dir, "workflow.lobster");
|
|
const started = join(dir, "started");
|
|
const terminated = join(dir, "terminated");
|
|
const command = [process.execPath, mockGog, "gmail", "search"]
|
|
.map((arg) => JSON.stringify(arg))
|
|
.join(" ");
|
|
await writeFile(
|
|
workflow,
|
|
JSON.stringify({ steps: [{ id: "waiting", run: command, timeout_ms: 10_000 }] }),
|
|
"utf8",
|
|
);
|
|
|
|
const controller = new AbortController();
|
|
const run = runToolRequest({
|
|
filePath: workflow,
|
|
ctx: {
|
|
cwd: dir,
|
|
env: {
|
|
...process.env,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: started,
|
|
MOCK_GOG_SEARCH_TERMINATED_FILE: terminated,
|
|
MOCK_GOG_TERMINATION_DELAY_MS: "20",
|
|
},
|
|
signal: controller.signal,
|
|
},
|
|
});
|
|
await waitForFile(started, 5000);
|
|
controller.abort(new Error("external abort"));
|
|
assertCancellationEnvelope(await run);
|
|
await waitForFile(terminated, 500);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("parallel wait:any preserves SIGTERM for a loser when a timeout is merely configured", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-parallel-any-graceful-cancel-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const workflow = join(dir, "workflow.lobster");
|
|
const loserStarted = join(dir, "loser-started");
|
|
const loserTerminated = join(dir, "loser-terminated");
|
|
const loser = [process.execPath, mockGog, "gmail", "search"]
|
|
.map((arg) => JSON.stringify(arg))
|
|
.join(" ");
|
|
const winner = `while [ ! -f ${JSON.stringify(loserStarted)} ]; do sleep 0.01; done; printf winner`;
|
|
await writeFile(
|
|
workflow,
|
|
JSON.stringify({
|
|
steps: [
|
|
{
|
|
id: "parallel",
|
|
parallel: {
|
|
wait: "any",
|
|
timeout_ms: 10_000,
|
|
branches: [
|
|
{ id: "loser", run: loser },
|
|
{ id: "winner", run: winner },
|
|
],
|
|
},
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
|
|
const result = await runToolRequest({
|
|
filePath: workflow,
|
|
ctx: {
|
|
cwd: dir,
|
|
env: {
|
|
...process.env,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: loserStarted,
|
|
MOCK_GOG_SEARCH_TERMINATED_FILE: loserTerminated,
|
|
MOCK_GOG_TERMINATION_DELAY_MS: "20",
|
|
},
|
|
},
|
|
});
|
|
assert.equal(result.ok, true, JSON.stringify(result));
|
|
await waitForFile(loserTerminated, 500);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test(
|
|
"CLI SIGINT terminates a detached workflow process tree before exiting",
|
|
{ skip: process.platform === "win32" },
|
|
async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cli-sigint-process-tree-"));
|
|
let cli: ReturnType<typeof spawn> | undefined;
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const workflow = join(dir, "workflow.lobster");
|
|
const searchStarted = join(dir, "search-started");
|
|
const descendantStarted = join(dir, "descendant-started");
|
|
const descendantCompleted = join(dir, "descendant-completed");
|
|
const command = [process.execPath, mockGog, "gmail", "search"]
|
|
.map((arg) => JSON.stringify(arg))
|
|
.join(" ");
|
|
await writeFile(workflow, JSON.stringify({ steps: [{ id: "shell", run: command }] }), "utf8");
|
|
|
|
cli = spawn(
|
|
process.execPath,
|
|
[join(repoRoot, "bin", "lobster.js"), "run", "--file", workflow],
|
|
{
|
|
cwd: dir,
|
|
env: {
|
|
...process.env,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: searchStarted,
|
|
MOCK_GOG_DESCENDANT_STARTED_FILE: descendantStarted,
|
|
MOCK_GOG_DESCENDANT_COMPLETED_FILE: descendantCompleted,
|
|
MOCK_GOG_TERMINATION_DELAY_MS: "1000",
|
|
},
|
|
stdio: "ignore",
|
|
},
|
|
);
|
|
await waitForFile(searchStarted, 5000);
|
|
await waitForFile(descendantStarted, 5000);
|
|
const descendantPid = Number(await readFile(descendantStarted, "utf8"));
|
|
assert.equal(cli.kill("SIGINT"), true);
|
|
const exited = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
|
|
(resolve, reject) => {
|
|
cli!.once("error", reject);
|
|
cli!.once("exit", (code, signal) => resolve({ code, signal }));
|
|
},
|
|
);
|
|
|
|
assert.equal(exited.signal, null);
|
|
assert.equal(exited.code, 130, "CLI must retain the conventional SIGINT exit status");
|
|
await new Promise((resolve) => setTimeout(resolve, 900));
|
|
assert.equal(processIsRunning(descendantPid), false);
|
|
assert.equal(
|
|
await fileExists(descendantCompleted),
|
|
false,
|
|
"a detached workflow descendant must not finish after Ctrl+C",
|
|
);
|
|
} finally {
|
|
cli?.kill("SIGKILL");
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
);
|
|
|
|
test(
|
|
"CLI SIGHUP terminates a detached workflow process tree before exiting",
|
|
{ skip: process.platform === "win32" },
|
|
async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cli-sighup-process-tree-"));
|
|
let cli: ReturnType<typeof spawn> | undefined;
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const workflow = join(dir, "workflow.lobster");
|
|
const searchStarted = join(dir, "search-started");
|
|
const descendantStarted = join(dir, "descendant-started");
|
|
const descendantCompleted = join(dir, "descendant-completed");
|
|
const command = [process.execPath, mockGog, "gmail", "search"]
|
|
.map((arg) => JSON.stringify(arg))
|
|
.join(" ");
|
|
await writeFile(
|
|
workflow,
|
|
JSON.stringify({ steps: [{ id: "shell", run: command, timeout_ms: 10_000 }] }),
|
|
"utf8",
|
|
);
|
|
|
|
cli = spawn(
|
|
process.execPath,
|
|
[join(repoRoot, "bin", "lobster.js"), "run", "--file", workflow],
|
|
{
|
|
cwd: dir,
|
|
env: {
|
|
...process.env,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: searchStarted,
|
|
MOCK_GOG_DESCENDANT_STARTED_FILE: descendantStarted,
|
|
MOCK_GOG_DESCENDANT_COMPLETED_FILE: descendantCompleted,
|
|
MOCK_GOG_TERMINATION_DELAY_MS: "1000",
|
|
},
|
|
stdio: "ignore",
|
|
},
|
|
);
|
|
await waitForFile(searchStarted, 5000);
|
|
await waitForFile(descendantStarted, 5000);
|
|
const descendantPid = Number(await readFile(descendantStarted, "utf8"));
|
|
assert.equal(cli.kill("SIGHUP"), true);
|
|
const exited = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
|
|
(resolve, reject) => {
|
|
cli!.once("error", reject);
|
|
cli!.once("exit", (code, signal) => resolve({ code, signal }));
|
|
},
|
|
);
|
|
|
|
assert.equal(exited.signal, null);
|
|
assert.equal(exited.code, 129, "CLI must retain the conventional SIGHUP exit status");
|
|
await new Promise((resolve) => setTimeout(resolve, 900));
|
|
assert.equal(processIsRunning(descendantPid), false);
|
|
assert.equal(
|
|
await fileExists(descendantCompleted),
|
|
false,
|
|
"a detached workflow descendant must not finish after terminal hangup",
|
|
);
|
|
} finally {
|
|
cli?.kill("SIGKILL");
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
);
|
|
|
|
test(
|
|
"a second CLI SIGINT immediately force-terminates a detached workflow process tree",
|
|
{ skip: process.platform === "win32" },
|
|
async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cli-second-sigint-process-tree-"));
|
|
let cli: ReturnType<typeof spawn> | undefined;
|
|
let descendantPid: number | undefined;
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const workflow = join(dir, "workflow.lobster");
|
|
const searchStarted = join(dir, "search-started");
|
|
const searchTerminated = join(dir, "search-terminated");
|
|
const descendantStarted = join(dir, "descendant-started");
|
|
const descendantCompleted = join(dir, "descendant-completed");
|
|
const command = [process.execPath, mockGog, "gmail", "search"]
|
|
.map((arg) => JSON.stringify(arg))
|
|
.join(" ");
|
|
await writeFile(
|
|
workflow,
|
|
JSON.stringify({ steps: [{ id: "shell", run: command, timeout_ms: 10_000 }] }),
|
|
"utf8",
|
|
);
|
|
|
|
cli = spawn(
|
|
process.execPath,
|
|
[join(repoRoot, "bin", "lobster.js"), "run", "--file", workflow],
|
|
{
|
|
cwd: dir,
|
|
env: {
|
|
...process.env,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: searchStarted,
|
|
MOCK_GOG_SEARCH_TERMINATED_FILE: searchTerminated,
|
|
MOCK_GOG_DESCENDANT_STARTED_FILE: descendantStarted,
|
|
MOCK_GOG_DESCENDANT_COMPLETED_FILE: descendantCompleted,
|
|
MOCK_GOG_TERMINATION_DELAY_MS: "1000",
|
|
},
|
|
stdio: "ignore",
|
|
},
|
|
);
|
|
await waitForFile(searchStarted, 5000);
|
|
await waitForFile(descendantStarted, 5000);
|
|
descendantPid = Number(await readFile(descendantStarted, "utf8"));
|
|
assert.equal(cli.kill("SIGINT"), true);
|
|
await waitForFile(searchTerminated, 5000);
|
|
assert.equal(cli.kill("SIGINT"), true);
|
|
await new Promise((resolve) => setTimeout(resolve, 75));
|
|
assert.equal(
|
|
processIsRunning(descendantPid),
|
|
false,
|
|
"a second interrupt must not wait for the graceful termination window",
|
|
);
|
|
} finally {
|
|
cli?.kill("SIGKILL");
|
|
if (descendantPid && processIsRunning(descendantPid)) process.kill(descendantPid, "SIGKILL");
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
);
|
|
|
|
test(
|
|
"CLI SIGINT prevents later pipeline stages from writing state",
|
|
{ skip: process.platform === "win32" },
|
|
async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cli-sigint-pipeline-handoff-"));
|
|
let cli: ReturnType<typeof startLobsterCli> | undefined;
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const stateDir = join(dir, "state");
|
|
const sendStarted = join(dir, "send-started");
|
|
const producer = `printf '${JSON.stringify([
|
|
{ to: "user@example.com", subject: "Hello", body: "World" },
|
|
])}'`;
|
|
const pipeline = `exec --json --shell ${JSON.stringify(producer)} | gog.gmail.send | state.set cli_after_abort`;
|
|
cli = startLobsterCli({
|
|
args: [pipeline],
|
|
cwd: dir,
|
|
env: {
|
|
...process.env,
|
|
LOBSTER_STATE_DIR: stateDir,
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_STARTED_FILE: sendStarted,
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "100",
|
|
},
|
|
});
|
|
await waitForFile(sendStarted, 5000);
|
|
assert.equal(cli.child.kill("SIGINT"), true);
|
|
const exited = await cli.result;
|
|
|
|
assert.equal(exited.signal, null);
|
|
assert.equal(exited.code, 130, "CLI must retain the conventional SIGINT exit status");
|
|
assert.equal(await fileExists(keyToPath(stateDir, "cli_after_abort")), false);
|
|
} finally {
|
|
cli?.child.kill("SIGKILL");
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
);
|
|
|
|
test(
|
|
"CLI SIGINT aborts a stalled OpenClaw HTTP call",
|
|
{ skip: process.platform === "win32" },
|
|
async () => {
|
|
let requestReceived!: () => void;
|
|
const requestStarted = new Promise<void>((resolve) => {
|
|
requestReceived = resolve;
|
|
});
|
|
const server = createServer((request) => {
|
|
request.resume();
|
|
requestReceived();
|
|
// Intentionally leave the response open so cancellation must abort fetch.
|
|
});
|
|
let cli: ReturnType<typeof startLobsterCli> | undefined;
|
|
try {
|
|
await new Promise<void>((resolve, reject) => {
|
|
server.once("error", reject);
|
|
server.listen(0, "127.0.0.1", resolve);
|
|
});
|
|
const address = server.address();
|
|
assert.ok(address && typeof address !== "string");
|
|
cli = startLobsterCli({
|
|
args: [
|
|
"openclaw.invoke",
|
|
"--url",
|
|
`http://127.0.0.1:${address.port}`,
|
|
"--tool",
|
|
"test",
|
|
"--action",
|
|
"wait",
|
|
],
|
|
cwd: tmpdir(),
|
|
env: process.env,
|
|
});
|
|
|
|
const requestObserved = await observeSettlement(requestStarted, 5000);
|
|
assert.equal(requestObserved.settled, true, "OpenClaw request must reach the server");
|
|
assert.equal(cli.child.kill("SIGINT"), true);
|
|
const exited = await observeSettlement(cli.result, 5000);
|
|
assert.equal(exited.settled, true, "CLI must exit after one SIGINT");
|
|
if (!exited.settled) return;
|
|
assert.equal(exited.value.signal, null);
|
|
assert.equal(exited.value.code, 130);
|
|
} finally {
|
|
cli?.child.kill("SIGKILL");
|
|
(server as any).closeAllConnections?.();
|
|
if (server.listening) {
|
|
await new Promise<void>((resolve) => server.close(() => resolve()));
|
|
}
|
|
}
|
|
},
|
|
);
|
|
|
|
test("interactive ctx.requestInput aborts its input read", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-request-input-abort-"));
|
|
try {
|
|
const stdin = new PassThrough() as PassThrough & { isTTY?: boolean };
|
|
Object.defineProperty(stdin, "isTTY", { value: true });
|
|
const stdout = new PassThrough();
|
|
let promptSeen!: () => void;
|
|
const prompt = new Promise<void>((resolve) => {
|
|
promptSeen = resolve;
|
|
});
|
|
stdout.once("data", () => promptSeen());
|
|
const controller = new AbortController();
|
|
const choose = {
|
|
name: "test.interactive-input",
|
|
async run({ ctx }: any) {
|
|
await ctx.requestInput({
|
|
prompt: "Choose a value",
|
|
responseSchema: { type: "object" },
|
|
});
|
|
return { output: streamOf([]) };
|
|
},
|
|
};
|
|
const run = runPipeline({
|
|
pipeline: [{ name: "test.interactive-input", args: { _: [] } }],
|
|
registry: withCommands(createDefaultRegistry(), choose),
|
|
input: [],
|
|
stdin,
|
|
stdout,
|
|
stderr: new PassThrough(),
|
|
env: { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") },
|
|
mode: "human",
|
|
signal: controller.signal,
|
|
});
|
|
|
|
await prompt;
|
|
controller.abort(new Error("input cancelled"));
|
|
await assert.rejects(run, /input cancelled/);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test(
|
|
"CLI SIGINT aborts interactive workflow input and approval prompts",
|
|
{ skip: process.platform !== "linux" },
|
|
async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cli-sigint-workflow-prompts-"));
|
|
try {
|
|
for (const scenario of [
|
|
{
|
|
name: "input",
|
|
prompt: /Enter JSON response:/,
|
|
workflow: {
|
|
steps: [
|
|
{
|
|
id: "answer",
|
|
input: { prompt: "Continue?", responseSchema: { type: "object" } },
|
|
},
|
|
],
|
|
},
|
|
},
|
|
{
|
|
name: "approval",
|
|
prompt: /Continue\? \[y\/N\]/,
|
|
workflow: { steps: [{ id: "approve", approval: "Continue?" }] },
|
|
},
|
|
]) {
|
|
const workflowPath = join(dir, `${scenario.name}.lobster`);
|
|
await writeFile(workflowPath, JSON.stringify(scenario.workflow), "utf8");
|
|
const cli = startLobsterCli({
|
|
args: ["run", "--mode", "human", "--file", workflowPath],
|
|
cwd: dir,
|
|
env: process.env,
|
|
interactive: true,
|
|
});
|
|
try {
|
|
await waitForCliOutput(cli.getStdout, scenario.prompt);
|
|
const cliPid = await waitForChildProcess(cli.child.pid!);
|
|
process.kill(cliPid, "SIGINT");
|
|
const exited = await observeSettlement(cli.result, 5000);
|
|
assert.equal(exited.settled, true, `${scenario.name} prompt must exit after one SIGINT`);
|
|
if (!exited.settled) continue;
|
|
assert.equal(exited.value.signal, null);
|
|
assert.equal(exited.value.code, 130);
|
|
} finally {
|
|
cli.child.kill("SIGKILL");
|
|
}
|
|
}
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
);
|
|
|
|
test(
|
|
"CLI SIGINT consumes a pipeline approval resume after execution starts",
|
|
{ skip: process.platform === "win32" },
|
|
async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cli-sigint-pipeline-resume-"));
|
|
let resumed: ReturnType<typeof startLobsterCli> | undefined;
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const stateDir = join(dir, "state");
|
|
const sendInvocations = join(dir, "send-invocations");
|
|
const sendCompleted = join(dir, "send-completed");
|
|
const searchStarted = join(dir, "search-started");
|
|
const searchTerminated = join(dir, "search-terminated");
|
|
const producer = `printf '${JSON.stringify([
|
|
{ to: "user@example.com", subject: "Hello", body: "World" },
|
|
])}'`;
|
|
const pipeline = `exec --json --shell ${JSON.stringify(producer)} | approve --prompt Send? | gog.gmail.send | gog.gmail.search --query newer_than:1d`;
|
|
const env = {
|
|
...process.env,
|
|
LOBSTER_STATE_DIR: stateDir,
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_INVOCATIONS_FILE: sendInvocations,
|
|
MOCK_GOG_SEND_COMPLETED_FILE: sendCompleted,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: searchStarted,
|
|
MOCK_GOG_SEARCH_TERMINATED_FILE: searchTerminated,
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "100",
|
|
};
|
|
const first = startLobsterCli({ args: ["run", "--mode", "tool", pipeline], cwd: dir, env });
|
|
const initial = await first.result;
|
|
assert.equal(initial.code, 0, initial.stderr);
|
|
const approval = JSON.parse(initial.stdout).requiresApproval;
|
|
assert.ok(approval?.resumeToken);
|
|
assert.ok(approval?.approvalId);
|
|
|
|
resumed = startLobsterCli({
|
|
args: ["resume", "--token", approval.resumeToken, "--approve", "yes"],
|
|
cwd: dir,
|
|
env,
|
|
});
|
|
await waitForFile(searchStarted, 5000);
|
|
assert.equal(resumed.child.kill("SIGINT"), true);
|
|
const exited = await resumed.result;
|
|
|
|
assert.equal(exited.signal, null);
|
|
assert.equal(exited.code, 130);
|
|
await waitForFile(sendCompleted, 1000);
|
|
await waitForFile(searchTerminated, 1000);
|
|
assert.equal((await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/).length, 1);
|
|
assert.equal(
|
|
(await listStateFiles(stateDir)).some((name) => /^(approval_|pipeline_resume_)/.test(name)),
|
|
false,
|
|
);
|
|
|
|
const replayById = startLobsterCli({
|
|
args: ["resume", "--id", approval.approvalId, "--approve", "yes"],
|
|
cwd: dir,
|
|
env,
|
|
});
|
|
const idReplay = await replayById.result;
|
|
assert.equal(idReplay.code, 2);
|
|
assert.match(idReplay.stdout, /not found or expired/);
|
|
const replayByToken = startLobsterCli({
|
|
args: ["resume", "--token", approval.resumeToken, "--approve", "yes"],
|
|
cwd: dir,
|
|
env,
|
|
});
|
|
const tokenReplay = await replayByToken.result;
|
|
assert.equal(tokenReplay.code, 1);
|
|
assert.match(tokenReplay.stdout, /Pipeline resume state not found/);
|
|
assert.equal(
|
|
(await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/).length,
|
|
1,
|
|
"retrying an interrupted CLI resume must not resend",
|
|
);
|
|
} finally {
|
|
resumed?.child.kill("SIGKILL");
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
);
|
|
|
|
test(
|
|
"CLI SIGINT reports a tool envelope while resume cleanup waits on a state lock",
|
|
{ skip: process.platform === "win32" },
|
|
async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cli-sigint-resume-cleanup-lock-"));
|
|
let resumed: ReturnType<typeof startLobsterCli> | undefined;
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const first = startLobsterCli({
|
|
args: ["run", "--mode", "tool", "approve --prompt Continue?"],
|
|
cwd: dir,
|
|
env,
|
|
});
|
|
const initial = await first.result;
|
|
assert.equal(initial.code, 0, initial.stderr);
|
|
const approval = JSON.parse(initial.stdout).requiresApproval;
|
|
assert.ok(approval?.resumeToken);
|
|
|
|
const payload = decodeResumeToken(approval.resumeToken);
|
|
const lockPath = `${keyToPath(stateDir, payload.stateKey)}.lock`;
|
|
await fsp.mkdir(lockPath);
|
|
await fsp.writeFile(join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
|
|
|
resumed = startLobsterCli({
|
|
args: ["resume", "--token", approval.resumeToken, "--cancel"],
|
|
cwd: dir,
|
|
env,
|
|
});
|
|
await new Promise((resolve) => setTimeout(resolve, 1500));
|
|
assert.equal(resumed.child.kill("SIGINT"), true);
|
|
const exited = await resumed.result;
|
|
|
|
assert.equal(exited.signal, null);
|
|
assert.equal(exited.code, 130);
|
|
const envelope = JSON.parse(exited.stdout);
|
|
assert.equal(envelope.ok, false);
|
|
assert.equal(envelope.error?.type, "runtime_error");
|
|
assert.match(envelope.error?.message ?? "", /Received SIGINT/);
|
|
await rm(lockPath, { recursive: true, force: true });
|
|
} finally {
|
|
resumed?.child.kill("SIGKILL");
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
);
|
|
|
|
test(
|
|
"CLI SIGINT consumes a workflow approval resume after execution starts",
|
|
{ skip: process.platform === "win32" },
|
|
async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cli-sigint-workflow-resume-"));
|
|
let resumed: ReturnType<typeof startLobsterCli> | undefined;
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const workflow = join(dir, "workflow.lobster");
|
|
const stateDir = join(dir, "state");
|
|
const sendInvocations = join(dir, "send-invocations");
|
|
const sendCompleted = join(dir, "send-completed");
|
|
const searchStarted = join(dir, "search-started");
|
|
const searchTerminated = join(dir, "search-terminated");
|
|
const producer = `printf '${JSON.stringify([
|
|
{ to: "user@example.com", subject: "Hello", body: "World" },
|
|
])}'`;
|
|
await writeFile(
|
|
workflow,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "confirm", approval: "Send?" },
|
|
{
|
|
id: "send",
|
|
pipeline: `exec --json --shell ${JSON.stringify(producer)} | gog.gmail.send | gog.gmail.search --query newer_than:1d`,
|
|
when: "$confirm.approved",
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const env = {
|
|
...process.env,
|
|
LOBSTER_STATE_DIR: stateDir,
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_INVOCATIONS_FILE: sendInvocations,
|
|
MOCK_GOG_SEND_COMPLETED_FILE: sendCompleted,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: searchStarted,
|
|
MOCK_GOG_SEARCH_TERMINATED_FILE: searchTerminated,
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "100",
|
|
};
|
|
const first = startLobsterCli({
|
|
args: ["run", "--mode", "tool", "--file", workflow],
|
|
cwd: dir,
|
|
env,
|
|
});
|
|
const initial = await first.result;
|
|
assert.equal(initial.code, 0, initial.stderr);
|
|
const approval = JSON.parse(initial.stdout).requiresApproval;
|
|
assert.ok(approval?.resumeToken);
|
|
assert.ok(approval?.approvalId);
|
|
|
|
resumed = startLobsterCli({
|
|
args: ["resume", "--token", approval.resumeToken, "--approve", "yes"],
|
|
cwd: dir,
|
|
env,
|
|
});
|
|
await waitForFile(searchStarted, 5000);
|
|
assert.equal(resumed.child.kill("SIGINT"), true);
|
|
const exited = await resumed.result;
|
|
|
|
assert.equal(exited.signal, null);
|
|
assert.equal(exited.code, 130);
|
|
await waitForFile(sendCompleted, 1000);
|
|
await waitForFile(searchTerminated, 1000);
|
|
assert.equal((await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/).length, 1);
|
|
assert.equal(
|
|
(await listStateFiles(stateDir)).some((name) =>
|
|
/^(approval_|workflow[-_]resume_)/.test(name),
|
|
),
|
|
false,
|
|
);
|
|
|
|
const replayById = startLobsterCli({
|
|
args: ["resume", "--id", approval.approvalId, "--approve", "yes"],
|
|
cwd: dir,
|
|
env,
|
|
});
|
|
const idReplay = await replayById.result;
|
|
assert.equal(idReplay.code, 2);
|
|
assert.match(idReplay.stdout, /not found or expired/);
|
|
const replayByToken = startLobsterCli({
|
|
args: ["resume", "--token", approval.resumeToken, "--approve", "yes"],
|
|
cwd: dir,
|
|
env,
|
|
});
|
|
const tokenReplay = await replayByToken.result;
|
|
assert.equal(tokenReplay.code, 1);
|
|
assert.match(tokenReplay.stdout, /Workflow resume state not found/);
|
|
assert.equal(
|
|
(await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/).length,
|
|
1,
|
|
"retrying an interrupted workflow resume must not resend",
|
|
);
|
|
} finally {
|
|
resumed?.child.kill("SIGKILL");
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
);
|
|
|
|
test("cancellation after Gmail search completion stops the next pipeline stage", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cancel-search-handoff-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const controller = new AbortController();
|
|
const signal = controller.signal;
|
|
const throwIfAborted = signal.throwIfAborted.bind(signal);
|
|
let signalChecks = 0;
|
|
Object.defineProperty(signal, "throwIfAborted", {
|
|
value() {
|
|
signalChecks += 1;
|
|
if (signalChecks === 3) controller.abort();
|
|
throwIfAborted();
|
|
},
|
|
});
|
|
|
|
let downstreamStarted = false;
|
|
const downstream = {
|
|
name: "test.side-effect",
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
downstreamStarted = true;
|
|
for await (const _item of input) {
|
|
// Drain search output before reporting success.
|
|
}
|
|
return { output: streamOf([{ ok: true }]) };
|
|
},
|
|
};
|
|
const envelope = await runToolRequest({
|
|
pipeline: "gog.gmail.search --query newer_than:1d | test.side-effect",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), downstream),
|
|
signal,
|
|
env: {
|
|
...process.env,
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "0",
|
|
},
|
|
},
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(signalChecks, 3);
|
|
assert.equal(downstreamStarted, false, "cancellation must stop downstream side effects");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("cancellation during eager stage cleanup stops the next tool stage", async () => {
|
|
const controller = new AbortController();
|
|
const source = {
|
|
name: "test.source",
|
|
async run() {
|
|
return {
|
|
output: (async function* () {
|
|
try {
|
|
yield { source: true };
|
|
yield { ignored: true };
|
|
} finally {
|
|
controller.abort();
|
|
}
|
|
})(),
|
|
};
|
|
},
|
|
};
|
|
const eager = {
|
|
name: "test.eager",
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
const iterator = input[Symbol.asyncIterator]();
|
|
const first = await iterator.next();
|
|
assert.equal(first.done, false);
|
|
return { output: [{ ok: true }] };
|
|
},
|
|
};
|
|
let downstreamStarted = false;
|
|
const downstream = {
|
|
name: "test.side-effect",
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
downstreamStarted = true;
|
|
const items = [];
|
|
for await (const item of input) items.push(item);
|
|
return { output: streamOf(items) };
|
|
},
|
|
};
|
|
|
|
const envelope = await runToolRequest({
|
|
pipeline: "test.source | test.eager | test.side-effect",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), source, eager, downstream),
|
|
signal: controller.signal,
|
|
},
|
|
});
|
|
|
|
assert.equal(controller.signal.aborted, true);
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(downstreamStarted, false, "cleanup cancellation must stop the next stage");
|
|
});
|
|
|
|
test("process-tree escalation survives a short-lived caller", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cancel-short-lived-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const runner = join(repoRoot, "dist", "src", "abortable_process.js");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const wrapper = join(repoRoot, "test", "fixtures", "abortable-process-short-lived.mjs");
|
|
const childStarted = join(dir, "search-started");
|
|
const descendantStarted = join(dir, "descendant-started");
|
|
const descendantCompleted = join(dir, "descendant-completed");
|
|
const child = spawn(process.execPath, [wrapper, runner, mockGog], {
|
|
env: {
|
|
...process.env,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: childStarted,
|
|
MOCK_GOG_DESCENDANT_STARTED_FILE: descendantStarted,
|
|
MOCK_GOG_DESCENDANT_COMPLETED_FILE: descendantCompleted,
|
|
MOCK_GOG_TERMINATION_DELAY_MS: "0",
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "5000",
|
|
},
|
|
stdio: "inherit",
|
|
});
|
|
const exitCode = await new Promise<number | null>((resolve, reject) => {
|
|
child.once("error", reject);
|
|
child.once("exit", resolve);
|
|
});
|
|
|
|
assert.equal(exitCode, 0, "the caller should settle cancellation before it exits");
|
|
await waitForFile(descendantStarted);
|
|
const descendantPid = Number(await readFile(descendantStarted, "utf8"));
|
|
await new Promise((resolve) => setTimeout(resolve, 900));
|
|
assert.equal(processIsRunning(descendantPid), false);
|
|
assert.equal(
|
|
await fileExists(descendantCompleted),
|
|
false,
|
|
"a descendant must not outlive a caller that exits after cancellation",
|
|
);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("an in-flight gog.gmail.send is terminated and the pipeline halts after cancellation", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cancel-send-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const sendStarted = join(dir, "send-started");
|
|
const sendTerminated = join(dir, "send-terminated");
|
|
const sendCompleted = join(dir, "send-completed");
|
|
const sendInvocations = join(dir, "send-invocations");
|
|
const controller = new AbortController();
|
|
const defaultRegistry = createDefaultRegistry();
|
|
const source = {
|
|
name: "draft",
|
|
async run() {
|
|
return {
|
|
output: streamOf([
|
|
{ to: "first@example.com", subject: "First", body: "Hello" },
|
|
{ to: "second@example.com", subject: "Second", body: "Hello again" },
|
|
]),
|
|
};
|
|
},
|
|
};
|
|
let downstreamStarted = false;
|
|
const downstream = {
|
|
name: "test.side-effect",
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
downstreamStarted = true;
|
|
const items = [];
|
|
for await (const item of input) items.push(item);
|
|
return { output: streamOf(items) };
|
|
},
|
|
};
|
|
const run = runToolRequest({
|
|
pipeline: "draft | gog.gmail.send | test.side-effect",
|
|
ctx: {
|
|
registry: withCommands(defaultRegistry, source, downstream),
|
|
signal: controller.signal,
|
|
env: {
|
|
...process.env,
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_STARTED_FILE: sendStarted,
|
|
MOCK_GOG_SEND_TERMINATED_FILE: sendTerminated,
|
|
MOCK_GOG_SEND_COMPLETED_FILE: sendCompleted,
|
|
MOCK_GOG_SEND_INVOCATIONS_FILE: sendInvocations,
|
|
MOCK_GOG_TERMINATION_DELAY_MS: "1000",
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "5000",
|
|
},
|
|
},
|
|
});
|
|
|
|
await waitForFile(sendStarted);
|
|
const childPid = Number(await readFile(sendStarted, "utf8"));
|
|
controller.abort();
|
|
const immediate = await observeSettlement(run, 50);
|
|
const envelope = await run;
|
|
|
|
assert.equal(immediate.settled, false, "workflow must wait for the send child to exit");
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(downstreamStarted, false, "cancellation must stop later pipeline stages");
|
|
if (process.platform !== "win32") {
|
|
await waitForFile(sendTerminated, 500);
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 700));
|
|
assert.equal(processIsRunning(childPid), false);
|
|
const invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1, "cancellation must prevent the second send from starting");
|
|
assert.equal(await fileExists(sendCompleted), false, "a cancelled send must not finish");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("cancellation after a Gmail send interrupts a blocked next draft read", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cancel-send-input-"));
|
|
let releaseBlockedRead!: () => void;
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const sendCompleted = join(dir, "send-completed");
|
|
const sendInvocations = join(dir, "send-invocations");
|
|
const controller = new AbortController();
|
|
let markBlockedReadStarted!: () => void;
|
|
const blockedReadStarted = new Promise<void>((resolve) => {
|
|
markBlockedReadStarted = resolve;
|
|
});
|
|
const blockedRead = new Promise<IteratorResult<unknown>>((resolve) => {
|
|
releaseBlockedRead = () => resolve({ done: true, value: undefined });
|
|
});
|
|
const source = {
|
|
name: "draft",
|
|
async run() {
|
|
return {
|
|
output: {
|
|
[Symbol.asyncIterator]() {
|
|
let reads = 0;
|
|
return {
|
|
async next() {
|
|
reads += 1;
|
|
if (reads === 1) {
|
|
return {
|
|
done: false,
|
|
value: {
|
|
to: "first@example.com",
|
|
subject: "First",
|
|
body: "Hello",
|
|
},
|
|
};
|
|
}
|
|
markBlockedReadStarted();
|
|
return blockedRead;
|
|
},
|
|
};
|
|
},
|
|
},
|
|
};
|
|
},
|
|
};
|
|
const run = runToolRequest({
|
|
pipeline: "draft | gog.gmail.send",
|
|
ctx: {
|
|
registry: withCommands(createDefaultRegistry(), source),
|
|
signal: controller.signal,
|
|
env: {
|
|
...process.env,
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_COMPLETED_FILE: sendCompleted,
|
|
MOCK_GOG_SEND_INVOCATIONS_FILE: sendInvocations,
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "0",
|
|
},
|
|
},
|
|
});
|
|
|
|
await waitForFile(sendCompleted);
|
|
await blockedReadStarted;
|
|
controller.abort(new Error("abort while waiting for another draft"));
|
|
const observed = await observeSettlement(run, 100);
|
|
releaseBlockedRead();
|
|
const envelope = observed.settled ? observed.value : await run;
|
|
|
|
assert.equal(observed.settled, true, "cancellation must interrupt the blocked input read");
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(envelope.error?.message, "abort while waiting for another draft");
|
|
const invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1, "cancellation must not start another send");
|
|
} finally {
|
|
releaseBlockedRead?.();
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow pipeline terminates an in-flight send and halts under cancellation", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-cancel-send-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const sendStarted = join(dir, "send-started");
|
|
const sendTerminated = join(dir, "send-terminated");
|
|
const sendCompleted = join(dir, "send-completed");
|
|
const sendInvocations = join(dir, "send-invocations");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{
|
|
id: "draft",
|
|
run: `${JSON.stringify(process.execPath)} -e "process.stdout.write(JSON.stringify({to:'user@example.com',subject:'Hello',body:'World'}))"`,
|
|
},
|
|
{
|
|
id: "send",
|
|
pipeline: "gog.gmail.send | test.nested-side-effect",
|
|
stdin: "$draft.json",
|
|
},
|
|
{ id: "later", pipeline: "test.workflow-side-effect" },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
let nestedStarted = false;
|
|
const nestedSideEffect = {
|
|
name: "test.nested-side-effect",
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
nestedStarted = true;
|
|
const items = [];
|
|
for await (const item of input) items.push(item);
|
|
return { output: streamOf(items) };
|
|
},
|
|
};
|
|
let laterStarted = false;
|
|
const workflowSideEffect = {
|
|
name: "test.workflow-side-effect",
|
|
async run() {
|
|
laterStarted = true;
|
|
return { output: streamOf([{ ok: true }]) };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), nestedSideEffect, workflowSideEffect);
|
|
const controller = new AbortController();
|
|
const run = runToolRequest({
|
|
filePath,
|
|
ctx: {
|
|
cwd: dir,
|
|
registry,
|
|
signal: controller.signal,
|
|
env: {
|
|
...process.env,
|
|
LOBSTER_STATE_DIR: join(dir, "state"),
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_STARTED_FILE: sendStarted,
|
|
MOCK_GOG_SEND_TERMINATED_FILE: sendTerminated,
|
|
MOCK_GOG_SEND_COMPLETED_FILE: sendCompleted,
|
|
MOCK_GOG_SEND_INVOCATIONS_FILE: sendInvocations,
|
|
MOCK_GOG_TERMINATION_DELAY_MS: "1000",
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "5000",
|
|
},
|
|
},
|
|
});
|
|
|
|
await waitForFile(sendStarted);
|
|
const childPid = Number(await readFile(sendStarted, "utf8"));
|
|
controller.abort();
|
|
const immediate = await observeSettlement(run, 50);
|
|
const envelope = await run;
|
|
|
|
assert.equal(immediate.settled, false, "workflow must wait for the send child to exit");
|
|
assertCancellationEnvelope(envelope);
|
|
if (process.platform !== "win32") {
|
|
await waitForFile(sendTerminated, 500);
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 700));
|
|
assert.equal(processIsRunning(childPid), false);
|
|
assert.equal(nestedStarted, false, "cancellation must stop the next nested pipeline stage");
|
|
assert.equal(laterStarted, false, "cancellation must stop later workflow steps");
|
|
const invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1);
|
|
assert.equal(await fileExists(sendCompleted), false, "a cancelled send must not finish");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("parallel timeout kills in-flight sends without retrying", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-parallel-timeout-send-settlement-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const sendStarted = join(dir, "send-started");
|
|
const sendCompleted = join(dir, "send-completed");
|
|
const sendInvocations = join(dir, "send-invocations");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{
|
|
id: "draft",
|
|
run: `${JSON.stringify(process.execPath)} -e "process.stdout.write(JSON.stringify({to:'user@example.com',subject:'Hello',body:'World'}))"`,
|
|
},
|
|
{
|
|
id: "send",
|
|
retry: { max: 2, delay_ms: 0 },
|
|
parallel: {
|
|
timeout_ms: 80,
|
|
branches: [
|
|
{
|
|
id: "mail",
|
|
pipeline: "gog.gmail.send",
|
|
stdin: "$draft.json",
|
|
},
|
|
],
|
|
},
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
|
|
const run = runToolRequest({
|
|
filePath,
|
|
ctx: {
|
|
cwd: dir,
|
|
registry: createDefaultRegistry(),
|
|
env: {
|
|
...process.env,
|
|
LOBSTER_STATE_DIR: join(dir, "state"),
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_STARTED_FILE: sendStarted,
|
|
MOCK_GOG_SEND_COMPLETED_FILE: sendCompleted,
|
|
MOCK_GOG_SEND_INVOCATIONS_FILE: sendInvocations,
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "300",
|
|
},
|
|
},
|
|
});
|
|
|
|
await waitForFile(sendStarted);
|
|
const result = await run;
|
|
assert.equal(result.ok, false);
|
|
assert.match(result.error?.message ?? "", /timed out|timeout|abort|cancel/i);
|
|
assert.equal(
|
|
await fileExists(sendCompleted),
|
|
false,
|
|
"a timeout must kill its in-flight send before retry policy returns",
|
|
);
|
|
const invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(
|
|
invocations.length,
|
|
1,
|
|
"a timed-out send must not retry; Gmail may already have accepted the first message",
|
|
);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("a templated Gmail send pipeline is not retried after timeout", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-templated-send-retry-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const sendStarted = join(dir, "send-started");
|
|
const sendInvocations = join(dir, "send-invocations");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
args: { send_pipeline: { default: "gog.gmail.send" } },
|
|
steps: [
|
|
{
|
|
id: "draft",
|
|
run: `${JSON.stringify(process.execPath)} -e "process.stdout.write(JSON.stringify({to:'user@example.com',subject:'Hello',body:'World'}))"`,
|
|
},
|
|
{
|
|
id: "send",
|
|
pipeline: "${send_pipeline}",
|
|
stdin: "$draft.json",
|
|
timeout_ms: 80,
|
|
retry: { max: 2, delay_ms: 0 },
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
|
|
const run = runToolRequest({
|
|
filePath,
|
|
ctx: {
|
|
cwd: dir,
|
|
registry: createDefaultRegistry(),
|
|
env: {
|
|
...process.env,
|
|
LOBSTER_STATE_DIR: join(dir, "state"),
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_STARTED_FILE: sendStarted,
|
|
MOCK_GOG_SEND_INVOCATIONS_FILE: sendInvocations,
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "300",
|
|
},
|
|
},
|
|
});
|
|
|
|
await waitForFile(sendStarted);
|
|
const result = await run;
|
|
assert.equal(result.ok, false);
|
|
assert.match(result.error?.message ?? "", /timed out|timeout|abort|cancel/i);
|
|
const invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("a parent does not retry a child workflow after the child sends Gmail", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-composed-send-retry-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const childPath = join(dir, "child.lobster");
|
|
const parentPath = join(dir, "parent.lobster");
|
|
const sendStarted = join(dir, "send-started");
|
|
const sendInvocations = join(dir, "send-invocations");
|
|
await writeFile(
|
|
childPath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{
|
|
id: "draft",
|
|
run: `${JSON.stringify(process.execPath)} -e "process.stdout.write(JSON.stringify({to:'user@example.com',subject:'Hello',body:'World'}))"`,
|
|
},
|
|
{ id: "send", pipeline: "gog.gmail.send", stdin: "$draft.json" },
|
|
{ id: "fail", run: `${JSON.stringify(process.execPath)} -e "process.exit(1)"` },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
await writeFile(
|
|
parentPath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{
|
|
id: "child",
|
|
workflow: "./child.lobster",
|
|
retry: { max: 2, delay_ms: 0 },
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
|
|
const result = await runToolRequest({
|
|
filePath: parentPath,
|
|
ctx: {
|
|
cwd: dir,
|
|
registry: createDefaultRegistry(),
|
|
env: {
|
|
...process.env,
|
|
LOBSTER_STATE_DIR: join(dir, "state"),
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_STARTED_FILE: sendStarted,
|
|
MOCK_GOG_SEND_INVOCATIONS_FILE: sendInvocations,
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "50",
|
|
},
|
|
},
|
|
});
|
|
|
|
assert.equal(result.ok, false);
|
|
assert.match(result.error?.message ?? "", /workflow command failed/);
|
|
const invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("approval resume token cannot replay a send after downstream cancellation", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-resume-cancel-replay-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const stateDir = join(dir, "state");
|
|
const sendStarted = join(dir, "send-started");
|
|
const sendCompleted = join(dir, "send-completed");
|
|
const sendInvocations = join(dir, "send-invocations");
|
|
const searchStarted = join(dir, "search-started");
|
|
const searchTerminated = join(dir, "search-terminated");
|
|
const defaultRegistry = createDefaultRegistry();
|
|
const source = {
|
|
name: "draft",
|
|
async run() {
|
|
return {
|
|
output: streamOf([{ to: "user@example.com", subject: "Hello", body: "World" }]),
|
|
};
|
|
},
|
|
};
|
|
const registry = withCommands(defaultRegistry, source);
|
|
const env = {
|
|
...process.env,
|
|
LOBSTER_STATE_DIR: stateDir,
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_STARTED_FILE: sendStarted,
|
|
MOCK_GOG_SEND_COMPLETED_FILE: sendCompleted,
|
|
MOCK_GOG_SEND_INVOCATIONS_FILE: sendInvocations,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: searchStarted,
|
|
MOCK_GOG_SEARCH_TERMINATED_FILE: searchTerminated,
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "100",
|
|
};
|
|
const first = await runToolRequest({
|
|
pipeline:
|
|
"draft | approve --prompt Send? | gog.gmail.send | gog.gmail.search --query newer_than:1d",
|
|
ctx: { registry, env },
|
|
});
|
|
|
|
assert.equal(first.ok, true);
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
assert.ok(first.requiresApproval.approvalId);
|
|
|
|
const controller = new AbortController();
|
|
const resumed = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { registry, signal: controller.signal, env },
|
|
});
|
|
await waitForFile(searchStarted, 3000);
|
|
controller.abort();
|
|
const envelope = await resumed;
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
await waitForFile(sendCompleted, 1000);
|
|
await waitForFile(searchTerminated, 1000);
|
|
let invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1);
|
|
|
|
const replayById = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: {
|
|
registry,
|
|
env: { ...env, MOCK_GOG_COMPLETION_DELAY_MS: "0" },
|
|
},
|
|
});
|
|
assert.equal(replayById.ok, false);
|
|
assert.match(replayById.error?.message ?? "", /not found or expired/);
|
|
|
|
const replayByToken = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: {
|
|
registry,
|
|
env: { ...env, MOCK_GOG_COMPLETION_DELAY_MS: "0" },
|
|
},
|
|
});
|
|
assert.equal(replayByToken.ok, false);
|
|
assert.match(replayByToken.error?.message ?? "", /Pipeline resume state not found/);
|
|
invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1, "retrying an aborted resume token must not resend");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pipeline resume state is consumed when cancellation uses a custom reason", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-resume-custom-abort-"));
|
|
try {
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const source = {
|
|
name: "item",
|
|
async run() {
|
|
return { output: streamOf([{ value: 1 }]) };
|
|
},
|
|
};
|
|
const sideEffect = createCustomAbortSideEffect();
|
|
const registry = withCommands(createDefaultRegistry(), source, sideEffect.command);
|
|
const first = await runToolRequest({
|
|
pipeline: "item | approve --prompt Continue? | test.custom-abort-side-effect",
|
|
ctx: { registry, env },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const controller = new AbortController();
|
|
const resumed = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { registry, signal: controller.signal, env },
|
|
});
|
|
await sideEffect.started;
|
|
controller.abort(new Error("shutdown"));
|
|
const aborted = await resumed;
|
|
assert.equal(aborted.ok, false);
|
|
assert.match(aborted.error?.message ?? "", /shutdown/);
|
|
|
|
const replay = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { registry, env },
|
|
});
|
|
assert.equal(replay.ok, false);
|
|
assert.match(replay.error?.message ?? "", /Pipeline resume state not found/);
|
|
assert.equal(
|
|
sideEffect.invocations,
|
|
1,
|
|
"a custom abort reason must not leave the side effect replayable",
|
|
);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow resume state is consumed when cancellation uses a custom reason", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-custom-abort-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const sideEffect = createCustomAbortSideEffect();
|
|
const registry = withCommands(createDefaultRegistry(), sideEffect.command);
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "confirm", approval: "Continue?" },
|
|
{
|
|
id: "effect",
|
|
pipeline: "test.custom-abort-side-effect",
|
|
when: "$confirm.approved",
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, registry, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const controller = new AbortController();
|
|
const resumed = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, registry, signal: controller.signal, env },
|
|
});
|
|
await sideEffect.started;
|
|
controller.abort(new Error("shutdown"));
|
|
const aborted = await resumed;
|
|
assert.equal(aborted.ok, false);
|
|
assert.match(aborted.error?.message ?? "", /shutdown/);
|
|
|
|
const replay = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, registry, env },
|
|
});
|
|
assert.equal(replay.ok, false);
|
|
assert.match(replay.error?.message ?? "", /Workflow resume state not found/);
|
|
assert.equal(
|
|
sideEffect.invocations,
|
|
1,
|
|
"a custom abort reason must not leave the side effect replayable",
|
|
);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("aliased workflow resume state is consumed after cancellation starts execution", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-alias-custom-abort-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const sideEffect = createCustomAbortSideEffect();
|
|
const registry = withCommands(createDefaultRegistry(), sideEffect.command);
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "confirm", approval: "Continue?" },
|
|
{
|
|
id: "effect",
|
|
pipeline: "test.custom-abort-side-effect",
|
|
when: "$confirm.approved",
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, registry, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
assert.equal(payload.kind, "workflow-file");
|
|
assert.ok(payload.stateKey?.startsWith("workflow_resume_"));
|
|
const aliasedStateKey = payload.stateKey.replace("workflow_resume_", "workflow-resume_");
|
|
const aliasedToken = encodeToken({ ...payload, stateKey: aliasedStateKey });
|
|
|
|
const controller = new AbortController();
|
|
const resumed = resumeToolRequest({
|
|
token: aliasedToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, registry, signal: controller.signal, env },
|
|
});
|
|
await sideEffect.started;
|
|
controller.abort(new Error("abort aliased workflow resume"));
|
|
const aborted = await resumed;
|
|
assertCancellationEnvelope(aborted);
|
|
|
|
const replayById = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, registry, env },
|
|
});
|
|
assert.equal(replayById.ok, false);
|
|
assert.match(replayById.error?.message ?? "", /not found or expired/);
|
|
|
|
const replayByToken = await resumeToolRequest({
|
|
token: aliasedToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, registry, env },
|
|
});
|
|
assert.equal(replayByToken.ok, false);
|
|
assert.match(replayByToken.error?.message ?? "", /Workflow resume state not found/);
|
|
assert.equal(sideEffect.invocations, 1, "the canonical workflow state must not be replayable");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("explicit cancellation consumes an aliased workflow resume state", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-alias-explicit-cancel-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [{ id: "confirm", approval: "Continue?" }],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
assert.equal(payload.kind, "workflow-file");
|
|
assert.ok(payload.stateKey?.startsWith("workflow_resume_"));
|
|
const aliasedToken = encodeToken({
|
|
...payload,
|
|
stateKey: payload.stateKey.replace("workflow_resume_", "workflow-resume_"),
|
|
});
|
|
const cancelled = await resumeToolRequest({
|
|
token: aliasedToken,
|
|
cancel: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(cancelled.ok, true);
|
|
assert.equal(cancelled.status, "cancelled");
|
|
|
|
const replayById = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(replayById.ok, false);
|
|
assert.match(replayById.error?.message ?? "", /not found or expired/);
|
|
const replayByToken = await resumeToolRequest({
|
|
token: aliasedToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(replayByToken.ok, false);
|
|
assert.match(replayByToken.error?.message ?? "", /Workflow resume state not found/);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("explicit cancellation stops waiting for a state lock without orphaning its approval ID", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-explicit-cancel-lock-abort-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const first = await runToolRequest({
|
|
pipeline: "approve --prompt Continue?",
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const lockPath = `${keyToPath(stateDir, payload.stateKey)}.lock`;
|
|
const approvalIndexPath = join(stateDir, `approval_${first.requiresApproval.approvalId}.json`);
|
|
await fsp.mkdir(lockPath);
|
|
await fsp.writeFile(join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
|
|
|
const controller = new AbortController();
|
|
const pending = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
cancel: true,
|
|
ctx: { cwd: dir, env, signal: controller.signal },
|
|
});
|
|
const completion = pending.then(
|
|
() => ({ kind: "success" as const }),
|
|
(error) => ({ kind: "error" as const, error }),
|
|
);
|
|
await new Promise((resolve) => setImmediate(resolve));
|
|
controller.abort(new Error("explicit cancellation stopped while waiting for a state lock"));
|
|
const early = await Promise.race([
|
|
completion,
|
|
new Promise<{ kind: "timeout" }>((resolve) =>
|
|
setTimeout(() => resolve({ kind: "timeout" }), 75),
|
|
),
|
|
]);
|
|
if (early.kind === "timeout") await rm(lockPath, { recursive: true, force: true });
|
|
const settled = early.kind === "timeout" ? await completion : early;
|
|
|
|
assert.notEqual(
|
|
early.kind,
|
|
"timeout",
|
|
"explicit cancellation must not remain blocked on a state lock",
|
|
);
|
|
assert.equal(settled.kind, "error");
|
|
if (settled.kind === "error") {
|
|
assert.match(
|
|
settled.error?.message ?? "",
|
|
/explicit cancellation stopped while waiting for a state lock/,
|
|
);
|
|
}
|
|
assert.equal(
|
|
await fileExists(approvalIndexPath),
|
|
true,
|
|
"the approval ID must remain usable after cancellation",
|
|
);
|
|
assert.equal(await fileExists(keyToPath(stateDir, payload.stateKey)), true);
|
|
await rm(lockPath, { recursive: true, force: true });
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow explicit cancellation keeps canonical state when its alternate spelling is locked", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-alternate-cancel-lock-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const filePath = join(dir, "workflow.lobster");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({ steps: [{ id: "confirm", approval: "Continue?" }] }),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
assert.ok(payload.stateKey?.startsWith("workflow_resume_"));
|
|
const alternateStateKey = payload.stateKey.replace("workflow_resume_", "workflow-resume_");
|
|
const alternateLock = `${keyToPath(stateDir, alternateStateKey)}.lock`;
|
|
const approvalIndexPath = join(stateDir, `approval_${first.requiresApproval.approvalId}.json`);
|
|
await fsp.mkdir(alternateLock);
|
|
await fsp.writeFile(join(alternateLock, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
|
|
|
const controller = new AbortController();
|
|
const pending = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
cancel: true,
|
|
ctx: { cwd: dir, env, signal: controller.signal },
|
|
}).then(
|
|
(value) => ({ kind: "value" as const, value }),
|
|
(error) => ({ kind: "error" as const, error }),
|
|
);
|
|
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
controller.abort(new Error("alternate cleanup lock interrupted cancellation"));
|
|
const settled = await observeSettlement(pending, 500);
|
|
assert.equal(settled.settled, true);
|
|
if (settled.settled) {
|
|
assert.equal(settled.value.kind, "error");
|
|
if (settled.value.kind === "error") {
|
|
assert.match(settled.value.error?.message ?? "", /alternate cleanup lock interrupted/);
|
|
}
|
|
}
|
|
assert.equal(await fileExists(keyToPath(stateDir, payload.stateKey)), true);
|
|
assert.equal(await fileExists(approvalIndexPath), true);
|
|
await rm(alternateLock, { recursive: true, force: true });
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("a failed post-effect resume cleanup leaves a non-replayable pipeline tombstone", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-consumed-pipeline-tombstone-"));
|
|
const originalUnlink = fsp.unlink;
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const effect = createCountingSideEffect("test.tombstone-effect");
|
|
const registry = withCommands(createDefaultRegistry(), effect.command);
|
|
const first = await runToolRequest({
|
|
pipeline: "approve --prompt Continue? | test.tombstone-effect",
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const statePath = keyToPath(stateDir, payload.stateKey);
|
|
Object.defineProperty(fsp, "unlink", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(filePath: Parameters<typeof fsp.unlink>[0]) {
|
|
if (String(filePath) === statePath) {
|
|
throw Object.assign(new Error("injected resume unlink failure"), { code: "EIO" });
|
|
}
|
|
return originalUnlink(filePath);
|
|
},
|
|
});
|
|
|
|
const resumed = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
assert.equal(resumed.ok, false);
|
|
assert.match(resumed.error?.message ?? "", /injected resume unlink failure/);
|
|
assert.equal(effect.invocations, 1);
|
|
|
|
const replay = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
assert.equal(replay.ok, false);
|
|
assert.match(replay.error?.message ?? "", /Pipeline resume state not found/);
|
|
assert.equal(effect.invocations, 1, "a tombstoned token must not repeat its side effect");
|
|
} finally {
|
|
Object.defineProperty(fsp, "unlink", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalUnlink,
|
|
});
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
async function assertApprovalResumeIsExclusive({
|
|
workflow,
|
|
resumeBy,
|
|
}: {
|
|
workflow: boolean;
|
|
resumeBy: "token" | "approvalId";
|
|
}) {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-exclusive-resume-"));
|
|
const originalRename = fsp.rename;
|
|
let releaseFirstRename!: () => void;
|
|
const firstRenameReached = new Promise<void>((resolve) => {
|
|
releaseFirstRename = resolve;
|
|
});
|
|
let renamePaused = false;
|
|
let releaseFirstEffect!: () => void;
|
|
const firstEffectMayFinish = new Promise<void>((resolve) => {
|
|
releaseFirstEffect = resolve;
|
|
});
|
|
let invocations = 0;
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const effect = {
|
|
name: "test.exclusive-resume-effect",
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
invocations += 1;
|
|
const items = [];
|
|
for await (const item of input) items.push(item);
|
|
if (invocations === 1) await firstEffectMayFinish;
|
|
return { output: streamOf(items.length ? items : [{ ok: true }]) };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), effect);
|
|
let first;
|
|
if (workflow) {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "approve", approval: "Continue?" },
|
|
{ id: "effect", pipeline: "test.exclusive-resume-effect" },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
first = await runToolRequest({ filePath, ctx: { cwd: dir, env, registry } });
|
|
} else {
|
|
first = await runToolRequest({
|
|
pipeline: "approve --prompt Continue? | test.exclusive-resume-effect",
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
}
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken!);
|
|
const statePath = keyToPath(stateDir, payload.stateKey);
|
|
|
|
Object.defineProperty(fsp, "rename", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(from: Parameters<typeof fsp.rename>[0], to: Parameters<typeof fsp.rename>[1]) {
|
|
if (String(to) === statePath && !renamePaused) {
|
|
renamePaused = true;
|
|
await firstRenameReached;
|
|
}
|
|
return originalRename(from, to);
|
|
},
|
|
});
|
|
|
|
const resumeArgs =
|
|
resumeBy === "token"
|
|
? { token: first.requiresApproval.resumeToken! }
|
|
: { approvalId: first.requiresApproval.approvalId! };
|
|
const firstResume = resumeToolRequest({
|
|
...resumeArgs,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
while (!renamePaused) await new Promise((resolve) => setImmediate(resolve));
|
|
const secondResume = resumeToolRequest({
|
|
...resumeArgs,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
releaseFirstRename();
|
|
// The first effect is held open. The second request has enough time to
|
|
// reach the guarded claim, but cannot be allowed to observe cleanup as a
|
|
// missing-state retry opportunity.
|
|
await new Promise((resolve) => setTimeout(resolve, 30));
|
|
releaseFirstEffect();
|
|
const outcomes = await Promise.all([firstResume, secondResume]);
|
|
|
|
assert.equal(invocations, 1, "one approval may dispatch exactly one effect");
|
|
assert.equal(outcomes.filter((outcome) => outcome.ok).length, 1);
|
|
assert.equal(outcomes.filter((outcome) => !outcome.ok).length, 1);
|
|
assert.match(
|
|
outcomes.find((outcome) => !outcome.ok)?.error?.message ?? "",
|
|
/resume state not found/i,
|
|
);
|
|
} finally {
|
|
Object.defineProperty(fsp, "rename", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalRename,
|
|
});
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
test("pipeline approval token and short ID each dispatch an effect at most once", async () => {
|
|
for (const resumeBy of ["token", "approvalId"] as const) {
|
|
await assertApprovalResumeIsExclusive({ workflow: false, resumeBy });
|
|
}
|
|
});
|
|
|
|
test("workflow approval token and short ID each dispatch an effect at most once", async () => {
|
|
for (const resumeBy of ["token", "approvalId"] as const) {
|
|
await assertApprovalResumeIsExclusive({ workflow: true, resumeBy });
|
|
}
|
|
});
|
|
|
|
async function assertApprovalGateHandoffIsExclusive({
|
|
workflow,
|
|
resumeBy,
|
|
}: {
|
|
workflow: boolean;
|
|
resumeBy: "token" | "approvalId";
|
|
}) {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-exclusive-gate-handoff-"));
|
|
let invocations = 0;
|
|
try {
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const effect = {
|
|
name: "test.exclusive-gate-handoff-effect",
|
|
async run() {
|
|
invocations += 1;
|
|
return { output: streamOf([{ ok: true }]) };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), effect);
|
|
let first;
|
|
if (workflow) {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "first", approval: "First?" },
|
|
{ id: "second", pipeline: "ask --prompt Second?" },
|
|
{ id: "effect", pipeline: "test.exclusive-gate-handoff-effect" },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
first = await runToolRequest({ filePath, ctx: { cwd: dir, env, registry } });
|
|
} else {
|
|
first = await runToolRequest({
|
|
pipeline:
|
|
"approve --prompt First? | ask --prompt Second? | test.exclusive-gate-handoff-effect",
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
}
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
|
|
const resumeArgs =
|
|
resumeBy === "token"
|
|
? { token: first.requiresApproval.resumeToken! }
|
|
: { approvalId: first.requiresApproval.approvalId! };
|
|
const outcomes = await Promise.all([
|
|
resumeToolRequest({ ...resumeArgs, approved: true, ctx: { cwd: dir, env, registry } }),
|
|
resumeToolRequest({ ...resumeArgs, approved: true, ctx: { cwd: dir, env, registry } }),
|
|
]);
|
|
|
|
const successor = outcomes.find((outcome) => outcome.ok && outcome.status === "needs_input");
|
|
assert.ok(successor?.requiresInput?.resumeToken, "exactly one successor gate must be returned");
|
|
assert.equal(
|
|
outcomes.filter((outcome) => outcome.ok && outcome.status === "needs_input").length,
|
|
1,
|
|
);
|
|
assert.equal(outcomes.filter((outcome) => !outcome.ok).length, 1);
|
|
assert.match(
|
|
outcomes.find((outcome) => !outcome.ok)?.error?.message ?? "",
|
|
/resume state not found/i,
|
|
);
|
|
|
|
const completed = await resumeToolRequest({
|
|
token: successor!.requiresInput!.resumeToken!,
|
|
response: { decision: "approve" },
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
assert.equal(completed.ok, true);
|
|
assert.equal(completed.status, "ok");
|
|
assert.equal(invocations, 1, "only the winning successor may reach the effect");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
test("pipeline approval token and short ID cannot fork a next input gate", async () => {
|
|
for (const resumeBy of ["token", "approvalId"] as const) {
|
|
await assertApprovalGateHandoffIsExclusive({ workflow: false, resumeBy });
|
|
}
|
|
});
|
|
|
|
test("workflow approval token and short ID cannot fork a next input gate", async () => {
|
|
for (const resumeBy of ["token", "approvalId"] as const) {
|
|
await assertApprovalGateHandoffIsExclusive({ workflow: true, resumeBy });
|
|
}
|
|
});
|
|
|
|
async function assertCancellationCannotRetireClaimedSuccessor({
|
|
workflow,
|
|
rejectApproval,
|
|
}: {
|
|
workflow: boolean;
|
|
rejectApproval: boolean;
|
|
}) {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cancel-claimed-successor-"));
|
|
const originalRename = fsp.rename;
|
|
let releaseClaim!: () => void;
|
|
const claimReleased = new Promise<void>((resolve) => {
|
|
releaseClaim = resolve;
|
|
});
|
|
let claimPublished!: () => void;
|
|
const claimReached = new Promise<void>((resolve) => {
|
|
claimPublished = resolve;
|
|
});
|
|
let claimPaused = false;
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
let first: any;
|
|
if (workflow) {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "approve", approval: "Continue?" },
|
|
{ id: "next-input", pipeline: "ask --prompt Next?" },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
} else {
|
|
first = await runToolRequest({
|
|
pipeline: "approve --prompt Continue? | ask --prompt Next?",
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
}
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken!);
|
|
const predecessorPath = keyToPath(stateDir, payload.stateKey);
|
|
|
|
Object.defineProperty(fsp, "rename", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(from: Parameters<typeof fsp.rename>[0], to: Parameters<typeof fsp.rename>[1]) {
|
|
const result = await originalRename(from, to);
|
|
if (String(to) === predecessorPath && !claimPaused) {
|
|
claimPaused = true;
|
|
claimPublished();
|
|
await claimReleased;
|
|
}
|
|
return result;
|
|
},
|
|
});
|
|
|
|
const resumed = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken!,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
await claimReached;
|
|
const cancelled = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken!,
|
|
...(rejectApproval ? { approved: false } : { cancel: true }),
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
releaseClaim();
|
|
|
|
const [resumedResult, cancelledResult] = await Promise.all([resumed, cancelled]);
|
|
assert.equal(resumedResult.ok, true);
|
|
assert.equal(resumedResult.status, "needs_input");
|
|
assert.ok(resumedResult.requiresInput?.resumeToken);
|
|
assert.equal(
|
|
cancelledResult.ok,
|
|
false,
|
|
"cancellation must reject a predecessor already claimed for a successor",
|
|
);
|
|
assert.match(cancelledResult.error?.message ?? "", /resume state not found/i);
|
|
const successor = decodeResumeToken(resumedResult.requiresInput.resumeToken!);
|
|
assert.notEqual(
|
|
await fsp.readFile(keyToPath(stateDir, successor.stateKey), "utf8"),
|
|
"",
|
|
"the returned successor must remain live",
|
|
);
|
|
} finally {
|
|
Object.defineProperty(fsp, "rename", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalRename,
|
|
});
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
test("cancellation and rejection cannot retire a claimed pipeline or workflow successor", async () => {
|
|
for (const workflow of [false, true]) {
|
|
for (const rejectApproval of [false, true]) {
|
|
await assertCancellationCannotRetireClaimedSuccessor({ workflow, rejectApproval });
|
|
}
|
|
}
|
|
});
|
|
|
|
test("a cancellation immediately after pipeline resume consumption restores the unstarted approval", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-consume-rename-abort-"));
|
|
const originalRename = fsp.rename;
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const effect = createCountingSideEffect("test.consume-rename-effect");
|
|
const registry = withCommands(createDefaultRegistry(), effect.command);
|
|
const first = await runToolRequest({
|
|
pipeline: "approve --prompt Continue? | test.consume-rename-effect",
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken!);
|
|
const statePath = keyToPath(stateDir, payload.stateKey);
|
|
const controller = new AbortController();
|
|
let abortedAtConsumption = false;
|
|
Object.defineProperty(fsp, "rename", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(from: Parameters<typeof fsp.rename>[0], to: Parameters<typeof fsp.rename>[1]) {
|
|
await originalRename(from, to);
|
|
if (String(to) === statePath && !abortedAtConsumption) {
|
|
abortedAtConsumption = true;
|
|
controller.abort(new Error("abort after consumed resume rename"));
|
|
}
|
|
},
|
|
});
|
|
|
|
const cancelled = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, registry, signal: controller.signal },
|
|
});
|
|
assertCancellationEnvelope(cancelled);
|
|
assert.equal(effect.invocations, 0);
|
|
assert.equal(abortedAtConsumption, true);
|
|
|
|
const retried = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
assert.equal(retried.ok, true);
|
|
assert.equal(effect.invocations, 1);
|
|
} finally {
|
|
Object.defineProperty(fsp, "rename", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalRename,
|
|
});
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("resumed pipeline and workflow approvals retain their short ID until the started invocation settles", async () => {
|
|
for (const workflow of [false, true]) {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-approval-index-dispatch-boundary-"));
|
|
let releaseEffect!: () => void;
|
|
const effectReleased = new Promise<void>((resolve) => {
|
|
releaseEffect = resolve;
|
|
});
|
|
let effectStarted!: () => void;
|
|
const effectEntered = new Promise<void>((resolve) => {
|
|
effectStarted = resolve;
|
|
});
|
|
const effect = {
|
|
name: "test.hold-before-terminal-cleanup",
|
|
async run() {
|
|
effectStarted();
|
|
await effectReleased;
|
|
return { output: streamOf([{ ok: true }]) };
|
|
},
|
|
};
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const registry = withCommands(createDefaultRegistry(), effect);
|
|
let first;
|
|
let filePath: string | undefined;
|
|
if (workflow) {
|
|
filePath = join(dir, "workflow.lobster");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "confirm", approval: "Continue?" },
|
|
{ id: "effect", pipeline: "test.hold-before-terminal-cleanup" },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
first = await runToolRequest({ filePath, ctx: { cwd: dir, env, registry } });
|
|
} else {
|
|
first = await runToolRequest({
|
|
pipeline: "approve --prompt Continue? | test.hold-before-terminal-cleanup",
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
}
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
const approvalIndexPath = join(
|
|
stateDir,
|
|
`approval_${first.requiresApproval.approvalId}.json`,
|
|
);
|
|
|
|
const resumed = resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
await effectEntered;
|
|
assert.equal(
|
|
await fileExists(approvalIndexPath),
|
|
true,
|
|
"the short approval ID must survive until the resumed invocation reaches a terminal outcome",
|
|
);
|
|
releaseEffect();
|
|
const settled = await resumed;
|
|
assert.equal(settled.status, "ok");
|
|
assert.equal(await fileExists(approvalIndexPath), false);
|
|
} finally {
|
|
releaseEffect();
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
});
|
|
|
|
test("approval rejection stops waiting for a state lock without orphaning its approval ID", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-approval-rejection-lock-abort-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const first = await runToolRequest({
|
|
pipeline: "approve --prompt Continue?",
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const lockPath = `${keyToPath(stateDir, payload.stateKey)}.lock`;
|
|
const approvalIndexPath = join(stateDir, `approval_${first.requiresApproval.approvalId}.json`);
|
|
await fsp.mkdir(lockPath);
|
|
await fsp.writeFile(join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
|
|
|
const controller = new AbortController();
|
|
const pending = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: false,
|
|
ctx: { cwd: dir, env, signal: controller.signal },
|
|
});
|
|
const completion = pending.then(
|
|
() => ({ kind: "success" as const }),
|
|
(error) => ({ kind: "error" as const, error }),
|
|
);
|
|
await new Promise((resolve) => setImmediate(resolve));
|
|
controller.abort(new Error("approval rejection stopped while waiting for a state lock"));
|
|
const early = await Promise.race([
|
|
completion,
|
|
new Promise<{ kind: "timeout" }>((resolve) =>
|
|
setTimeout(() => resolve({ kind: "timeout" }), 75),
|
|
),
|
|
]);
|
|
if (early.kind === "timeout") await rm(lockPath, { recursive: true, force: true });
|
|
const settled = early.kind === "timeout" ? await completion : early;
|
|
|
|
assert.notEqual(
|
|
early.kind,
|
|
"timeout",
|
|
"approval rejection must not remain blocked on a state lock",
|
|
);
|
|
assert.equal(settled.kind, "error");
|
|
if (settled.kind === "error") {
|
|
assert.match(
|
|
settled.error?.message ?? "",
|
|
/approval rejection stopped while waiting for a state lock/,
|
|
);
|
|
}
|
|
assert.equal(
|
|
await fileExists(approvalIndexPath),
|
|
true,
|
|
"the approval ID must remain usable after cancelled rejection cleanup",
|
|
);
|
|
assert.equal(await fileExists(keyToPath(stateDir, payload.stateKey)), true);
|
|
await rm(lockPath, { recursive: true, force: true });
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow approval rejection keeps its approval ID when state cleanup is cancelled", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-approval-rejection-lock-abort-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({ steps: [{ id: "confirm", approval: "Continue?" }] }),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const lockPath = `${keyToPath(stateDir, payload.stateKey)}.lock`;
|
|
const approvalIndexPath = join(stateDir, `approval_${first.requiresApproval.approvalId}.json`);
|
|
await fsp.mkdir(lockPath);
|
|
await fsp.writeFile(join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
|
|
|
const controller = new AbortController();
|
|
const pending = resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: false,
|
|
ctx: { cwd: dir, env, signal: controller.signal },
|
|
});
|
|
await new Promise((resolve) => setImmediate(resolve));
|
|
controller.abort(new Error("workflow rejection stopped while waiting for a state lock"));
|
|
const result = await observeSettlement(pending, 500);
|
|
assert.equal(result.settled, true, "workflow rejection must stop waiting for the state lock");
|
|
if (result.settled) {
|
|
assert.equal(result.value.ok, false);
|
|
assert.match(
|
|
result.value.error?.message ?? "",
|
|
/workflow rejection stopped while waiting for a state lock/,
|
|
);
|
|
}
|
|
assert.equal(await fileExists(approvalIndexPath), true);
|
|
assert.equal(await fileExists(keyToPath(stateDir, payload.stateKey)), true);
|
|
await rm(lockPath, { recursive: true, force: true });
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("terminal pipeline resume cleanup stops waiting for a state lock", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-terminal-pipeline-lock-abort-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const first = await runToolRequest({
|
|
pipeline: "approve --prompt Continue?",
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const lockPath = `${keyToPath(stateDir, payload.stateKey)}.lock`;
|
|
const approvalIndexPath = join(stateDir, `approval_${first.requiresApproval.approvalId}.json`);
|
|
await fsp.mkdir(lockPath);
|
|
await fsp.writeFile(join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
|
|
|
const controller = new AbortController();
|
|
const pending = resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, signal: controller.signal },
|
|
});
|
|
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
controller.abort(new Error("terminal pipeline cleanup stopped while waiting for a state lock"));
|
|
const early = await observeSettlement(pending, 75);
|
|
if (!early.settled) await rm(lockPath, { recursive: true, force: true });
|
|
const settled = early.settled ? early : await observeSettlement(pending, 75);
|
|
|
|
assert.equal(settled.settled, true, "terminal cleanup must not remain blocked on a state lock");
|
|
if (settled.settled) {
|
|
assert.equal(settled.value.ok, false);
|
|
assert.match(
|
|
settled.value.error?.message ?? "",
|
|
/terminal pipeline cleanup stopped while waiting for a state lock/,
|
|
);
|
|
}
|
|
assert.equal(await fileExists(approvalIndexPath), true);
|
|
assert.equal(await fileExists(keyToPath(stateDir, payload.stateKey)), true);
|
|
await rm(lockPath, { recursive: true, force: true });
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(retried.status, "ok");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("pipeline gate replacement stops waiting for a state lock", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-pipeline-gate-lock-abort-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const first = await runToolRequest({
|
|
pipeline: "approve --prompt First? | approve --prompt Second?",
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const lockPath = `${keyToPath(stateDir, payload.stateKey)}.lock`;
|
|
const approvalIndexPath = join(stateDir, `approval_${first.requiresApproval.approvalId}.json`);
|
|
await fsp.mkdir(lockPath);
|
|
await fsp.writeFile(join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
|
|
|
const controller = new AbortController();
|
|
const originalMkdir = fsp.mkdir;
|
|
let signalLockAttempt: ((value: boolean) => void) | undefined;
|
|
const lockAttempted = new Promise<boolean>((resolve) => {
|
|
signalLockAttempt = resolve;
|
|
});
|
|
let sawLockAttempt = false;
|
|
Object.defineProperty(fsp, "mkdir", {
|
|
configurable: true,
|
|
writable: true,
|
|
async value(
|
|
filePath: Parameters<typeof fsp.mkdir>[0],
|
|
options?: Parameters<typeof fsp.mkdir>[1],
|
|
) {
|
|
if (!sawLockAttempt && String(filePath) === lockPath) {
|
|
sawLockAttempt = true;
|
|
signalLockAttempt?.(true);
|
|
}
|
|
return originalMkdir(filePath, options);
|
|
},
|
|
});
|
|
let settled;
|
|
try {
|
|
const pending = resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, signal: controller.signal },
|
|
});
|
|
const reachedLock = await Promise.race([
|
|
lockAttempted,
|
|
new Promise<boolean>((resolve) => setTimeout(() => resolve(false), 500)),
|
|
]);
|
|
assert.equal(reachedLock, true, "resume must reach the held state lock");
|
|
controller.abort(
|
|
new Error("pipeline gate replacement stopped while waiting for a state lock"),
|
|
);
|
|
const early = await observeSettlement(pending, 75);
|
|
if (!early.settled) await rm(lockPath, { recursive: true, force: true });
|
|
settled = early.settled ? early : await observeSettlement(pending, 75);
|
|
} finally {
|
|
Object.defineProperty(fsp, "mkdir", {
|
|
configurable: true,
|
|
writable: true,
|
|
value: originalMkdir,
|
|
});
|
|
}
|
|
|
|
assert.equal(
|
|
settled?.settled,
|
|
true,
|
|
"gate replacement must not remain blocked on a state lock",
|
|
);
|
|
if (settled?.settled) {
|
|
assert.equal(settled.value.ok, false);
|
|
assert.match(
|
|
settled.value.error?.message ?? "",
|
|
/pipeline gate replacement stopped while waiting for a state lock/,
|
|
);
|
|
}
|
|
assert.equal(await fileExists(approvalIndexPath), true);
|
|
assert.equal(await fileExists(keyToPath(stateDir, payload.stateKey)), true);
|
|
await rm(lockPath, { recursive: true, force: true });
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(retried.status, "needs_approval");
|
|
assert.equal(retried.requiresApproval?.prompt, "Second?");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("terminal workflow resume cleanup stops waiting for a state lock", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-terminal-workflow-lock-abort-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const filePath = join(dir, "workflow.lobster");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({ steps: [{ id: "approve", approval: "Continue?" }] }),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const lockPath = `${keyToPath(stateDir, payload.stateKey)}.lock`;
|
|
const approvalIndexPath = join(stateDir, `approval_${first.requiresApproval.approvalId}.json`);
|
|
await fsp.mkdir(lockPath);
|
|
await fsp.writeFile(join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
|
|
|
const controller = new AbortController();
|
|
const pending = resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, signal: controller.signal },
|
|
});
|
|
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
controller.abort(new Error("terminal workflow cleanup stopped while waiting for a state lock"));
|
|
const early = await observeSettlement(pending, 75);
|
|
if (!early.settled) await rm(lockPath, { recursive: true, force: true });
|
|
const settled = early.settled ? early : await observeSettlement(pending, 75);
|
|
assert.equal(
|
|
settled.settled,
|
|
true,
|
|
"terminal workflow cleanup must not remain blocked on a state lock",
|
|
);
|
|
if (settled.settled) {
|
|
assert.equal(settled.value.ok, false);
|
|
assert.match(
|
|
settled.value.error?.message ?? "",
|
|
/terminal workflow cleanup stopped while waiting for a state lock/,
|
|
);
|
|
}
|
|
assert.equal(await fileExists(approvalIndexPath), true);
|
|
assert.equal(await fileExists(keyToPath(stateDir, payload.stateKey)), true);
|
|
await rm(lockPath, { recursive: true, force: true });
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(retried.status, "ok");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("started pipeline and workflow cleanup never retries an interrupted state lock without its signal", async () => {
|
|
for (const workflow of [false, true]) {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-started-cleanup-lock-abort-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
let lockPath = "";
|
|
let effectFinished!: () => void;
|
|
const effectFinishedPromise = new Promise<void>((resolve) => {
|
|
effectFinished = resolve;
|
|
});
|
|
let invocations = 0;
|
|
const effect = {
|
|
name: "test.started-cleanup-effect",
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
for await (const _item of input) {
|
|
// Drain the approval output before returning the effect result.
|
|
}
|
|
invocations += 1;
|
|
await fsp.mkdir(lockPath);
|
|
await fsp.writeFile(join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
|
effectFinished();
|
|
return { output: streamOf([{ ok: true }]) };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), effect);
|
|
const filePath = join(dir, "workflow.lobster");
|
|
if (workflow) {
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "approve", approval: "Continue?" },
|
|
{ id: "effect", pipeline: "test.started-cleanup-effect" },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
}
|
|
const first = await (workflow
|
|
? runToolRequest({ filePath, ctx: { cwd: dir, env, registry } })
|
|
: runToolRequest({
|
|
pipeline: "approve --prompt Continue? | test.started-cleanup-effect",
|
|
ctx: { cwd: dir, env, registry },
|
|
}));
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken!);
|
|
lockPath = `${keyToPath(stateDir, payload.stateKey)}.lock`;
|
|
const approvalIndexPath = join(
|
|
stateDir,
|
|
`approval_${first.requiresApproval.approvalId}.json`,
|
|
);
|
|
|
|
const controller = new AbortController();
|
|
const pending = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, registry, signal: controller.signal },
|
|
});
|
|
await effectFinishedPromise;
|
|
controller.abort(new Error("started cleanup stopped while waiting for a state lock"));
|
|
const settled = await observeSettlement(pending, 500);
|
|
|
|
assert.equal(
|
|
settled.settled,
|
|
true,
|
|
`${workflow ? "workflow" : "pipeline"} cleanup must not re-enter a held lock after cancellation`,
|
|
);
|
|
if (settled.settled) {
|
|
assert.equal(settled.value.ok, false);
|
|
assert.match(
|
|
settled.value.error?.message ?? "",
|
|
/started cleanup stopped while waiting for a state lock/,
|
|
);
|
|
}
|
|
assert.equal(invocations, 1);
|
|
assert.equal(
|
|
await fileExists(approvalIndexPath),
|
|
false,
|
|
"a started effect must retire its approval ID even when tombstone cleanup times out",
|
|
);
|
|
await rm(lockPath, { recursive: true, force: true });
|
|
|
|
const replay = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, registry },
|
|
});
|
|
assert.equal(replay.ok, false);
|
|
assert.match(replay.error?.message ?? "", /resume state not found/i);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
});
|
|
|
|
test("workflow gate replacement stops waiting for a state lock", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-gate-lock-abort-"));
|
|
try {
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
const filePath = join(dir, "workflow.lobster");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "first", approval: "First?" },
|
|
{ id: "second", approval: "Second?" },
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
const lockPath = `${keyToPath(stateDir, payload.stateKey)}.lock`;
|
|
const approvalIndexPath = join(stateDir, `approval_${first.requiresApproval.approvalId}.json`);
|
|
await fsp.mkdir(lockPath);
|
|
await fsp.writeFile(join(lockPath, "owner"), `${process.pid}::live-writer\n`, "utf8");
|
|
|
|
const controller = new AbortController();
|
|
const pending = resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env, signal: controller.signal },
|
|
});
|
|
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
controller.abort(new Error("workflow gate replacement stopped while waiting for a state lock"));
|
|
const early = await observeSettlement(pending, 75);
|
|
if (!early.settled) await rm(lockPath, { recursive: true, force: true });
|
|
const settled = early.settled ? early : await observeSettlement(pending, 75);
|
|
assert.equal(settled.settled, true, "gate replacement must not remain blocked on a state lock");
|
|
if (settled.settled) {
|
|
assert.equal(settled.value.ok, false);
|
|
assert.match(
|
|
settled.value.error?.message ?? "",
|
|
/workflow gate replacement stopped while waiting for a state lock/,
|
|
);
|
|
}
|
|
assert.equal(await fileExists(approvalIndexPath), true);
|
|
assert.equal(await fileExists(keyToPath(stateDir, payload.stateKey)), true);
|
|
await rm(lockPath, { recursive: true, force: true });
|
|
|
|
const retried = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(retried.status, "needs_approval");
|
|
assert.equal(retried.requiresApproval?.prompt, "Second?");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("lazy output iterator acquisition failure closes the active stage", async () => {
|
|
let inputClosed = false;
|
|
const input = {
|
|
[Symbol.asyncIterator]() {
|
|
return {
|
|
async next() {
|
|
return { done: false, value: { item: 1 } };
|
|
},
|
|
async return() {
|
|
inputClosed = true;
|
|
return { done: true, value: undefined };
|
|
},
|
|
};
|
|
},
|
|
};
|
|
const command = {
|
|
name: "test.throw-on-iterator-acquisition",
|
|
async run({ input: stageInput }: { input: AsyncIterable<unknown> }) {
|
|
await stageInput[Symbol.asyncIterator]().next();
|
|
return {
|
|
output: {
|
|
[Symbol.asyncIterator]() {
|
|
throw new Error("iterator acquisition failed");
|
|
},
|
|
},
|
|
};
|
|
},
|
|
};
|
|
|
|
await assert.rejects(
|
|
runPipeline({
|
|
pipeline: [{ name: command.name, args: {} }],
|
|
registry: withCommands(createDefaultRegistry(), command),
|
|
stdin: process.stdin,
|
|
stdout: process.stdout,
|
|
stderr: process.stderr,
|
|
env: process.env,
|
|
input,
|
|
}),
|
|
/iterator acquisition failed/,
|
|
);
|
|
assert.equal(inputClosed, true, "iterator acquisition failure must finish and close the stage");
|
|
});
|
|
|
|
test("explicit cancellation removes a corrupt workflow state through an aliased token", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-alias-corrupt-cancel-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const stateDir = join(dir, "state");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: stateDir };
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [{ id: "confirm", approval: "Continue?" }],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.approvalId);
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
|
|
const payload = decodeResumeToken(first.requiresApproval.resumeToken);
|
|
assert.equal(payload.kind, "workflow-file");
|
|
assert.ok(payload.stateKey?.startsWith("workflow_resume_"));
|
|
await writeFile(keyToPath(stateDir, payload.stateKey), '{"corrupt"', "utf8");
|
|
const aliasedToken = encodeToken({
|
|
...payload,
|
|
stateKey: payload.stateKey.replace("workflow_resume_", "workflow-resume_"),
|
|
});
|
|
|
|
const cancelled = await resumeToolRequest({
|
|
token: aliasedToken,
|
|
cancel: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(cancelled.ok, true);
|
|
assert.equal(cancelled.status, "cancelled");
|
|
assert.equal(await fileExists(keyToPath(stateDir, payload.stateKey)), false);
|
|
const replayById = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env },
|
|
});
|
|
assert.equal(replayById.ok, false);
|
|
assert.match(replayById.error?.message ?? "", /not found or expired/);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow on_error cannot swallow cancellation with a custom reason", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-on-error-custom-abort-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
const sideEffect = createCustomAbortSideEffect();
|
|
const registry = withCommands(createDefaultRegistry(), sideEffect.command);
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{
|
|
id: "effect",
|
|
pipeline: "test.custom-abort-side-effect",
|
|
on_error: "continue",
|
|
},
|
|
{
|
|
id: "later",
|
|
input: {
|
|
prompt: "This input must not start",
|
|
responseSchema: { type: "boolean" },
|
|
},
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const controller = new AbortController();
|
|
const run = runToolRequest({
|
|
filePath,
|
|
ctx: { cwd: dir, registry, signal: controller.signal, env },
|
|
});
|
|
await sideEffect.started;
|
|
controller.abort(new Error("shutdown"));
|
|
|
|
const envelope = await run;
|
|
|
|
assert.equal(envelope.ok, false);
|
|
assert.match(envelope.error?.message ?? "", /shutdown/);
|
|
assert.equal(
|
|
sideEffect.invocations,
|
|
1,
|
|
"custom cancellation must stop the workflow before later steps",
|
|
);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("approval resume token cannot replay after a non-abort failure past an unsafe boundary", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-resume-retry-"));
|
|
try {
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
let attempts = 0;
|
|
const source = {
|
|
name: "item",
|
|
async run() {
|
|
return { output: streamOf([{ value: 1 }]) };
|
|
},
|
|
};
|
|
const failOnce = {
|
|
name: "test.fail-once",
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
attempts += 1;
|
|
if (attempts === 1) throw new Error("retryable failure");
|
|
const items = [];
|
|
for await (const item of input) items.push(item);
|
|
return { output: streamOf(items) };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), source, failOnce);
|
|
const first = await runToolRequest({
|
|
pipeline: "item | approve --prompt Continue? | test.fail-once",
|
|
ctx: { registry, env },
|
|
});
|
|
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
const failed = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { registry, env },
|
|
});
|
|
assert.equal(failed.ok, false);
|
|
assert.match(failed.error?.message ?? "", /retryable failure/);
|
|
|
|
const replay = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { registry, env },
|
|
});
|
|
assert.equal(replay.ok, false);
|
|
assert.match(replay.error?.message ?? "", /Pipeline resume state not found/);
|
|
assert.equal(attempts, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("next-stage input resume cannot replay after a non-abort failure past an unsafe boundary", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-input-resume-retry-"));
|
|
try {
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
let attempts = 0;
|
|
const failOnce = {
|
|
name: "test.fail-once",
|
|
async run({ input }: { input: AsyncIterable<unknown> }) {
|
|
attempts += 1;
|
|
if (attempts === 1) throw new Error("retryable failure");
|
|
const items = [];
|
|
for await (const item of input) items.push(item);
|
|
return { output: streamOf(items) };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), failOnce);
|
|
const responseSchema = JSON.stringify({
|
|
type: "object",
|
|
properties: { value: { type: "number" } },
|
|
required: ["value"],
|
|
});
|
|
const first = await runToolRequest({
|
|
pipeline: `ask --emit --prompt Continue? --schema '${responseSchema}' | test.fail-once`,
|
|
ctx: { registry, env },
|
|
});
|
|
|
|
assert.equal(first.status, "needs_input");
|
|
assert.ok(first.requiresInput?.resumeToken);
|
|
const failed = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response: { value: 1 },
|
|
ctx: { registry, env },
|
|
});
|
|
assert.equal(failed.ok, false);
|
|
assert.match(failed.error?.message ?? "", /retryable failure/);
|
|
const replay = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response: { value: 1 },
|
|
ctx: { registry, env },
|
|
});
|
|
assert.equal(replay.ok, false);
|
|
assert.match(replay.error?.message ?? "", /Pipeline resume state not found/);
|
|
assert.equal(attempts, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow resume cannot replay after a non-abort failure past an unsafe boundary", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-resume-retry-"));
|
|
try {
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const env = { ...process.env, LOBSTER_STATE_DIR: join(dir, "state") };
|
|
let attempts = 0;
|
|
const failOnce = {
|
|
name: "test.fail-once",
|
|
async run() {
|
|
attempts += 1;
|
|
if (attempts === 1) throw new Error("retryable failure");
|
|
return { output: streamOf([{ ok: true }]) };
|
|
},
|
|
};
|
|
const registry = withCommands(createDefaultRegistry(), failOnce);
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "confirm", approval: "Continue?" },
|
|
{
|
|
id: "finish",
|
|
pipeline: "test.fail-once",
|
|
when: "$confirm.approved",
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const first = await runToolRequest({
|
|
filePath,
|
|
ctx: { cwd: dir, registry, env },
|
|
});
|
|
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
const failed = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, registry, env },
|
|
});
|
|
assert.equal(failed.ok, false);
|
|
assert.match(failed.error?.message ?? "", /retryable failure/);
|
|
const replay = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, registry, env },
|
|
});
|
|
assert.equal(replay.ok, false);
|
|
assert.match(replay.error?.message ?? "", /Workflow resume state not found/);
|
|
assert.equal(attempts, 1);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("next-stage input resume cannot replay a send after downstream cancellation", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-input-resume-cancel-replay-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const stateDir = join(dir, "state");
|
|
const sendCompleted = join(dir, "send-completed");
|
|
const sendInvocations = join(dir, "send-invocations");
|
|
const searchStarted = join(dir, "search-started");
|
|
const searchTerminated = join(dir, "search-terminated");
|
|
const env = {
|
|
...process.env,
|
|
LOBSTER_STATE_DIR: stateDir,
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_COMPLETED_FILE: sendCompleted,
|
|
MOCK_GOG_SEND_INVOCATIONS_FILE: sendInvocations,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: searchStarted,
|
|
MOCK_GOG_SEARCH_TERMINATED_FILE: searchTerminated,
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "100",
|
|
};
|
|
const responseSchema = JSON.stringify({
|
|
type: "object",
|
|
properties: {
|
|
to: { type: "string" },
|
|
subject: { type: "string" },
|
|
body: { type: "string" },
|
|
},
|
|
required: ["to", "subject", "body"],
|
|
});
|
|
const pipeline = `ask --emit --prompt Draft --schema '${responseSchema}' | gog.gmail.send | gog.gmail.search --query newer_than:1d`;
|
|
const response = { to: "user@example.com", subject: "Hello", body: "World" };
|
|
const first = await runToolRequest({ pipeline, ctx: { env } });
|
|
|
|
assert.equal(first.status, "needs_input");
|
|
assert.ok(first.requiresInput?.resumeToken);
|
|
const controller = new AbortController();
|
|
const resumed = resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response,
|
|
ctx: { signal: controller.signal, env },
|
|
});
|
|
await waitForFile(searchStarted, 3000);
|
|
controller.abort();
|
|
const envelope = await resumed;
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
await waitForFile(sendCompleted, 1000);
|
|
await waitForFile(searchTerminated, 1000);
|
|
let invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1);
|
|
|
|
const replay = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response,
|
|
ctx: { env: { ...env, MOCK_GOG_COMPLETION_DELAY_MS: "0" } },
|
|
});
|
|
assert.equal(replay.ok, false);
|
|
assert.match(replay.error?.message ?? "", /Pipeline resume state not found/);
|
|
invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1, "retrying an aborted input token must not resend");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow approval resume cannot replay a send after downstream cancellation", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-approval-cancel-replay-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const stateDir = join(dir, "state");
|
|
const sendCompleted = join(dir, "send-completed");
|
|
const sendInvocations = join(dir, "send-invocations");
|
|
const searchStarted = join(dir, "search-started");
|
|
const searchTerminated = join(dir, "search-terminated");
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{
|
|
id: "draft",
|
|
run: `${JSON.stringify(process.execPath)} -e "process.stdout.write(JSON.stringify({to:'user@example.com',subject:'Hello',body:'World'}))"`,
|
|
},
|
|
{ id: "confirm", approval: "Send?", stdin: "$draft.json" },
|
|
{
|
|
id: "send",
|
|
pipeline: "gog.gmail.send",
|
|
stdin: "$draft.json",
|
|
when: "$confirm.approved",
|
|
},
|
|
{
|
|
id: "search",
|
|
pipeline: "gog.gmail.search --query newer_than:1d",
|
|
when: "$confirm.approved",
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const env = {
|
|
...process.env,
|
|
LOBSTER_STATE_DIR: stateDir,
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_COMPLETED_FILE: sendCompleted,
|
|
MOCK_GOG_SEND_INVOCATIONS_FILE: sendInvocations,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: searchStarted,
|
|
MOCK_GOG_SEARCH_TERMINATED_FILE: searchTerminated,
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "100",
|
|
};
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
|
|
assert.equal(first.status, "needs_approval");
|
|
assert.ok(first.requiresApproval?.resumeToken);
|
|
assert.ok(first.requiresApproval.approvalId);
|
|
const controller = new AbortController();
|
|
const resumed = resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, signal: controller.signal, env },
|
|
});
|
|
await waitForFile(searchStarted, 3000);
|
|
controller.abort();
|
|
const envelope = await resumed;
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
await waitForFile(sendCompleted, 1000);
|
|
await waitForFile(searchTerminated, 1000);
|
|
let invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1);
|
|
|
|
const replayById = await resumeToolRequest({
|
|
approvalId: first.requiresApproval.approvalId,
|
|
approved: true,
|
|
ctx: { cwd: dir, env: { ...env, MOCK_GOG_COMPLETION_DELAY_MS: "0" } },
|
|
});
|
|
assert.equal(replayById.ok, false);
|
|
assert.match(replayById.error?.message ?? "", /not found or expired/);
|
|
const replayByToken = await resumeToolRequest({
|
|
token: first.requiresApproval.resumeToken,
|
|
approved: true,
|
|
ctx: { cwd: dir, env: { ...env, MOCK_GOG_COMPLETION_DELAY_MS: "0" } },
|
|
});
|
|
assert.equal(replayByToken.ok, false);
|
|
assert.match(replayByToken.error?.message ?? "", /Workflow resume state not found/);
|
|
invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1, "retrying an aborted workflow approval must not resend");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("workflow input resume cannot replay a send after downstream cancellation", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-workflow-input-cancel-replay-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGog = join(repoRoot, "test", "fixtures", "mock-gog-cancellation.mjs");
|
|
const filePath = join(dir, "workflow.lobster");
|
|
const stateDir = join(dir, "state");
|
|
const sendCompleted = join(dir, "send-completed");
|
|
const sendInvocations = join(dir, "send-invocations");
|
|
const searchStarted = join(dir, "search-started");
|
|
const searchTerminated = join(dir, "search-terminated");
|
|
const responseSchema = {
|
|
type: "object",
|
|
properties: {
|
|
to: { type: "string" },
|
|
subject: { type: "string" },
|
|
body: { type: "string" },
|
|
},
|
|
required: ["to", "subject", "body"],
|
|
};
|
|
await writeFile(
|
|
filePath,
|
|
JSON.stringify({
|
|
steps: [
|
|
{ id: "draft", input: { prompt: "Draft?", responseSchema } },
|
|
{ id: "send", pipeline: "gog.gmail.send", stdin: "$draft.response" },
|
|
{
|
|
id: "search",
|
|
pipeline: "gog.gmail.search --query newer_than:1d",
|
|
},
|
|
],
|
|
}),
|
|
"utf8",
|
|
);
|
|
const env = {
|
|
...process.env,
|
|
LOBSTER_STATE_DIR: stateDir,
|
|
GOG_BIN: mockGog,
|
|
MOCK_GOG_SEND_COMPLETED_FILE: sendCompleted,
|
|
MOCK_GOG_SEND_INVOCATIONS_FILE: sendInvocations,
|
|
MOCK_GOG_SEARCH_STARTED_FILE: searchStarted,
|
|
MOCK_GOG_SEARCH_TERMINATED_FILE: searchTerminated,
|
|
MOCK_GOG_COMPLETION_DELAY_MS: "100",
|
|
};
|
|
const response = { to: "user@example.com", subject: "Hello", body: "World" };
|
|
const first = await runToolRequest({ filePath, ctx: { cwd: dir, env } });
|
|
|
|
assert.equal(first.status, "needs_input");
|
|
assert.ok(first.requiresInput?.resumeToken);
|
|
const controller = new AbortController();
|
|
const resumed = resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response,
|
|
ctx: { cwd: dir, signal: controller.signal, env },
|
|
});
|
|
await waitForFile(searchStarted, 3000);
|
|
controller.abort();
|
|
const envelope = await resumed;
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
await waitForFile(sendCompleted, 1000);
|
|
await waitForFile(searchTerminated, 1000);
|
|
let invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1);
|
|
|
|
const replay = await resumeToolRequest({
|
|
token: first.requiresInput.resumeToken,
|
|
response,
|
|
ctx: { cwd: dir, env: { ...env, MOCK_GOG_COMPLETION_DELAY_MS: "0" } },
|
|
});
|
|
assert.equal(replay.ok, false);
|
|
assert.match(replay.error?.message ?? "", /Workflow resume state not found/);
|
|
invocations = (await readFile(sendInvocations, "utf8")).trim().split(/\r?\n/);
|
|
assert.equal(invocations.length, 1, "retrying an aborted workflow input must not resend");
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test(
|
|
"parent cancellation terminates the default github.pr.monitor subprocess",
|
|
{ skip: process.platform === "win32" },
|
|
async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cancel-gh-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGh = join(repoRoot, "test", "fixtures", "mock-gh-cancellation.mjs");
|
|
const ghBin = join(dir, "gh");
|
|
const started = join(dir, "gh-started");
|
|
const terminated = join(dir, "gh-terminated");
|
|
const completed = join(dir, "gh-completed");
|
|
await copyFile(mockGh, ghBin);
|
|
await chmod(ghBin, 0o755);
|
|
const controller = new AbortController();
|
|
const run = runToolRequest({
|
|
pipeline:
|
|
'workflows.run --name github.pr.monitor --args-json \'{"repo":"openclaw/lobster","pr":1}\'',
|
|
ctx: {
|
|
signal: controller.signal,
|
|
env: {
|
|
...process.env,
|
|
PATH: `${dir}:${process.env.PATH ?? ""}`,
|
|
LOBSTER_STATE_DIR: join(dir, "state"),
|
|
MOCK_GH_STARTED_FILE: started,
|
|
MOCK_GH_TERMINATED_FILE: terminated,
|
|
MOCK_GH_COMPLETED_FILE: completed,
|
|
MOCK_GH_TERMINATION_DELAY_MS: "1000",
|
|
},
|
|
},
|
|
});
|
|
|
|
await waitForFile(started, 15_000);
|
|
const childPid = Number(await readFile(started, "utf8"));
|
|
controller.abort();
|
|
const immediate = await observeSettlement(run, 50);
|
|
const observed = await observeSettlement(run, 500);
|
|
const envelope = observed.settled ? observed.value : await run;
|
|
|
|
assert.equal(immediate.settled, false, "workflow must wait for gh to exit");
|
|
assert.equal(observed.settled, true, "workflow subprocess should stop after abort");
|
|
assertCancellationEnvelope(envelope);
|
|
await waitForFile(terminated, 500);
|
|
assert.equal(processIsRunning(childPid), false);
|
|
assert.equal(await fileExists(terminated), true);
|
|
assert.equal(await fileExists(completed), false);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
);
|
|
|
|
test(
|
|
"cancellation before github.pr.monitor persistence does not advance state",
|
|
{ skip: process.platform === "win32" },
|
|
async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "lobster-cancel-gh-state-"));
|
|
try {
|
|
const repoRoot = join(__dirname, "..", "..");
|
|
const mockGh = join(repoRoot, "test", "fixtures", "mock-gh-cancellation.mjs");
|
|
const ghBin = join(dir, "gh");
|
|
const stateDir = join(dir, "state");
|
|
await copyFile(mockGh, ghBin);
|
|
await chmod(ghBin, 0o755);
|
|
const controller = new AbortController();
|
|
const signal = controller.signal;
|
|
const throwIfAborted = signal.throwIfAborted.bind(signal);
|
|
let signalChecks = 0;
|
|
Object.defineProperty(signal, "throwIfAborted", {
|
|
value() {
|
|
signalChecks += 1;
|
|
if (signalChecks === 4) {
|
|
controller.abort(new Error("abort before monitor state write"));
|
|
}
|
|
throwIfAborted();
|
|
},
|
|
});
|
|
const key = "github.pr:openclaw/lobster#1";
|
|
|
|
const envelope = await runToolRequest({
|
|
pipeline:
|
|
'workflows.run --name github.pr.monitor --args-json \'{"repo":"openclaw/lobster","pr":1,"changesOnly":true}\'',
|
|
ctx: {
|
|
signal,
|
|
env: {
|
|
...process.env,
|
|
PATH: `${dir}:${process.env.PATH ?? ""}`,
|
|
LOBSTER_STATE_DIR: stateDir,
|
|
MOCK_GH_COMPLETION_DELAY_MS: "0",
|
|
},
|
|
},
|
|
});
|
|
|
|
assertCancellationEnvelope(envelope);
|
|
assert.equal(signalChecks, 4);
|
|
assert.equal(
|
|
await fileExists(keyToPath(stateDir, key)),
|
|
false,
|
|
"cancelled monitor must not persist its snapshot",
|
|
);
|
|
} finally {
|
|
await rm(dir, { recursive: true, force: true });
|
|
}
|
|
},
|
|
);
|