diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.md index 3a11fd1464..eaee145c11 100644 --- a/doc/native-workspace-finalization-recovery.md +++ b/doc/native-workspace-finalization-recovery.md @@ -15,6 +15,18 @@ exact source proof under publication row locks. A changed source or checkpoint keeps local authority. Provider stop verification and any later file export remain the recovery owner's separate obligations. +The same local file handoff can preserve a blocked prepared source after a +native session is interrupted with ambiguous state, or after its file export +recovery is exhausted. The interruption may precede a provider session. Its +recorded session flag must be a boolean; it does not prove remote recovery. +The interrupted source must retain its exact task failure block and declared +recovery allocations. The exhausted source must retain its accepted +result, bound original allocation, exhausted system recovery and blocked file +recovery. An empty list of additional recovery allocations does not remove the +original prepared allocation binding. The migration checks these records again +under the same physical and task locks before publication. It keeps the failed +run, accepted result, recovery action and remote file obligation unchanged. + Periodic heartbeat recovery runs one pass at a time in each server process. A tick that arrives during a slow pass reports its age and starts no duplicate. A later tick can start a fresh pass after the current work settles. Shutdown diff --git a/server/src/services/online-legacy-workspace-recovery.ts b/server/src/services/online-legacy-workspace-recovery.ts index 2b3e5dc18f..4f6795d602 100644 --- a/server/src/services/online-legacy-workspace-recovery.ts +++ b/server/src/services/online-legacy-workspace-recovery.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { and, eq } from "drizzle-orm"; -import { environmentLeases, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { environmentLeases, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db"; import { adapterExecutionControls } from "./adapter-execution-control.js"; import { inspectFinalizedWorkspaceRecoveryHold, type FinalizedWorkspaceRecoveryHold } from "./finalized-workspace-recovery-migration.js"; import { isLegacyWorkspaceTaskRelated, readLegacyWorkspaceRecoveryAllocations, type LegacyWorkspaceRecoverySource } from "./legacy-workspace-recovery-migration.js"; @@ -8,6 +8,8 @@ import { tryLockNativeWorkspaceFinalization } from "./native-runtime/native-work import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js"; import { workspaceExportCauses } from "./workspace-recovery-source.js"; import { nativeSha256 } from "./native-runtime/canonical.js"; +import { readSystemRecovery } from "./system-recovery-state.js"; +import { readWorkspaceRecovery } from "./workspace-recovery-receipt.js"; type ReadDb = Pick; type Workspace = Pick; @@ -131,13 +133,16 @@ export async function inspectOnlineLegacyWorkspaceRecoveryHold(db: ReadDb, input if (!settled) return null; const failureBlock = recovery.evidence.nativeFailureBlock as { runId?: unknown; statusVersion?: unknown } | undefined; if (recovery.kind === "active_run_watchdog" && recovery.status === "active" - && recovery.cause === "runner_remote_provider_artifact_incompatible" && settled.run.status === "failed" + && ["runner_remote_provider_artifact_incompatible", "native_session_interrupted"].includes(recovery.cause) && settled.run.status === "failed" && settled.owner.phase === "terminal_failure" && settled.owner.nextAttemptAt === null && settled.owner.resultId === null && settled.owner.failureCode === recovery.cause && recovery.sourceIssueId === settled.run.nativeIssueId && recovery.ownerType === "board" && recovery.ownerAgentId === null && recovery.returnOwnerAgentId === settled.run.agentId && recovery.wakePolicy === null && recovery.evidence.sourceFailureCode === recovery.cause - && recovery.evidence.recoveryDisposition === recovery.cause && recovery.evidence.providerSessionEstablished === true - && recovery.evidence.recoveryMode === "exact_checkpoint_resume" && failureBlock?.runId === settled.run.id + && recovery.evidence.recoveryDisposition === recovery.cause + && (recovery.cause === "native_session_interrupted" + ? typeof recovery.evidence.providerSessionEstablished === "boolean" && recovery.evidence.recoveryMode === "ambiguous_state" + : recovery.evidence.providerSessionEstablished === true && recovery.evidence.recoveryMode === "exact_checkpoint_resume") + && failureBlock?.runId === settled.run.id && Number.isSafeInteger(failureBlock.statusVersion) && Number(failureBlock.statusVersion) > 0 && recovery.fingerprint === createHash("sha256").update(`${settled.run.id}:${recovery.cause}`).digest("hex") && !fileKeys.filter(key => key !== "nativeFailureBlock").some(key => recovery.evidence?.[key] != null)) { @@ -160,6 +165,29 @@ export async function inspectOnlineLegacyWorkspaceRecoveryHold(db: ReadDb, input if (source?.reference?.state === "prepared" && source.issueId === recovery.sourceIssueId) return { fingerprint: fingerprint({ recovery, source }), finalizedHold: null, preparedSource: source }; } + const system = readSystemRecovery(recovery.evidence), workspaceRecovery = readWorkspaceRecovery(recovery.evidence); + if (recovery.kind === "active_run_watchdog" && recovery.status === "active" + && recovery.cause === "native_workspace_sync_out_retry_exhausted" && settled.run.status === "failed" + && settled.owner.phase === "terminal_failure" && settled.owner.nextAttemptAt === null && settled.owner.resultId + && settled.owner.failureCode === recovery.cause && settled.run.completionContractId + && recovery.sourceIssueId === settled.run.nativeIssueId && ["system", "board"].includes(recovery.ownerType) + && recovery.ownerAgentId === null && recovery.returnOwnerAgentId === settled.run.agentId && recovery.wakePolicy === null + && recovery.fingerprint === nativeSha256({ runId: settled.run.id, failureCode: recovery.cause }) + && system?.sourceRunId === settled.run.id && system.state === "exhausted" && system.nextAttemptAt === null + && workspaceRecovery?.runId === settled.run.id && workspaceRecovery.status === "blocked" + && !["workspaceRestoreRecovery", "historicalWorkspaceSource", "nativeFailureBlock"].some(key => recovery.evidence?.[key] != null)) { + const [task] = await db.select({ id: issues.id, status: issues.status, statusVersion: issues.statusVersion, + executionRunId: issues.executionRunId, checkoutRunId: issues.checkoutRunId }).from(issues).where(and( + eq(issues.companyId, input.workspace.companyId), eq(issues.id, recovery.sourceIssueId))); + const [result] = await db.select().from(nativeRunResults).where(and( + eq(nativeRunResults.companyId, input.workspace.companyId), eq(nativeRunResults.issueId, recovery.sourceIssueId), + eq(nativeRunResults.runId, settled.run.id), eq(nativeRunResults.id, settled.owner.resultId), + eq(nativeRunResults.completionContractId, settled.run.completionContractId), eq(nativeRunResults.schemaStatus, "accepted"))); + if (!task || task.status !== "blocked" || task.executionRunId !== null || task.checkoutRunId !== null || !result) return null; + const source = await inspectOnlineLegacyWorkspaceRecoverySource(db, { workspace: input.workspace, runId: settled.run.id }); + if (source?.reference?.state === "prepared" && source.issueId === recovery.sourceIssueId) + return { fingerprint: fingerprint({ recovery, system, workspaceRecovery, source, task, result }), finalizedHold: null, preparedSource: source }; + } const held = await inspectFinalizedWorkspaceRecoveryHold(db, { companyId: input.workspace.companyId, workspaceId: input.workspace.id, recovery, }); diff --git a/server/src/services/remote-workspace-online-source.test.ts b/server/src/services/remote-workspace-online-source.test.ts index 576af4605f..7776d6f104 100644 --- a/server/src/services/remote-workspace-online-source.test.ts +++ b/server/src/services/remote-workspace-online-source.test.ts @@ -6,7 +6,7 @@ import os from "node:os"; import path from "node:path"; import { sql, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { activityLog, agents, companies, createDb, environmentLeases, environments, executionWorkspaceSnapshots, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices, type Db } from "@paperclipai/db"; +import { activityLog, agents, companies, completionContracts, createDb, environmentLeases, environments, executionWorkspaceSnapshots, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices, type Db } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority, readRemoteWorkspaceRunReference } from "@paperclipai/shared"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { resolveManagedProjectWorkspaceDir, resolvePaperclipInstanceRoot } from "../home-paths.js"; @@ -21,10 +21,13 @@ import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifec import { IsolatedRepositorySourceBusy, withIsolatedRepositorySourceLock, withWorkspaceSourceLocks } from "./workspace-source-lock.js"; import { tryLockNativeWorkspaceFinalization } from "./native-runtime/native-workspace-finalization-ownership.js"; import { recordNativeFinalizationFailure } from "./native-runtime/native-run-finalizer.js"; +import { nativeProviderRecoveryEvidence, nativeSessionFailureDisposition } from "./native-runtime/native-session-executor.js"; import { inspectOnlineWorkspaceSource } from "./online-workspace-source.js"; import { inspectOnlineLegacyWorkspaceRecoveryHold, inspectOnlineLegacyWorkspaceRecoverySource } from "./online-legacy-workspace-recovery.js"; import { nativeWorkspaceSyncInternals } from "./native-runtime/native-workspace-sync.js"; import { readMigratedLegacyWorkspaceSource } from "./legacy-workspace-recovery-migration.js"; +import { createSystemRecovery } from "./system-recovery-state.js"; +import { nativeSha256 } from "./native-runtime/canonical.js"; import { verifiedWorkspaceRepairReceipt } from "./workspace-recovery-receipt.js"; import { adapterExecutionControls, createAdapterExecutionControl } from "./adapter-execution-control.js"; import { withWorkspaceMoveLocks } from "./isolated-repository-rehome.js"; @@ -53,10 +56,12 @@ describe.skipIf(!support.supported)("online ordinary physical source fence", () await db.delete(executionWorkspaceSnapshots); await db.delete(workspaceRuntimeServices); await db.delete(nativeRunFinalizations); + await db.delete(nativeRunResults); await db.delete(environmentLeases); await db.delete(environments); await db.delete(heartbeatRunEvents); await db.delete(heartbeatRuns); + await db.delete(completionContracts); await db.delete(issues); await db.delete(agents); await db.delete(executionWorkspaces); @@ -520,31 +525,135 @@ describe.skipIf(!support.supported)("online ordinary physical source fence", () status: "active", evidence: { runId } }); return { ...f, run, owner, lease, warning }; } - async function artifactFailureFixture(descendant = false) { - const f = await recoveryFixture(descendant), cause = "runner_remote_provider_artifact_incompatible", statusVersion = 2; - const profile = { ...f.run.runnerProfileJson, sessionCheckpoint: { backendKind: "runner", providerSessionId: "fixture-established-session" } }; + async function artifactFailureFixture(descendant = false, cause: "runner_remote_provider_artifact_incompatible" | "native_session_interrupted" = "runner_remote_provider_artifact_incompatible") { + const f = await recoveryFixture(descendant), statusVersion = 2; + const profile = { ...f.run.runnerProfileJson, sessionCheckpoint: cause === "native_session_interrupted" + ? { backendKind: "runner" } : { backendKind: "runner", providerSessionId: "fixture-established-session" } }; const restore = { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] }; - await db.update(heartbeatRuns).set({ runnerProfileJson: profile, nextEventSeq: 250, + await db.update(heartbeatRuns).set({ runnerProfileJson: profile, nextEventSeq: cause === "native_session_interrupted" ? 1 : 250, resultJson: { ...f.run.resultJson, workspaceRestoreRecovery: restore } }).where(eq(heartbeatRuns.id, f.run.id)); - await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: cause, nextAttemptAt: null }) - .where(eq(nativeRunFinalizations.runId, f.run.id)); + const events = cause === "native_session_interrupted" ? [] : ["session.started", ...Array.from({ length: 124 }, () => ["tool.execution.started", "tool.execution.completed"]).flat()]; + if (events.length) await db.insert(heartbeatRunEvents).values(events.map((eventType, index) => ({ companyId: f.companyId, + runId: f.run.id, agentId: f.run.agentId, seq: index + 1, eventType }))); + const evidence = await nativeProviderRecoveryEvidence({ db, runId: f.run.id, metadataOnly: true, sourceFailureCode: cause }); + expect(evidence).toMatchObject({ recoveryMode: cause === "native_session_interrupted" ? "ambiguous_state" : "exact_checkpoint_resume", providerSessionEstablished: cause !== "native_session_interrupted" }); + const disposition = nativeSessionFailureDisposition(1, new Date(), cause); + const phase = evidence.recoveryMode === "ambiguous_state" ? "terminal_failure" : disposition.phase; + const failureCode = evidence.recoveryMode === "ambiguous_state" ? cause : disposition.failureCode; + expect({ phase, failureCode }).toEqual({ phase: "terminal_failure", failureCode: cause }); + await db.update(nativeRunFinalizations).set({ phase, failureCode, nextAttemptAt: null }).where(eq(nativeRunFinalizations.runId, f.run.id)); await db.update(issues).set({ status: "blocked", statusVersion, executionRunId: null, checkoutRunId: null }) .where(eq(issues.id, f.run.nativeIssueId!)); await db.update(issueRecoveryActions).set({ cause, ownerType: "board", ownerAgentId: null, returnOwnerAgentId: f.run.agentId, wakePolicy: null, fingerprint: createHash("sha256").update(`${f.run.id}:${cause}`).digest("hex"), evidence: { runId: f.run.id, coordinatorAttempt: 3, sourceFailureCode: cause, recoveryDisposition: cause, - nativeFailureBlock: { runId: f.run.id, statusVersion }, providerSessionEstablished: true, recoveryMode: "exact_checkpoint_resume", + nativeFailureBlock: { runId: f.run.id, statusVersion }, providerSessionEstablished: evidence.providerSessionEstablished, recoveryMode: evidence.recoveryMode, } }).where(eq(issueRecoveryActions.id, f.warning.id)); - const events = ["session.started", ...Array.from({ length: 124 }, () => ["tool.execution.started", "tool.execution.completed"]).flat()]; - await db.insert(heartbeatRunEvents).values(events.map((eventType, index) => ({ companyId: f.companyId, - runId: f.run.id, agentId: f.run.agentId, seq: index + 1, eventType }))); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); const [owner] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.run.id)); const [warning] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.warning.id)); const [issue] = await db.select().from(issues).where(eq(issues.id, run.nativeIssueId!)); return { ...f, run, owner, warning, issue }; } + async function heldLocalFixture(kind: "interrupted" | "exhausted") { + const f = kind === "interrupted" ? await artifactFailureFixture(false, "native_session_interrupted") : await recoveryFixture(); + const reference = { ...f.run.runnerProfileJson!.nativeWorkspaceSync as Record, resourceDisposition: "keep_running" }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...f.run.runnerProfileJson, nativeWorkspaceSync: reference } }) + .where(eq(heartbeatRuns.id, f.run.id)); + if (kind === "exhausted") { + const cause = "native_workspace_sync_out_retry_exhausted", contractId = randomUUID(), resultId = randomUUID(); + await db.insert(completionContracts).values({ id: contractId, companyId: f.companyId, issueId: f.run.nativeIssueId!, revision: 1, + schemaVersion: "v1", policyVersion: "v1", risk: "low", completionAuthority: "agent", incompleteCriteriaPolicy: "block", + contractJson: {}, canonicalSha256: "45".repeat(32), createdByActorType: "system", createdByActorId: "fixture" }); + await db.update(heartbeatRuns).set({ completionContractId: contractId, completionContractSha256: "45".repeat(32), + resultJson: { retained: "original failed-run result" } }).where(eq(heartbeatRuns.id, f.run.id)); + await db.insert(nativeRunResults).values({ id: resultId, companyId: f.companyId, issueId: f.run.nativeIssueId!, runId: f.run.id, + completionContractId: contractId, serverFingerprint: "accepted retained export result", schemaStatus: "accepted", + resultJson: { terminal: { runTerminalState: "succeeded" }, retained: "original accepted result" }, canonicalSha256: "67".repeat(32) }); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", resultId, failureCode: cause, nextAttemptAt: null, + recoveryHistory: [{ retained: "original remote export remains unresolved" }] }).where(eq(nativeRunFinalizations.runId, f.run.id)); + await db.update(issues).set({ status: "blocked", statusVersion: 5, executionRunId: null, checkoutRunId: null }) + .where(eq(issues.id, f.run.nativeIssueId!)); + await db.update(issueRecoveryActions).set({ cause, ownerType: "system", ownerAgentId: null, returnOwnerAgentId: f.run.agentId, + wakePolicy: null, fingerprint: nativeSha256({ runId: f.run.id, failureCode: cause }), evidence: { + runId: f.run.id, originalFailureCode: "native_workspace_sync_out_transient", coordinatorAttempt: 3, workspaceFinalizeAttempt: 3, + systemRecovery: { ...createSystemRecovery(f.run.id), phase: "preserve_files", state: "exhausted", attemptId: randomUUID(), attempts: 4, retriesUsed: 3 }, + workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: f.run.id, status: "blocked", stage: "verifying_stop", attempt: 4, requestIds: [], retryable: true }, + previousWorkspaceRecovery: { retained: "original recovery history" }, + } }).where(eq(issueRecoveryActions.id, f.warning.id)); + } + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + const [owner] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.run.id)); + const [warning] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.warning.id)); + const [issue] = await db.select().from(issues).where(eq(issues.id, f.run.nativeIssueId!)); + return { ...f, run, owner, warning, issue }; + } + it.each(["interrupted", "exhausted"] as const)("preserves a stable local Git copy with its %s remote hold intact", async kind => { + const recovery = await heldLocalFixture(kind), f = { ...recovery, ...await gitFixture(recovery) }; + const results = await db.select().from(nativeRunResults), unrelated = await remoteWriter(f.companyId, f.run.agentId); + await expect(migrateRemoteWorkspace({ db, storage, companyId: f.companyId, workspaceId: f.child.id, apply: true })) + .resolves.toMatchObject({ applied: true }); + const put = storage.putFile.bind(storage); + vi.spyOn(storage, "putFile").mockImplementationOnce(async input => { + await db.transaction(async tx => expect(await tryLockNativeWorkspaceFinalization(tx as unknown as Db, { + companyId: f.companyId, runId: f.run.id, + })).toBe(false)); + return put(input); + }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).resolves.toMatchObject({ applied: true }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + expect(run.status).toBe("failed"); expect(run.resultJson).toEqual(f.run.resultJson); + expect(run.runnerProfileJson).toMatchObject({ nativeWorkspaceSync: f.run.runnerProfileJson!.nativeWorkspaceSync }); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.run.id)))[0]).toEqual(f.owner); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]).toEqual(f.lease); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.warning.id)))[0]).toEqual(f.warning); + expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0]).toEqual(f.issue); + expect(await db.select().from(nativeRunResults)).toEqual(results); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, unrelated.id)))[0]).toEqual(unrelated); + expect(await verifiedWorkspaceRepairReceipt(db, run)).toBeNull(); + const authority = readRemoteWorkspaceAuthority((await stored(f.workspaceId)).metadata)!; + const reference = (await getWorkspaceSnapshot(db, { ...f, snapshotId: authority.snapshotId! }))!; + const archivePath = path.join(home, "held-local.pcws"), targetDir = path.join(home, "held-local-restored"); + await loadWorkspaceSnapshotArchive({ storage, reference, targetPath: archivePath }); + await fs.rm(f.cwd, { recursive: true }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: reference.manifestDigest }); + expect(await fs.readFile(path.join(targetDir, "saved.txt"), "utf8")).toBe("saved legacy edits\n"); + }); + it.each((["interrupted", "exhausted"] as const).flatMap(kind => + ["owner", "active", "fingerprint", "allocation", "recovery", ...(kind === "interrupted" ? ["mode", "provider-flag"] : [])].map(mode => ({ kind, mode }))))("keeps $kind local files held on $mode mismatch", async ({ kind, mode }) => { + const f = await heldLocalFixture(kind); + if (mode === "owner") await db.update(nativeRunFinalizations).set({ leaseOwner: "unreleased", leaseExpiresAt: new Date(0) }).where(eq(nativeRunFinalizations.runId, f.run.id)); + if (mode === "active") await db.update(heartbeatRuns).set({ controllerLeaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(heartbeatRuns.id, f.run.id)); + if (mode === "fingerprint") await db.update(issueRecoveryActions).set({ fingerprint: "00".repeat(32) }).where(eq(issueRecoveryActions.id, f.warning.id)); + if (mode === "mode") await db.update(issueRecoveryActions).set({ evidence: { ...f.warning.evidence, recoveryMode: "exact_checkpoint_resume" } }).where(eq(issueRecoveryActions.id, f.warning.id)); + if (mode === "provider-flag") await db.update(issueRecoveryActions).set({ evidence: { ...f.warning.evidence, providerSessionEstablished: "false" } }).where(eq(issueRecoveryActions.id, f.warning.id)); + if (mode === "allocation") await db.update(environmentLeases).set({ providerLeaseId: randomUUID() }).where(eq(environmentLeases.id, f.lease.id)); + if (mode === "recovery") await db.update(issueRecoveryActions).set({ evidence: kind === "exhausted" + ? { ...f.warning.evidence, systemRecovery: { ...createSystemRecovery(f.run.id), state: "running" } } + : { ...f.warning.evidence, nativeFailureBlock: { runId: f.run.id, statusVersion: f.issue.statusVersion + 1 } } }).where(eq(issueRecoveryActions.id, f.warning.id)); + const warning = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.warning.id)); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow(); + expect(await db.select().from(workspaceOperations)).toHaveLength(0); + expect(readRemoteWorkspaceAuthority((await stored(f.workspaceId)).metadata)).toBeNull(); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.warning.id))).toEqual(warning); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("saved legacy edits\n"); + }); + it.each(["interrupted", "exhausted"] as const)("rejects a %s hold change during local capture", async kind => { + const f = await heldLocalFixture(kind), put = storage.putFile.bind(storage); + vi.spyOn(storage, "putFile").mockImplementationOnce(async input => { + const result = await put(input); + if (kind === "interrupted") await db.update(issueRecoveryActions).set({ evidence: { ...f.warning.evidence, coordinatorAttempt: 4 } }).where(eq(issueRecoveryActions.id, f.warning.id)); + else await db.update(nativeRunResults).set({ resultJson: { changed: "new accepted result bytes" } }).where(eq(nativeRunResults.runId, f.run.id)); + return result; + }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow(); + expect(readRemoteWorkspaceAuthority((await stored(f.workspaceId)).metadata)).toBeNull(); + expect((await db.select().from(workspaceOperations))[0].status).toBe("running"); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.run.id)))[0]).toEqual(f.owner); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]).toEqual(f.lease); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("saved legacy edits\n"); + }); it("preserves provider-established artifact failure holds while migrating local Git aliases and their source recipe", async () => { const recovery = await artifactFailureFixture(), f = { ...recovery, ...await gitFixture(recovery) }; const original = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id));