diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 4e92bd2d15..cdaefa0e50 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3524,6 +3524,11 @@ provider pack when its verified durable session binding and pack receipt remain available. A new controller release or diagnostics setting does not replace the proxy behind the saved provider state. If the exact artifacts are unavailable, continuation stays blocked; this rule grants no fresh-session or replay authority. +A configured controller pack with a different identity is not uploaded in its +place: before clearing or uploading anything, the run fails as +`runner_remote_provider_artifact_incompatible` and names both digest prefixes. +Archive integrity failures keep `runner_remote_provider_archive_invalid` with a +fixed reason such as `size_mismatch` or `sha256_mismatch`. A newly launched daemon binds its current Runner artifact while retaining the provider pack. Only a surviving daemon retains its previous Runner artifact. An authorized continuity break releases the pack pin after archiving the old diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c965fa59cb..0fbc7367d3 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -12858,6 +12858,7 @@ export async function prepareRemoteProviderPackOnTarget(input: { } await uploadVerifiedProviderPack({ root: configuredProviderPackRoot, + sourceDigest: readRemoteProviderPackIdentity(configuredProviderPackRoot, false).digest, expectedDigest: expectedProviderPackManifest.digest, runner: remoteCommandRunner, cwd: remoteTarget.remoteCwd, diff --git a/server/src/services/native-runtime/provider-pack-archive.test.ts b/server/src/services/native-runtime/provider-pack-archive.test.ts index af815e0747..e602f88764 100644 --- a/server/src/services/native-runtime/provider-pack-archive.test.ts +++ b/server/src/services/native-runtime/provider-pack-archive.test.ts @@ -148,7 +148,9 @@ it.skipIf(!tar || !hasZstd)("rejects a corrupt missing layer before transfer and expect((await fs.readdir(root)).some(name => name.includes(".upload-"))).toBe(false); await fs.appendFile(`${pack}.archives/0.tar.zst`, "corrupt"); remote.syncIn.mockClear(); - await expect(stageProviderPackArchive({ archive, runner: remote, targetPath: `${targetPath}-2` })).rejects.toThrow("runner_remote_provider_archive_invalid"); + await expect(stageProviderPackArchive({ archive, runner: remote, targetPath: `${targetPath}-2` })).rejects.toThrow("runner_remote_provider_archive_invalid: size_mismatch"); + await fs.unlink(`${pack}.archives/0.tar.zst`); + await expect(stageProviderPackArchive({ archive, runner: remote, targetPath: `${targetPath}-3` })).rejects.toThrow("runner_remote_provider_archive_invalid: archive_unreadable"); expect(remote.syncIn).not.toHaveBeenCalled(); }); @@ -248,7 +250,7 @@ it.skipIf(!tar || !hasZstd).each(["v1", "v2"])("installs verified %s archive byt expect(await fs.readFile(path.join(targetPath, "bin/tool"))).toEqual(expected); expect(JSON.parse(await fs.readFile(path.join(targetPath, "provider-pack.json"), "utf8")).digest).toBe(digest); }); - await uploadVerifiedProviderPack({ root: pack, expectedDigest: digest, runner: remote, cwd: root, + await uploadVerifiedProviderPack({ root: pack, sourceDigest: digest, expectedDigest: digest, runner: remote, cwd: root, targetPath, reuseRoots: [], verifyDirectory, stageDirectory, verifyUploaded }); expect(verifyDirectory).not.toHaveBeenCalled(); expect(stageDirectory).not.toHaveBeenCalled(); @@ -263,7 +265,7 @@ it.each(["absent", "unsupported"])("verifies directory bytes before fallback whe remote.execute.mockResolvedValueOnce({ exitCode: 0, timedOut: false }); remote.execute.mockResolvedValue({ exitCode: 127, timedOut: false }); const stageDirectory = vi.fn(), verifyUploaded = vi.fn(); - await expect(uploadVerifiedProviderPack({ root: pack, expectedDigest: digest, runner: remote, cwd: root, + await expect(uploadVerifiedProviderPack({ root: pack, sourceDigest: digest, expectedDigest: digest, runner: remote, cwd: root, targetPath: path.join(root, "uploaded"), reuseRoots: [], verifyDirectory: () => { throw Error("directory_digest_mismatch"); }, stageDirectory, verifyUploaded })).rejects.toThrow("directory_digest_mismatch"); expect(remote.syncIn).not.toHaveBeenCalled(); @@ -271,11 +273,46 @@ it.each(["absent", "unsupported"])("verifies directory bytes before fallback whe expect(verifyUploaded).not.toHaveBeenCalled(); }); +// A retained continuation can select a pack from an earlier release. The +// controller's build-time archives then carry a different identity. That is an +// artifact incompatibility, not archive corruption, and it must not clear the +// remote pack or upload the controller's different pack. +it.each(["v1", "v2", "directory"])("refuses a selected pack this controller does not ship (%s source)", async source => { + const { root, pack } = source === "v2" ? await layeredFixture() : await fixture(); + if (source === "directory") await fs.unlink(`${pack}.archive.json`); + const retained = `sha256:${"b".repeat(64)}`; + const remote = { syncIn: vi.fn(), execute: vi.fn().mockResolvedValue({ exitCode: 0, timedOut: false, stdout: "", stderr: "" }) }; + const verifyDirectory = vi.fn(), stageDirectory = vi.fn(), verifyUploaded = vi.fn(); + const upload = uploadVerifiedProviderPack({ root: pack, sourceDigest: digest, expectedDigest: retained, runner: remote, cwd: root, + targetPath: path.join(root, "uploaded"), reuseRoots: [], verifyDirectory, stageDirectory, verifyUploaded }); + await expect(upload).rejects.toThrow( + `runner_remote_provider_artifact_incompatible: selected provider pack ${"b".repeat(12)} is not the controller's pack ${"a".repeat(12)}`); + expect(remote.execute).not.toHaveBeenCalled(); + expect(remote.syncIn).not.toHaveBeenCalled(); + for (const step of [verifyDirectory, stageDirectory, verifyUploaded]) expect(step).not.toHaveBeenCalled(); +}); + +it.skipIf(!tar || !hasZstd).each(["v1", "v2"])("names the failed %s archive check in the run-visible error", async version => { + const { pack } = version === "v1" ? await fixture() : await layeredFixture(); + const archive = version === "v1" ? `${pack}.tar.zst` : `${pack}.archives/overlay.tar.zst`; + const bytes = await fs.readFile(archive); + await fs.appendFile(archive, "x"); + await expect(readProviderPackArchive(pack, digest)).rejects.toThrow("runner_remote_provider_archive_invalid: size_mismatch"); + bytes[bytes.length - 1] ^= 1; + await fs.writeFile(archive, bytes); + await expect(readProviderPackArchive(pack, digest)).rejects.toThrow("runner_remote_provider_archive_invalid: sha256_mismatch"); + const receipt = JSON.parse(await fs.readFile(`${pack}.archive.json`, "utf8")); + await fs.writeFile(`${pack}.archive.json`, JSON.stringify({ ...receipt, providerPackDigest: `sha256:${"c".repeat(64)}` })); + await expect(readProviderPackArchive(pack, digest)).rejects.toThrow("runner_remote_provider_archive_invalid: receipt_identity_mismatch"); + await fs.writeFile(`${pack}.archive.json`, "{"); + await expect(readProviderPackArchive(pack, digest)).rejects.toThrow("runner_remote_provider_archive_invalid: receipt_unreadable"); +}); + it.skipIf(!tar || !hasZstd)("rejects an installed pack that fails final verification", async () => { const { root, pack } = await fixture(); const remote = await runner(root), targetPath = path.join(root, "uploaded"); const verifyUploaded = vi.fn(async () => { throw Error("installed_digest_mismatch"); }); - await expect(uploadVerifiedProviderPack({ root: pack, expectedDigest: digest, runner: remote, cwd: root, + await expect(uploadVerifiedProviderPack({ root: pack, sourceDigest: digest, expectedDigest: digest, runner: remote, cwd: root, targetPath, reuseRoots: [], verifyDirectory: () => {}, stageDirectory: async () => {}, verifyUploaded })) .rejects.toThrow("installed_digest_mismatch"); expect(remote.syncIn).toHaveBeenCalledOnce(); diff --git a/server/src/services/native-runtime/provider-pack-archive.ts b/server/src/services/native-runtime/provider-pack-archive.ts index 2dde86fc28..0afc418789 100644 --- a/server/src/services/native-runtime/provider-pack-archive.ts +++ b/server/src/services/native-runtime/provider-pack-archive.ts @@ -1,9 +1,8 @@ -import { createHash, randomUUID } from "node:crypto"; -import { constants } from "node:fs"; -import { open, readFile } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; import type { NativeRunTrace } from "./native-run-trace.js"; -import { readProviderPackLayers, stageProviderPackLayers } from "./provider-pack-layers.js"; +import { invalidArchive, readProviderPackLayers, stageProviderPackLayers, verifyArchive } from "./provider-pack-layers.js"; import { providerPackExtractionProbe } from "./provider-pack-extraction.js"; const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; @@ -14,7 +13,7 @@ export async function readProviderPackArchive(root: string, expectedDigest: stri receipt = JSON.parse(await readFile(`${root}.archive.json`, "utf8")); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; - throw new Error("runner_remote_provider_archive_invalid", { cause: error }); + throw invalidArchive("receipt_unreadable", error); } if (receipt.schema === "paperclip-runner/provider-pack-archive/v2") { return readProviderPackLayers(root, expectedDigest, receipt); @@ -24,19 +23,10 @@ export async function readProviderPackArchive(root: string, expectedDigest: stri || receipt.providerPackDigest !== expectedDigest || !/^[a-f0-9]{64}$/.test(receipt.sha256 ?? "") || !Number.isSafeInteger(receipt.bytes) || receipt.bytes! <= 0) { - throw new Error("runner_remote_provider_archive_invalid"); + throw invalidArchive(receipt.providerPackDigest !== expectedDigest ? "receipt_identity_mismatch" : "receipt_invalid"); } const sourcePath = `${root}.tar.zst`; - const file = await open(sourcePath, constants.O_RDONLY | constants.O_NOFOLLOW); - try { - const metadata = await file.stat(); - if (!metadata.isFile() || metadata.size !== receipt.bytes) throw new Error("runner_remote_provider_archive_invalid"); - const hash = createHash("sha256"); - for await (const chunk of file.createReadStream({ autoClose: false })) hash.update(chunk); - if (hash.digest("hex") !== receipt.sha256) throw new Error("runner_remote_provider_archive_invalid"); - } finally { - await file.close(); - } + await verifyArchive(sourcePath, { sha256: receipt.sha256!, bytes: receipt.bytes! }); return { sourcePath, sha256: receipt.sha256!, bytes: receipt.bytes! }; } diff --git a/server/src/services/native-runtime/provider-pack-layers.ts b/server/src/services/native-runtime/provider-pack-layers.ts index 393183777e..158c1ca910 100644 --- a/server/src/services/native-runtime/provider-pack-layers.ts +++ b/server/src/services/native-runtime/provider-pack-layers.ts @@ -23,22 +23,31 @@ const validArchive = (value: unknown): value is Archive => { && Number.isSafeInteger(entry.bytes) && entry.bytes > 0; }; -async function verifyArchive(path: string, expected: Archive) { - const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); +// The run report keeps only the outer message. Name the failed check with a +// fixed reason so a recurrence is diagnosable without exposing private paths. +export const invalidArchive = (reason: string, cause?: unknown) => + new Error(`runner_remote_provider_archive_invalid: ${reason}`, cause === undefined ? undefined : { cause }); + +export async function verifyArchive(path: string, expected: Archive) { + let file; + try { file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); } + catch (error) { throw invalidArchive("archive_unreadable", error); } try { const metadata = await file.stat(); - if (!metadata.isFile() || metadata.size !== expected.bytes) throw Error("runner_remote_provider_archive_invalid"); + if (!metadata.isFile()) throw invalidArchive("archive_not_file"); + if (metadata.size !== expected.bytes) throw invalidArchive("size_mismatch"); const hash = createHash("sha256"); for await (const chunk of file.createReadStream({ autoClose: false })) hash.update(chunk); - if (hash.digest("hex") !== expected.sha256) throw Error("runner_remote_provider_archive_invalid"); + if (hash.digest("hex") !== expected.sha256) throw invalidArchive("sha256_mismatch"); } finally { await file.close(); } } export async function readProviderPackLayers(root: string, expectedDigest: string, value: unknown): Promise { const receipt = value as { encoding: string; providerPackDigest: string; overlay: Archive; files: FileLayer[] }; - if (receipt.encoding !== "tar+zstd" || receipt.providerPackDigest !== expectedDigest + if (receipt.providerPackDigest !== expectedDigest) throw invalidArchive("receipt_identity_mismatch"); + if (receipt.encoding !== "tar+zstd" || !validArchive(receipt.overlay) || !Array.isArray(receipt.files) || receipt.files.length > 10_000) { - throw Error("runner_remote_provider_archive_invalid"); + throw invalidArchive("receipt_invalid"); } const paths = new Set(); for (const file of receipt.files) { @@ -47,19 +56,20 @@ export async function readProviderPackLayers(root: string, expectedDigest: strin || file.path.startsWith("/") || file.path.includes("\\") || file.path.includes("\0") || file.path.split("/").some(part => !part || part === "." || part === "..") || !Number.isSafeInteger(file.mode) || file.mode < 0 || file.mode > 0o777 || paths.has(file.path)) { - throw Error("runner_remote_provider_archive_invalid"); + throw invalidArchive("layer_invalid"); } paths.add(file.path); } for (const path of paths) { let parent = posix.dirname(path); while (parent !== ".") { - if (paths.has(parent)) throw Error("runner_remote_provider_archive_invalid"); + if (paths.has(parent)) throw invalidArchive("layer_invalid"); parent = posix.dirname(parent); } } const sourceRoot = `${root}.archives`; - if (!(await lstat(sourceRoot)).isDirectory()) throw Error("runner_remote_provider_archive_invalid"); + const layers = await lstat(sourceRoot).catch(error => { throw invalidArchive("layers_unreadable", error); }); + if (!layers.isDirectory()) throw invalidArchive("layers_unreadable"); await verifyArchive(`${sourceRoot}/overlay.tar.zst`, receipt.overlay); return { kind: "layers", sourceRoot, overlay: receipt.overlay, files: receipt.files }; } @@ -142,14 +152,15 @@ export async function stageProviderPackLayers(input: { const hash = createHash("sha256"); await measure("provider_pack.archive.verify_missing", () => pipeline((async function* () { for (const layer of layers) { - const file = await open(`${archive.sourceRoot}/${layer.name}`, constants.O_RDONLY | constants.O_NOFOLLOW); + const file = await open(`${archive.sourceRoot}/${layer.name}`, constants.O_RDONLY | constants.O_NOFOLLOW) + .catch(error => { throw invalidArchive("archive_unreadable", error); }); try { const metadata = await file.stat(), layerHash = createHash("sha256"); - if (!metadata.isFile() || metadata.size !== layer.bytes) throw Error("runner_remote_provider_archive_invalid"); + if (!metadata.isFile() || metadata.size !== layer.bytes) throw invalidArchive("size_mismatch"); for await (const chunk of file.createReadStream({ autoClose: false })) { layerHash.update(chunk); hash.update(chunk); yield chunk; } - if (layerHash.digest("hex") !== layer.sha256) throw Error("runner_remote_provider_archive_invalid"); + if (layerHash.digest("hex") !== layer.sha256) throw invalidArchive("sha256_mismatch"); } finally { await file.close(); } } })(), createWriteStream(bundlePath, { flags: "wx", mode: 0o600 }))); diff --git a/server/src/services/native-runtime/provider-pack-upload.ts b/server/src/services/native-runtime/provider-pack-upload.ts index aff0ca4256..5e34872184 100644 --- a/server/src/services/native-runtime/provider-pack-upload.ts +++ b/server/src/services/native-runtime/provider-pack-upload.ts @@ -7,6 +7,9 @@ const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; /** Upload a pack and verify the installed bytes before exposing it to a provider. */ export async function uploadVerifiedProviderPack(input: { root: string; + /** Identity of the controller pack at `root`. */ + sourceDigest: string; + /** Identity selected for this execution; a retained session can pin an older release. */ expectedDigest: string; runner: CommandManagedRuntimeRunner; cwd: string; @@ -17,6 +20,13 @@ export async function uploadVerifiedProviderPack(input: { stageDirectory: () => Promise; verifyUploaded: () => Promise; }): Promise { + // The controller can only stage the pack it ships. A retained continuation + // pinned to an earlier release's pack cannot be satisfied by these bytes, by + // either transport. Fail before clearing the remote pack or uploading. + if (input.sourceDigest !== input.expectedDigest) { + const short = (digest: string) => digest.replace(/^sha256:/, "").slice(0, 12); + throw new Error(`runner_remote_provider_artifact_incompatible: selected provider pack ${short(input.expectedDigest)} is not the controller's pack ${short(input.sourceDigest)}; install the matching Paperclip package and Daytona image`); + } const measure = (name: string, fn: () => Promise) => input.trace ? input.trace.measure(name, fn) : fn(); const verifyDirectory = () => measure("provider_pack.directory.verify", async () => input.verifyDirectory()); // Archive installation reads the sealed archives, not the expanded tree.