diff --git a/doc/runner-api-tools.md b/doc/runner-api-tools.md index b59a144367..fdf085f124 100644 --- a/doc/runner-api-tools.md +++ b/doc/runner-api-tools.md @@ -176,6 +176,52 @@ real PostgreSQL and filesystem operations. Filesystem fault tests exercise the actual command with simulated provider transport; they do not establish a live Daytona RPC or deployed recovery. + +### Image generation for OpenAI API-key Codex agents + +Codex shows its built-in `image_gen` tool only when it signs in with ChatGPT. +A Codex agent that uses an OpenAI API key does not get that tool. The runner +also keeps the provider key out of the agent's shell, so the imagegen skill's +`scripts/image_gen.py` fallback cannot run. + +For these agents, the controller advertises `generate_image`. The tool is +advertised only when all of these conditions are true: + +- The run is a native Codex standard-mode run. +- The run was admitted on an OpenAI `api_key` connection with no custom routing. +- The run has a workspace. + +Dot, Muse, Grok, native reviews and plugin-owned tasks do not get the tool. + +The server reads the key from that same connection on each call. It sends the +key only to `https://api.openai.com/v1/images/generations` (model +`gpt-image-2`). The key never enters the provider process, the execution +command or the tool result. + +Every call does these checks again before any spend: + +- The live run, task and budget authority. +- `issue:mutate` permission. +- The connection grant, audience and agent installation. + +A budget hard-stop that pauses the agent refuses the call before OpenAI is +contacted. + +The PNG is saved as a new, create-only file in the workspace root: + +- Local hosts use symlink-confined opens. +- Remote targets use the same installer as `download_artifact`. + +The result returns the relative path, the size and the SHA-256. + +Each generated image records one `cost_events` row (`metered_api`, +`costStatus: "estimated"`) from a reviewed `gpt-image-2` rate card. The row is +recorded even when saving the image fails. Each call also writes +content-free `runner.image_generate` audit records. + +A per-call receipt in the run result makes a retried or recovered call replay +the first result instead of paying again. + ### Media files and upload limits The 1 GiB limit covers new snapshots returned by `call_api`, such as large JSON diff --git a/packages/paperclip-runner/src/drivers/codex/codex-api-key-image-tools.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-api-key-image-tools.test.ts new file mode 100644 index 0000000000..33e31140fe --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-api-key-image-tools.test.ts @@ -0,0 +1,97 @@ +import { spawn } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createServer, type IncomingMessage } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import { describe, expect, it } from "vitest"; +import { resolvePinnedCodexCommand } from "./codex-command.js"; +import { createIsolatedCodexAppServerArgs, createSecuredCodexThreadParams } from "./codex-security-config.js"; +import { WORKSPACE } from "./codex-app-server-driver.test-support.js"; + +/** + * N-12 (walk step 26): what an OpenAI API-key Codex agent actually gets to call. + * Drives the pinned Codex app-server with the runner's isolated launch and the + * API-key auth file the runner writes, against a local Responses endpoint, and + * reads the tool list from the first model request. + */ +describe("Codex API-key image tools (pinned binary)", () => { + it("hides Codex's built-in image_gen under API-key auth, so Paperclip's generate_image is the image path", async () => { + const requests: string[] = []; + const server = createServer((req: IncomingMessage, res) => { + const chunks: Buffer[] = []; + req.on("data", chunk => chunks.push(chunk)); + req.on("end", () => { + if (req.method === "POST" && req.url?.startsWith("/v1/responses")) requests.push(Buffer.concat(chunks).toString("utf8")); + // 426 makes Codex fall back from its Responses websocket to HTTP. + res.writeHead(req.method === "GET" && req.url?.startsWith("/v1/responses") ? 426 : 400, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: "test endpoint", type: "invalid_request_error" } })); + }); + }); + await new Promise(done => server.listen(0, "127.0.0.1", done)); + const port = (server.address() as { port: number }).port; + const codexHome = mkdtempSync(join(tmpdir(), "paperclip-n12-api-key-home-")); + // The same files prepareIsolatedCodexHome writes for an API-key agent. + writeFileSync(join(codexHome, "config.toml"), 'cli_auth_credentials_store = "file"\n[features]\nshell_snapshot = false\n'); + writeFileSync(join(codexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "sk-test-not-a-real-key" }), { mode: 0o600 }); + // Linux hosts run Codex inside an outer runner sandbox (the server sets + // this launch mode), and CI containers cannot create bwrap namespaces. + const source: NodeJS.ProcessEnv = { PATH: process.env.PATH, HOME: codexHome, CODEX_HOME: codexHome, + ...(process.platform === "linux" ? { PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: "1" } : {}) }; + const launch = createIsolatedCodexAppServerArgs(source); + const child = spawn(resolvePinnedCodexCommand(), [...launch.slice(0, -1), + "-c", `openai_base_url="http://127.0.0.1:${port}/v1"`, "-c", `chatgpt_base_url="http://127.0.0.1:${port}/backend-api"`, launch.at(-1)!], { + env: source, stdio: ["pipe", "pipe", "ignore"], + }); + try { + let nextId = 0; + const pending = new Map) => void>(); + createInterface({ input: child.stdout! }).on("line", line => { + try { + const message = JSON.parse(line) as Record; + if (typeof message.id === "number" && pending.has(message.id)) { pending.get(message.id)!(message); pending.delete(message.id); } + } catch { /* notifications and logs */ } + }); + const call = (method: string, params: unknown) => new Promise>(done => { + const id = ++nextId; + pending.set(id, done); + child.stdin!.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`); + }); + await call("initialize", { clientInfo: { name: "paperclip-runner", title: "Paperclip Runner", version: "test" }, capabilities: { experimentalApi: true } }); + child.stdin!.write(`${JSON.stringify({ jsonrpc: "2.0", method: "initialized" })}\n`); + const thread = await call("thread/start", { + ...createSecuredCodexThreadParams(WORKSPACE, "default", true, false, source), + model: "gpt-5.6-sol", + dynamicTools: [{ name: "generate_image", description: "Generate a PNG with the agent's existing OpenAI connection.", + inputSchema: { type: "object", properties: { prompt: { type: "string" } }, required: ["prompt"] } }], + }); + expect(thread.error, JSON.stringify(thread.error)).toBeUndefined(); + const threadId = String((thread.result as { thread: { id: string } }).thread.id); + await call("turn/start", { threadId, input: [{ type: "text", text: "Make a cake logo image." }] }); + const deadline = Date.now() + 30_000; + while (!requests.length && Date.now() < deadline) await new Promise(done => setTimeout(done, 200)); + expect(requests.length).toBeGreaterThan(0); + const body = JSON.parse(requests[0]!) as { tools?: Array>; input: Array> }; + const names: string[] = []; + for (const tool of body.tools ?? []) names.push(String(tool.name ?? tool.type)); + for (const item of body.input) { + if (item.type !== "additional_tools") continue; + for (const namespace of item.tools as Array>) { + for (const tool of (namespace.tools as Array> | undefined) ?? [namespace]) { + names.push(`${String(namespace.name ?? namespace.type)}.${String(tool.name ?? tool.type)}`); + } + } + } + // Codex gates image_gen on ChatGPT/Codex-backend auth, so an API key never gets it. + expect(names.some(name => /image_gen|imagegen|image_generation/.test(name))).toBe(false); + // Paperclip's server-side tool does reach the model on this path. + expect(JSON.stringify(body)).toContain("generate_image"); + } finally { + const exited = child.exitCode !== null || child.signalCode !== null ? Promise.resolve() : new Promise(done => child.once("exit", done)); + child.kill("SIGKILL"); + await exited; + server.close(); + rmSync(codexHome, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }, 90_000); +}); diff --git a/server/src/services/ai-connections.ts b/server/src/services/ai-connections.ts index 16086ebaee..25b812ffc3 100644 --- a/server/src/services/ai-connections.ts +++ b/server/src/services/ai-connections.ts @@ -456,6 +456,42 @@ export function aiConnectionService(db: Db) { } return { ...row, provider }; } + /** + * A native run's image tool reuses the exact AI connection the run was + * admitted with. Recheck the same permissions as runtime selection on every + * call: an OpenAI API key sent directly to OpenAI, active and healthy, shared + * with the responsible user and installed for this agent. + */ + async function selectImageGeneration(input: { + companyId: string; connectionId: string; grantId: string; + userId: string | null; agentId: string; + }) { + const [row] = await db.select({ connection: toolConnections, grant: connectionGrants }) + .from(toolConnections).innerJoin(connectionGrants, and( + eq(connectionGrants.connectionId, toolConnections.id), eq(connectionGrants.companyId, toolConnections.companyId), + )).where(and(eq(toolConnections.companyId, input.companyId), eq(toolConnections.id, input.connectionId), + eq(connectionGrants.id, input.grantId), eq(toolConnections.connectionPurpose, "ai"))).limit(1); + if (!row) throw unprocessable("AI connection unavailable", { code: "connection_unavailable" }); + const metadata = aiConnectionMetadataSchema.safeParse(row.connection.config.ai); + if (!metadata.success || metadata.data.provider !== "openai" || metadata.data.method !== "api_key" || metadata.data.routing) + throw unprocessable("Image generation needs an OpenAI API-key connection", { code: "incompatible_connection" }); + if (row.grant.status !== "active" || !row.connection.enabled || row.connection.status !== "active" || row.connection.healthStatus !== "ok") + throw unprocessable("Reconnect or enable this connection", { code: "connection_unavailable" }); + if (!input.userId || !(await membership(input.companyId, input.userId))) throw forbidden("Active responsible user required"); + if (row.grant.kind === "user" && !(await membership(input.companyId, row.grant.subjectUserId))) + throw forbidden("The account owner is no longer an active company member"); + const audience = await db.select().from(connectionGrantMembers).where(and( + eq(connectionGrantMembers.companyId, input.companyId), eq(connectionGrantMembers.grantId, input.grantId), + )); + if (!canUseCredential(row.grant, input.userId, audience)) throw forbidden("Connection is not shared with this user"); + const [install] = await db.select({ id: toolConnectionInstalls.id }).from(toolConnectionInstalls).where(and( + eq(toolConnectionInstalls.companyId, input.companyId), eq(toolConnectionInstalls.connectionId, input.connectionId), + or(and(eq(toolConnectionInstalls.targetType, "company"), eq(toolConnectionInstalls.targetId, input.companyId)), + and(eq(toolConnectionInstalls.targetType, "agent"), eq(toolConnectionInstalls.targetId, input.agentId))), + )).limit(1); + if (!install) throw forbidden("Connection is not permitted for this agent"); + return row; + } /** Internal fast-response use has its own capability check, never a pretend CLI harness. */ async function selectFastResponse(input: { companyId: string; connectionId: string; grantId: string; @@ -1233,5 +1269,5 @@ export function aiConnectionService(db: Db) { return [{ ...row, summary }]; }); } - return { list, selectDecision, selectFastResponse, selectPromptInjectionPrevention, quotaAccounts, subscriptionAccounts: (companyId: string, userId: string) => quotaAccounts(companyId, userId, true), refreshQuotaCredential, select, credential, runtimeCredential, probeUsage, accountIdentity, save, setDefault, membership, markAuthenticationFailed }; + return { list, selectDecision, selectFastResponse, selectImageGeneration, selectPromptInjectionPrevention, quotaAccounts, subscriptionAccounts: (companyId: string, userId: string) => quotaAccounts(companyId, userId, true), refreshQuotaCredential, select, credential, runtimeCredential, probeUsage, accountIdentity, save, setDefault, membership, markAuthenticationFailed }; } diff --git a/server/src/services/native-runtime/generate-image-tool.test.ts b/server/src/services/native-runtime/generate-image-tool.test.ts new file mode 100644 index 0000000000..e3120bd446 --- /dev/null +++ b/server/src/services/native-runtime/generate-image-tool.test.ts @@ -0,0 +1,335 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdtemp, readFile, readdir, realpath, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { and, eq, sql } from "drizzle-orm"; +import { activityLog, agents, authUsers, budgetPolicies, companies, companyMemberships, costEvents, createDb, heartbeatRuns, issues, projects } from "@paperclipai/db"; +import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; +import { aiConnectionService } from "../ai-connections.js"; +import { accountRunCost } from "../run-cost-accounting.js"; +import { budgetServiceInTransaction } from "../budgets.js"; +import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; +import { imageGenerationBindingForRun, priceGeneratedImage, requestOpenAiImage } from "./generate-image-tool.js"; + +// One transparent 1x1 PNG. The handler only needs real PNG bytes to save. +const PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", "base64"); +const OPENAI_KEY = "sk-agent-existing-provider-key"; + +describe("generate_image for OpenAI API-key Codex agents (N-12)", () => { + let temporary: Awaited> | null = null; + let db: ReturnType; + let home: string; + const workspaces: string[] = []; + + beforeAll(async () => { + home = await realpath(await mkdtemp(path.join(os.tmpdir(), "paperclip-image-tool-home-"))); + vi.stubEnv("PAPERCLIP_HOME", home); + vi.stubEnv("PAPERCLIP_INSTANCE_ID", "image-tool-fixture"); + temporary = await startRecoveryPostgres("paperclip-image-tool-"); + db = createDb(temporary.connectionString); + await db.insert(authUsers).values({ id: "alice", name: "Alice", email: "alice@example.test", createdAt: new Date(), updatedAt: new Date() }); + }, 90_000); + afterEach(() => vi.unstubAllGlobals()); + afterAll(async () => { + await db?.$client.end({ timeout: 1 }); + await temporary?.cleanup(); + for (const workspace of workspaces) await rm(workspace, { recursive: true, force: true }); + await rm(home, { recursive: true, force: true }); + vi.unstubAllEnvs(); + }); + + /** An onboarding-style company: one Codex agent running on an OpenAI API key. */ + async function fixture() { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Pixel Bakery", issuePrefix: `PX${companyId.slice(0, 4).toUpperCase()}` }); + await db.insert(companyMemberships).values({ companyId, principalId: "alice", principalType: "user", status: "active", membershipRole: "owner" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Head baker", adapterType: "paperclip_runner", + adapterConfig: { provider: "codex" }, status: "active" }); + const connection = await aiConnectionService(db).save(companyId, "alice", { + provider: "openai", method: "api_key", name: "OpenAI", ownership: "shared", + apiKey: OPENAI_KEY, agentIds: [], allAgents: true, + } as never, OPENAI_KEY); + await db.insert(issues).values({ id: issueId, companyId, title: "Create printable cake menu and logo", + status: "in_progress", workMode: "standard", assigneeAgentId: agentId }); + const aiConnection = { connectionId: connection.connectionId, grantId: connection.grantId, + provider: "openai", method: "api_key", mode: "shared", responsibleUserId: "alice" }; + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", + nativeIssueId: issueId, invocationSource: "assignment", triggerDetail: "system", responsibleUserId: "alice", + contextSnapshot: { issueId, aiConnection } }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + const workspaceRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), "paperclip-image-tool-ws-"))); + workspaces.push(workspaceRoot); + const binding = { companyId, agentId, issueId, runId, workspaceRoot, + imageGeneration: { connectionId: connection.connectionId, grantId: connection.grantId } }; + return { ...binding, binding, aiConnection }; + } + + function stubOpenAiImages() { + const fetchMock = vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => new Response(JSON.stringify({ + created: 1, data: [{ b64_json: PNG.toString("base64") }], + usage: { input_tokens: 50, output_tokens: 1000, input_tokens_details: { text_tokens: 50, image_tokens: 0 } }, + }), { status: 200, headers: { "content-type": "application/json", "x-request-id": "req_image_1" } })); + vi.stubGlobal("fetch", fetchMock); + return fetchMock; + } + + it("lets an API-key Codex agent make an image with its existing OpenAI key, saved in its workspace and billed", async () => { + const f = await fixture(); + const fetchMock = stubOpenAiImages(); + const authority = new PaperclipRunnerToolAuthority(db, f.binding as never); + + expect(authority.definitions().map(tool => tool.name)).toContain("generate_image"); + + const callId = randomUUID(); + const result = await authority.execute({ tool: "generate_image", callId, + arguments: { prompt: "A warm, playful cake logo for Pixel Bakery", name: "pixel-bakery-logo" } }) as Record; + + // The trusted server made the call with the agent's existing key; no second key. + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0]!; + expect(String(url)).toBe("https://api.openai.com/v1/images/generations"); + expect(new Headers(init?.headers).get("authorization")).toBe(`Bearer ${OPENAI_KEY}`); + expect(JSON.parse(String(init?.body))).toMatchObject({ model: "gpt-image-2", prompt: "A warm, playful cake logo for Pixel Bakery" }); + + // The PNG lands in the workspace under the returned relative path. + expect(result).toMatchObject({ outcome: "completed", contentType: "image/png", byteSize: PNG.length, + sha256: createHash("sha256").update(PNG).digest("hex") }); + expect(String(result.path)).toMatch(/^pixel-bakery-logo-[a-f0-9]{8}\.png$/); + expect(await readFile(path.join(f.workspaceRoot, String(result.path)))).toEqual(PNG); + expect(JSON.stringify(result)).not.toContain(OPENAI_KEY); + + // Spend is recorded on the budget ledger for this agent, run and task. + const costs = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.runId)); + expect(costs).toHaveLength(1); + expect(costs[0]).toMatchObject({ agentId: f.agentId, issueId: f.issueId, provider: "openai", biller: "openai", + billingType: "metered_api", model: "gpt-image-2", costStatus: "estimated", inputTokens: 50, outputTokens: 1000, + providerRequestId: "req_image_1" }); + // 50 text tokens at $5/M + 1000 image output tokens at $30/M = $0.03025. + expect(Number(costs[0]!.costCents)).toBeCloseTo(3.025, 6); + + const activity = await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.companyId), eq(activityLog.action, "runner.image_generate"))); + expect(activity.map(row => (row.details as Record).outcome)).toEqual(expect.arrayContaining(["started", "completed"])); + expect(JSON.stringify(activity)).not.toContain(OPENAI_KEY); + + // A provider retry of the same call replays the receipt instead of paying again. + const replay = await authority.execute({ tool: "generate_image", callId, + arguments: { prompt: "A warm, playful cake logo for Pixel Bakery", name: "pixel-bakery-logo" } }); + expect(replay).toEqual(result); + expect(await authority.recover({ tool: "generate_image", callId, + arguments: { prompt: "A warm, playful cake logo for Pixel Bakery", name: "pixel-bakery-logo" } })).toEqual({ result }); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.runId))).toHaveLength(1); + }); + + it("is not offered without a server-verified OpenAI API-key binding", async () => { + const f = await fixture(); + const { imageGeneration: _omitted, ...withoutBinding } = f.binding; + const authority = new PaperclipRunnerToolAuthority(db, withoutBinding); + expect(authority.definitions().map(tool => tool.name)).not.toContain("generate_image"); + await expect(authority.execute({ tool: "generate_image", callId: randomUUID(), arguments: { prompt: "logo" } })) + .rejects.toThrow(); + }); + + it("refuses before calling OpenAI once a budget hard-stop has paused the agent", async () => { + const f = await fixture(); + const fetchMock = stubOpenAiImages(); + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.agentId)); + const authority = new PaperclipRunnerToolAuthority(db, f.binding as never); + await expect(authority.execute({ tool: "generate_image", callId: randomUUID(), arguments: { prompt: "logo" } })) + .rejects.toThrow(); + expect(fetchMock).not.toHaveBeenCalled(); + expect(await readdir(f.workspaceRoot)).toEqual([]); + }); + + it("still records the spend when a paid image cannot be saved", async () => { + const f = await fixture(); + const fetchMock = stubOpenAiImages(); + await rm(f.workspaceRoot, { recursive: true, force: true }); + const authority = new PaperclipRunnerToolAuthority(db, f.binding as never); + const callId = randomUUID(); + await expect(authority.execute({ tool: "generate_image", callId, arguments: { prompt: "logo" } })).rejects.toThrow(); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.runId))).toHaveLength(1); + // The failed call is settled: a retry with the same call ID neither pays nor claims success. + await expect(authority.execute({ tool: "generate_image", callId, arguments: { prompt: "logo" } })) + .rejects.toThrow("could not be saved"); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("refuses before calling OpenAI when the task's project budget has stopped work", async () => { + const f = await fixture(); + const fetchMock = stubOpenAiImages(); + const [project] = await db.insert(projects).values({ companyId: f.companyId, name: "Menu", pausedAt: new Date(), pauseReason: "budget" }).returning(); + await db.update(issues).set({ projectId: project!.id }).where(eq(issues.id, f.issueId)); + const authority = new PaperclipRunnerToolAuthority(db, f.binding as never); + await expect(authority.execute({ tool: "generate_image", callId: randomUUID(), arguments: { prompt: "logo" } })) + .rejects.toThrow("budget"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("settles a paid image whose run stopped before the charge was recorded, without buying another", async () => { + const f = await fixture(); + const fetchMock = stubOpenAiImages(); + const authority = new PaperclipRunnerToolAuthority(db, f.binding as never); + const callId = randomUUID(); + const input = { prompt: "logo", size: "1024x1024", quality: "auto" }; + // The state a crash leaves after OpenAI charged but before accounting committed. + const key = createHash("sha256").update(callId).digest("hex"); + const digest = createHash("sha256").update(JSON.stringify({ prompt: "logo", quality: "auto", size: "1024x1024" })).digest("hex"); + await db.update(heartbeatRuns).set({ resultJson: { imageToolReceipts: { [key]: { digest, state: "charged", charge: { + model: "gpt-image-2", providerRequestId: "req_lost", usage: { textInputTokens: 10, imageInputTokens: 0, outputTokens: 100 }, + costCents: "0.3050000", pricingProvenance: { source: "rate_card", version: "openai-image-standard-2026-11-06" }, + projectId: null, destination: "generated-image-0a1b2c3d.png", byteSize: PNG.length, sha256: "0".repeat(64), + chargedAt: "2026-11-06T12:00:00.000Z" } } } } }) + .where(eq(heartbeatRuns.id, f.runId)); + expect(await authority.recover({ tool: "generate_image", callId, arguments: input })).toBeNull(); + await expect(authority.execute({ tool: "generate_image", callId, arguments: input })).rejects.toThrow("generated-image-0a1b2c3d.png"); + const costs = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.runId)); + expect(costs).toHaveLength(1); + expect(costs[0]).toMatchObject({ providerRequestId: "req_lost", outputTokens: 100 }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("run accounting settles a paid image that no tool call settled", async () => { + const f = await fixture(); + const key = createHash("sha256").update(randomUUID()).digest("hex"); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), costAccountingPending: true, + resultJson: { imageToolReceipts: { [key]: { digest: "d", state: "charged", charge: { + model: "gpt-image-2", providerRequestId: "req_unsettled", usage: { textInputTokens: 10, imageInputTokens: 0, outputTokens: 100 }, + costCents: "0.3050000", pricingProvenance: { source: "rate_card", version: "openai-image-standard-2026-11-06" }, + projectId: null, destination: "generated-image-0a1b2c3d.png", byteSize: PNG.length, sha256: "0".repeat(64), + chargedAt: "2026-11-06T12:00:00.000Z" } } } } }) + .where(eq(heartbeatRuns.id, f.runId)); + await accountRunCost(db, f.runId); + const costs = await db.select().from(costEvents).where(and(eq(costEvents.heartbeatRunId, f.runId), eq(costEvents.providerRequestId, "req_unsettled"))); + expect(costs).toHaveLength(1); + expect(costs[0]).toMatchObject({ agentId: f.agentId, issueId: f.issueId, model: "gpt-image-2", outputTokens: 100, costStatus: "estimated" }); + // The task is deleted while the receipt is still `charged`; later accounting must not conflict. + await db.update(heartbeatRuns).set({ costAccountingPending: true, costAccountedAt: null, nativeIssueId: null }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(costEvents).set({ issueId: null }).where(eq(costEvents.id, costs[0]!.id)); + await expect(accountRunCost(db, f.runId)).resolves.not.toThrow(); + expect(await db.select().from(costEvents).where(and(eq(costEvents.heartbeatRunId, f.runId), eq(costEvents.providerRequestId, "req_unsettled")))).toHaveLength(1); + }); + + it("records unknown spend when OpenAI's response is lost after the request was sent", async () => { + const f = await fixture(); + const fetchMock = vi.fn(async () => { throw new TypeError("fetch failed: socket hang up"); }); + vi.stubGlobal("fetch", fetchMock); + const authority = new PaperclipRunnerToolAuthority(db, f.binding as never); + const callId = randomUUID(); + await expect(authority.execute({ tool: "generate_image", callId, arguments: { prompt: "logo" } })).rejects.toThrow("may have been billed"); + const costs = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.runId)); + expect(costs).toHaveLength(1); + expect(costs[0]).toMatchObject({ provider: "openai", model: "gpt-image-2", costStatus: "unpriced", billingType: "unknown" }); + await expect(authority.execute({ tool: "generate_image", callId, arguments: { prompt: "logo" } })).rejects.toThrow("may have been billed"); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.runId))).toHaveLength(1); + // A budget that blocks unpriced spend now holds further paid work. + await db.insert(budgetPolicies).values({ companyId: f.companyId, scopeType: "agent", scopeId: f.agentId, windowKind: "calendar_month_utc", + amount: 100_000, unpricedUsagePolicy: "block" }); + expect(await budgetServiceInTransaction(db).getInvocationBlock(f.companyId, f.agentId, { issueId: f.issueId })).toMatchObject({ + reason: expect.stringContaining("no reliable price") }); + }); + + it("records no spend when OpenAI rejects the request", async () => { + const f = await fixture(); + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ error: { message: "Your request was rejected by the safety system" } }), { status: 400 }))); + const authority = new PaperclipRunnerToolAuthority(db, f.binding as never); + await expect(authority.execute({ tool: "generate_image", callId: randomUUID(), arguments: { prompt: "logo" } })).rejects.toThrow("safety system"); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.runId))).toHaveLength(0); + }); + + it("does not call OpenAI when Stop commits after the first authority check", async () => { + const f = await fixture(); + const fetchMock = stubOpenAiImages(); + // Stop lands while the handler writes its receipt and audit entry. + await db.execute(sql.raw(`create or replace function n12_stop_on_image_start() returns trigger language plpgsql as $$ + begin + if new.action = 'runner.image_generate' and new.details->>'outcome' = 'started' then + update agents set status = 'paused' where id = new.actor_id::uuid; + end if; + return new; + end $$; + create trigger n12_stop_on_image_start after insert on activity_log for each row execute function n12_stop_on_image_start();`)); + try { + const authority = new PaperclipRunnerToolAuthority(db, f.binding as never); + await expect(authority.execute({ tool: "generate_image", callId: randomUUID(), arguments: { prompt: "logo" } })).rejects.toThrow(); + expect(fetchMock).not.toHaveBeenCalled(); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.runId))).toHaveLength(0); + } finally { + await db.execute(sql.raw("drop trigger if exists n12_stop_on_image_start on activity_log; drop function if exists n12_stop_on_image_start();")); + } + }); + + it("does not call OpenAI when the connection grant is revoked after the first check", async () => { + const f = await fixture(); + const fetchMock = stubOpenAiImages(); + await db.execute(sql.raw(`create or replace function n12_revoke_on_image_start() returns trigger language plpgsql as $$ + begin + if new.action = 'runner.image_generate' and new.details->>'outcome' = 'started' then + update connection_grants set status = 'revoked' where id = '${f.binding.imageGeneration.grantId}'; + end if; + return new; + end $$; + create trigger n12_revoke_on_image_start after insert on activity_log for each row execute function n12_revoke_on_image_start();`)); + try { + const authority = new PaperclipRunnerToolAuthority(db, f.binding as never); + await expect(authority.execute({ tool: "generate_image", callId: randomUUID(), arguments: { prompt: "logo" } })).rejects.toThrow(); + expect(fetchMock).not.toHaveBeenCalled(); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.runId))).toHaveLength(0); + } finally { + await db.execute(sql.raw("drop trigger if exists n12_revoke_on_image_start on activity_log; drop function if exists n12_revoke_on_image_start();")); + } + }); + + it("refuses when the run's AI connection is no longer an active OpenAI API key", async () => { + const f = await fixture(); + const fetchMock = stubOpenAiImages(); + const authority = new PaperclipRunnerToolAuthority(db, { ...f.binding, + imageGeneration: { connectionId: f.binding.imageGeneration.connectionId, grantId: randomUUID() } } as never); + await expect(authority.execute({ tool: "generate_image", callId: randomUUID(), arguments: { prompt: "logo" } })) + .rejects.toThrow(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe("generate_image helpers", () => { + const aiConnection = { connectionId: "c1", grantId: "g1", provider: "openai", method: "api_key" }; + + it("binds only native Codex runs on a direct OpenAI API-key connection", () => { + expect(imageGenerationBindingForRun({ providerKind: "codex", contextSnapshot: { aiConnection } })) + .toEqual({ connectionId: "c1", grantId: "g1" }); + // ChatGPT sign-in keeps Codex's own image tool; other harnesses and routed keys get nothing. + expect(imageGenerationBindingForRun({ providerKind: "codex", contextSnapshot: { aiConnection: { ...aiConnection, method: "subscription" } } })).toBeUndefined(); + expect(imageGenerationBindingForRun({ providerKind: "acpx", contextSnapshot: { aiConnection } })).toBeUndefined(); + expect(imageGenerationBindingForRun({ providerKind: "codex", contextSnapshot: { aiConnection: { ...aiConnection, provider: "openrouter" } } })).toBeUndefined(); + expect(imageGenerationBindingForRun({ providerKind: "codex", contextSnapshot: { aiConnection, aiRouterSelection: { model: "x" } } })).toBeUndefined(); + expect(imageGenerationBindingForRun({ providerKind: "codex", contextSnapshot: {} })).toBeUndefined(); + }); + + it("prices gpt-image-2 from the reviewed rate card", () => { + const price = priceGeneratedImage({ bytes: PNG, model: "gpt-image-2", providerRequestId: null, + usage: { textInputTokens: 1_000_000, imageInputTokens: 1_000_000, outputTokens: 1_000_000 } }); + expect(price.costUsd).toBe("43.000000000"); + expect(price.pricingProvenance).toMatchObject({ source: "rate_card", version: "openai-image-standard-2026-11-06" }); + }); + + it("stops reading an oversized provider reply before parsing it", async () => { + const huge = vi.fn(async () => new Response(new ReadableStream({ start(controller) { + for (let i = 0; i < 8; i++) controller.enqueue(new Uint8Array(1024)); + controller.close(); + } }), { status: 200 })); + await expect(requestOpenAiImage({ prompt: "logo", size: "1024x1024", quality: "auto", apiKey: OPENAI_KEY }, huge as never, { maxResponseBytes: 4096 })) + .rejects.toThrow("too large"); + }); + + it("reports provider errors without echoing the key and rejects non-PNG data", async () => { + const failing = vi.fn(async () => new Response(JSON.stringify({ error: { message: "Billing hard limit reached" } }), { status: 400 })); + await expect(requestOpenAiImage({ prompt: "logo", size: "1024x1024", quality: "auto", apiKey: OPENAI_KEY }, failing as never)) + .rejects.toThrow("OpenAI image generation failed: Billing hard limit reached"); + const notPng = vi.fn(async () => new Response(JSON.stringify({ data: [{ b64_json: Buffer.from("").toString("base64") }] }), { status: 200 })); + await expect(requestOpenAiImage({ prompt: "logo", size: "1024x1024", quality: "auto", apiKey: OPENAI_KEY }, notPng as never)) + .rejects.toThrow("invalid PNG"); + }); +}); diff --git a/server/src/services/native-runtime/generate-image-tool.ts b/server/src/services/native-runtime/generate-image-tool.ts new file mode 100644 index 0000000000..ae735acc3b --- /dev/null +++ b/server/src/services/native-runtime/generate-image-tool.ts @@ -0,0 +1,247 @@ +import { createHash, randomBytes } from "node:crypto"; +import { constants } from "node:fs"; +import { realpath } from "node:fs/promises"; +import { join } from "node:path"; +import { z } from "zod"; +import { centsToUsd, unitsToCents, usdToUnits } from "@paperclipai/shared"; +import { and, eq } from "drizzle-orm"; +import { costEvents, issues, projects, type Db } from "@paperclipai/db"; +import type { ActivityPublication } from "../activity-log.js"; +import { createCostEventInTransaction } from "../costs.js"; +import { openRunnerWorkspaceFile } from "./runner-api-files.js"; + +/** + * Codex hides its built-in image tool unless it is signed in with ChatGPT + * (codex-rs core/src/tools/spec_plan.rs `image_generation_available`), and the + * runner keeps the provider key out of the agent's shell. For an OpenAI API-key + * agent this server-side tool makes the image with that existing key instead. + */ +export const GENERATE_IMAGE = "generate_image"; +export const GENERATE_IMAGE_MODEL = "gpt-image-2"; +const OPENAI_IMAGES_URL = "https://api.openai.com/v1/images/generations"; +const MAX_IMAGE_BYTES = 20 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 180_000; +// Base64 of the largest accepted PNG plus room for the JSON envelope. +const MAX_RESPONSE_BYTES = Math.ceil(MAX_IMAGE_BYTES / 3) * 4 + 64 * 1024; + +export const generateImageSchema = z.object({ + prompt: z.string().trim().min(1).max(8000).describe("What to draw. Include subject, style, colors and any exact text."), + size: z.enum(["1024x1024", "1536x1024", "1024x1536", "auto"]).default("1024x1024"), + quality: z.enum(["low", "medium", "high", "auto"]).default("auto"), + name: z.string().regex(/^[a-z0-9][a-z0-9-]{0,47}$/).optional() + .describe("Short lowercase file name stem, for example cake-logo. A unique suffix and .png are added."), +}).strict(); +export type GenerateImageInput = z.infer; + +export const GENERATE_IMAGE_DEFINITION = { + name: GENERATE_IMAGE, + description: "Generate a new raster image (PNG) with OpenAI's image model, using this agent's existing OpenAI connection. " + + "No extra API key is needed. Use this instead of the imagegen skill's built-in image_gen tool or scripts/image_gen.py, which are unavailable here. " + + "The image is saved as a new file in the workspace root and the result gives its relative path; move or copy it from there, " + + "and use register_deliverable if the user should receive it. Each call is billed to this agent's budget, so generate only what the task needs.", + inputSchema: z.toJSONSchema(generateImageSchema), +}; + +/** Only a native Codex run on an OpenAI API-key connection, without custom routing, gets the tool. */ +export function imageGenerationBindingForRun(input: { + providerKind: string; + contextSnapshot: unknown; +}): { connectionId: string; grantId: string } | undefined { + if (input.providerKind !== "codex") return undefined; + const snapshot = record(input.contextSnapshot); + if (snapshot.aiRouterSelection || snapshot.managedAiRouting) return undefined; + const connection = record(snapshot.aiConnection); + if (connection.provider !== "openai" || connection.method !== "api_key") return undefined; + if (typeof connection.connectionId !== "string" || typeof connection.grantId !== "string") return undefined; + return { connectionId: connection.connectionId, grantId: connection.grantId }; +} + +export type GeneratedImage = { + bytes: Buffer; + model: string; + providerRequestId: string | null; + usage: { textInputTokens: number; imageInputTokens: number; outputTokens: number }; +}; + +/** Calls the OpenAI Images API. The key is used only in this request header. */ +export async function requestOpenAiImage(input: GenerateImageInput & { apiKey: string }, fetchImpl: typeof fetch = fetch, + limits: { maxResponseBytes?: number } = {}): Promise { + const response = await fetchImpl(OPENAI_IMAGES_URL, { + method: "POST", + headers: { authorization: `Bearer ${input.apiKey}`, "content-type": "application/json" }, + body: JSON.stringify({ model: GENERATE_IMAGE_MODEL, prompt: input.prompt, size: input.size, quality: input.quality, n: 1 }), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + const text = await readBoundedText(response, limits.maxResponseBytes ?? MAX_RESPONSE_BYTES).catch((error: unknown) => { + if (!response.ok) throw new ImageGenerationProviderError(`OpenAI image generation failed: HTTP ${response.status}`, response.status, true); + throw error; + }); + let body: Record = {}; + try { body = record(JSON.parse(text)); } catch { /* reported below */ } + if (!response.ok) { + // Provider messages are safe to show; they never include the request key. + const message = typeof record(body.error).message === "string" ? String(record(body.error).message).slice(0, 500) : `HTTP ${response.status}`; + throw new ImageGenerationProviderError(`OpenAI image generation failed: ${message}`, response.status, true); + } + const first = record(Array.isArray(body.data) ? body.data[0] : undefined); + if (typeof first.b64_json !== "string") throw new ImageGenerationProviderError("OpenAI image generation returned no image", 502); + const bytes = Buffer.from(first.b64_json, "base64"); + if (bytes.length === 0 || bytes.length > MAX_IMAGE_BYTES || !bytes.subarray(0, 8).equals(PNG_SIGNATURE)) { + throw new ImageGenerationProviderError("OpenAI image generation returned an invalid PNG", 502); + } + const usage = record(body.usage); + const details = record(usage.input_tokens_details); + const inputTokens = count(usage.input_tokens); + const imageInputTokens = count(details.image_tokens); + return { + bytes, + model: GENERATE_IMAGE_MODEL, + providerRequestId: response.headers.get("x-request-id"), + usage: { + // Text tokens default to all input tokens that are not reported as image input. + textInputTokens: details.text_tokens === undefined ? Math.max(0, inputTokens - imageInputTokens) : count(details.text_tokens), + imageInputTokens, + outputTokens: count(usage.output_tokens), + }, + }; +} + +/** Reads at most `limit` bytes; a larger reply is cancelled before it is parsed or decoded. */ +async function readBoundedText(response: Response, limit: number): Promise { + const declared = Number(response.headers.get("content-length")); + if (Number.isFinite(declared) && declared > limit) { + await response.body?.cancel().catch(() => undefined); + throw new ImageGenerationProviderError("OpenAI image generation returned a response that is too large", 502); + } + if (!response.body) return ""; + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > limit) { + await reader.cancel().catch(() => undefined); + throw new ImageGenerationProviderError("OpenAI image generation returned a response that is too large", 502); + } + chunks.push(value); + } + return Buffer.concat(chunks).toString("utf8"); +} + +/** + * `rejected` is true only for an HTTP error response, where OpenAI did not + * generate (and does not bill) an image. Any other failure, including a lost + * or unreadable success response, may have been billed. + */ +export class ImageGenerationProviderError extends Error { + constructor(message: string, readonly status: number, readonly rejected = false) { super(message); } +} + +// Immutable, reviewed rate card (USD per 1M tokens, standard processing). +// https://developers.openai.com/api/docs/pricing (retrieved 2026-11-06) +const RATE_CARD_VERSION = "openai-image-standard-2026-11-06"; +const RATES = { [GENERATE_IMAGE_MODEL]: { textInput: "5", imageInput: "8", imageOutput: "30" } } as const; + +/** Token-price estimate for the budget ledger; not an invoice. Output is priced as image output. */ +export function priceGeneratedImage(image: GeneratedImage) { + const rates = RATES[image.model as keyof typeof RATES]; + const units = BigInt(image.usage.textInputTokens) * usdToUnits(rates.textInput) + + BigInt(image.usage.imageInputTokens) * usdToUnits(rates.imageInput) + + BigInt(image.usage.outputTokens) * usdToUnits(rates.imageOutput); + const costCents = unitsToCents((units + 500_000n) / 1_000_000n); + return { + costCents, + costUsd: centsToUsd(costCents), + pricingProvenance: { + source: "rate_card" as const, + version: RATE_CARD_VERSION, + evidence: "https://developers.openai.com/api/docs/pricing; image output rate applied to all output tokens", + inputCentsPerMillion: unitsToCents(usdToUnits(rates.textInput)), + outputCentsPerMillion: unitsToCents(usdToUnits(rates.imageOutput)), + serviceTier: "standard", + }, + }; +} + +/** + * The cost row for one paid image. The idempotency key and the stored charge + * time make every settlement of the same receipt record the charge once. + */ +export function imageChargeCostEvent(input: { agentId: string; issueId: string | null; runId: string; key: string; charge: Record }) { + const usage = record(input.charge.usage); + const chargedAt = typeof input.charge.chargedAt === "string" && !Number.isNaN(Date.parse(input.charge.chargedAt)) ? new Date(input.charge.chargedAt) : new Date(0); + return { + agentId: input.agentId, issueId: input.issueId, projectId: typeof input.charge.projectId === "string" ? input.charge.projectId : null, + // An uncertain charge has no usage. Billing type `unknown` keeps it from the + // zero-token metered exemption, so budgets that block unpriced spend see it. + heartbeatRunId: input.runId, provider: "openai", biller: "openai", + billingType: input.charge.costStatus === "unpriced" ? "unknown" as const : "metered_api" as const, model: String(input.charge.model), + costStatus: input.charge.costStatus === "unpriced" ? "unpriced" as const : "estimated" as const, inputTokens: count(usage.textInputTokens) + count(usage.imageInputTokens), outputTokens: count(usage.outputTokens), + costCents: String(input.charge.costCents), pricingProvenance: input.charge.pricingProvenance as never, + providerRequestId: typeof input.charge.providerRequestId === "string" ? input.charge.providerRequestId : null, + occurredAt: chargedAt, idempotencyKey: `image_gen:${input.runId}:${input.key}`, + }; +} + +/** + * Run accounting also settles paid images whose tool call never finished + * recording them, so a lost acknowledgement cannot drop the spend. + */ +export async function settleChargedImageReceiptsInTransaction(tx: Db, run: { id: string; companyId: string; agentId: string; nativeIssueId: string | null; resultJson: unknown }, + publications: ActivityPublication[]): Promise> { + const settled: Array = []; + for (const [key, value] of Object.entries(record(record(run.resultJson).imageToolReceipts))) { + const receipt = record(value); + if (receipt.state !== "charged" || !/^[a-f0-9]{64}$/.test(key)) continue; + const event = imageChargeCostEvent({ agentId: run.agentId, issueId: run.nativeIssueId, runId: run.id, key, charge: record(receipt.charge) }); + // Already recorded: its attribution may have changed since (a deleted task + // nulls cost rows), so never rebuild and compare a fresh receipt. + if ((await tx.select({ id: costEvents.id }).from(costEvents).where(and(eq(costEvents.companyId, run.companyId), + eq(costEvents.idempotencyKey, event.idempotencyKey))).limit(1))[0]) continue; + // Deleted attribution targets become unallocated; spend is still recorded. + if (event.issueId && !(await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.id, event.issueId), eq(issues.companyId, run.companyId))))[0]) event.issueId = null; + if (event.projectId && !(await tx.select({ id: projects.id }).from(projects).where(and(eq(projects.id, event.projectId), eq(projects.companyId, run.companyId))))[0]) event.projectId = null; + await createCostEventInTransaction(tx, run.companyId, event, publications, undefined, { deferBudgetEvaluation: true }); + settled.push(event.projectId); + } + return settled; +} + +/** A request OpenAI may have billed although no usable response arrived: spend is recorded as unpriced. */ +export function uncertainImageCharge(input: { projectId: string | null; reason: string }) { + return { + model: GENERATE_IMAGE_MODEL, providerRequestId: null, usage: { textInputTokens: 0, imageInputTokens: 0, outputTokens: 0 }, + costCents: "0", costStatus: "unpriced", projectId: input.projectId, chargedAt: new Date().toISOString(), + pricingProvenance: { source: "unknown" as const, evidence: `Image request outcome unknown: ${input.reason}`.slice(0, 1000) }, + }; +} + +/** Server-chosen, create-only destination in the workspace root. */ +export function generatedImageDestination(name: string | undefined): string { + return `${name ?? "generated-image"}-${randomBytes(4).toString("hex")}.png`; +} + +export function imageIdentity(bytes: Buffer) { + return { byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }; +} + +/** Local workspace write: symlink-confined, never overwrites an existing file. */ +export async function writeGeneratedImageLocally(workspaceRoot: string, destination: string, bytes: Buffer): Promise { + if (!/^[a-z0-9][a-z0-9-]{0,63}-[a-f0-9]{8}\.png$/.test(destination)) throw new Error("generated_image_path_denied"); + const target = join(await realpath(workspaceRoot), destination); + const file = await openRunnerWorkspaceFile(target, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o644); + try { + await file.writeFile(bytes); + await file.sync(); + } finally { await file.close(); } +} + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} +function count(value: unknown): number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 && value <= 2_147_483_647 ? value : 0; +} diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index be7d2a2775..19e6a761dd 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -97,6 +97,7 @@ import { resolveGitHubOperationCredentials } from "../github-operation-credentia import { bindManagedNativeCredentialTurn, completeManagedNativeCredentialTurn } from "./managed-native-credentials.js"; import { createLocalNativeQuestionBridge } from "./local-native-question-bridge.js"; import { createArtifactInstaller } from "./artifact-transfer.js"; +import { imageGenerationBindingForRun } from "./generate-image-tool.js"; import { readVerifiedRemoteWorkspaceFile } from "./remote-deliverable-file.js"; import { copyBackCodexAuth } from "@paperclipai/adapter-codex-local/server"; import { nativeCompletionFeedback } from "./native-completion-feedback.js"; @@ -14288,6 +14289,8 @@ async function createRunnerdBackendWithinSessionClaim( executionTargetKind: target.kind, installRemoteArtifact: !pluginTask && remoteTarget && remoteCommandRunner ? createArtifactInstaller(remoteTarget.remoteCwd, remoteCommandRunner) : undefined, + imageGeneration: nativeReview || pluginTask ? undefined + : imageGenerationBindingForRun({ providerKind: input.execution.provider.kind, contextSnapshot: reviewRun?.contextSnapshot }), readRemoteWorkspaceFile: !pluginTask && remoteTarget && remoteCommandRunner ? (file) => readVerifiedRemoteWorkspaceFile({ runner: remoteCommandRunner, workspaceRoot: remoteTarget.remoteCwd, ...file }) : undefined, diff --git a/server/src/services/native-runtime/native-session-resume.ts b/server/src/services/native-runtime/native-session-resume.ts index 08d2e9287e..2a2155d9d7 100644 --- a/server/src/services/native-runtime/native-session-resume.ts +++ b/server/src/services/native-runtime/native-session-resume.ts @@ -36,7 +36,7 @@ export function nativeToolContractFingerprintForTarget( return `sha256:${createHash("sha256") .update( JSON.stringify({ - schema: "paperclip.native-tool-contract.v18", + schema: "paperclip.native-tool-contract.v19", executionTargetKind, advertisementPolicy: { // Direct provider threads retain declarations from thread/start. @@ -46,6 +46,9 @@ export function nativeToolContractFingerprintForTarget( historicalChatAttachments: "always_advertised_conversation_binding_gated.v1", registerDeliverable: "verified_local_or_remote_workspace.v2", + // Advertised only to Codex runs admitted on an OpenAI API-key + // connection; the key stays in the server. + generateImage: "codex_openai_api_key_connection_server_key.v1", readChatAttachment: "always_advertised_run_scope_local_staging.v2", structuredHumanInput: "always_advertised_run_issue_agent_binding_gated_current_task_description.v2", @@ -63,6 +66,7 @@ export function nativeToolContractFingerprintForTarget( }, tools: [ { name: "register_deliverable", version: 2 }, + { name: "generate_image", semanticContract: "paperclip.server-openai-image-generation.v1", version: 1 }, { name: "read_current_wake_comments", semanticContract: "paperclip.server-current-wake-comments.v1", diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index 696f74781d..5417f052b4 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -2,6 +2,10 @@ import { budgetServiceInTransaction } from "../budgets.js"; import { resolveTaskWatchdogMutationScope, taskWatchdogScopeAllowsIssueMutation } from "../task-watchdog-scope.js"; import { resolveNewTaskParent } from "../issue-hierarchy.js"; import { DOWNLOAD_ARTIFACT, DOWNLOAD_ARTIFACT_DEFINITION, downloadArtifactSchema, createArtifactInstaller, type ArtifactInstaller } from "./artifact-transfer.js"; +import { GENERATE_IMAGE, GENERATE_IMAGE_DEFINITION, GENERATE_IMAGE_MODEL, generateImageSchema, generatedImageDestination, imageChargeCostEvent, imageIdentity, ImageGenerationProviderError, priceGeneratedImage, requestOpenAiImage, uncertainImageCharge, writeGeneratedImageLocally } from "./generate-image-tool.js"; +import { aiConnectionService } from "../ai-connections.js"; +import { createCostEventInTransaction } from "../costs.js"; +import { withOperationAccountingTransaction } from "../accounting-transaction.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { nativeRunKeyScope, nativeSkillTestToolAllowed, assertNativeSkillTestTool } from "./native-skill-test-scope.js"; import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; @@ -212,6 +216,8 @@ type Binding = { readRemoteWorkspaceFile?: RemoteWorkspaceFileReader; readRemoteTaskFile?: RemoteWorkspaceFileReader; installRemoteArtifact?: ArtifactInstaller; + /** Server-derived from the run's admitted OpenAI API-key connection; never tool input. */ + imageGeneration?: { connectionId: string; grantId: string }; readRemoteApiFile?: (path: string, maxBytes: number) => Promise; currentWakeComments?: CurrentWakeCommentsBinding; chatAttachmentReadScope?: NativeChatAttachmentReadScope; @@ -392,6 +398,7 @@ export class PaperclipRunnerToolAuthority { if (!this.binding.firstPartyOnly && !this.binding.museRuntime && !this.binding.dotRuntime && workMode === "standard" && this.binding.workspaceRoot && runnerApiToolsEnabled(this.binding.companyId, this.binding.apiToolsEnabled) && (this.binding.executionTargetKind === "remote" ? this.binding.installRemoteArtifact : process.platform === "linux")) definitions.push(DOWNLOAD_ARTIFACT_DEFINITION); + if (this.#imageGenerationAdvertised(workMode)) definitions.push(GENERATE_IMAGE_DEFINITION); definitions.push(READ_CURRENT_WAKE_COMMENTS_TOOL_DEFINITION); definitions.push(LIST_CHAT_ATTACHMENTS_TOOL_DEFINITION); definitions.push(REUSE_CHAT_ATTACHMENT_TOOL_DEFINITION); @@ -413,6 +420,13 @@ export class PaperclipRunnerToolAuthority { if (!this.binding.museRuntime) await this.binding.assertBridgeAuthority?.(); const context = await this.#boundContext({ tool: call.tool }); const { run } = context; + if (call.tool === GENERATE_IMAGE) { + const input = generateImageSchema.safeParse(call.arguments); + const receipt = record(record(run.resultJson?.imageToolReceipts)[toolCallKey(call.callId)]); + if (!input.success || receipt.digest !== toolInputDigest(input.data)) return null; + if (receipt.state === "charged") await this.#recordImageCharge(toolCallKey(call.callId), record(receipt.charge)); + return receipt.state === "completed" && Object.hasOwn(receipt, "result") ? { result: receipt.result } : null; + } if (call.tool === DOWNLOAD_ARTIFACT) { try { return { result: await this.#downloadArtifact(call.arguments, true) }; } catch (error) { @@ -637,6 +651,7 @@ export class PaperclipRunnerToolAuthority { if (this.binding.firstPartyOnly && (call.tool === READ_TOOL_RESULT || this.binding.assignedMcpTools?.has(call.tool) || isConnectorTool(call.tool) || RUNTIME_CONNECTION_TOOL_DEFINITIONS.some(tool => tool.name === call.tool))) throw forbidden("This Grok Bot connection exposes first-party Paperclip operations. Use your own provider apps under their existing permissions."); if (call.tool === DOWNLOAD_ARTIFACT) return this.#downloadArtifact(call.arguments); + if (call.tool === GENERATE_IMAGE) return this.#generateImage(call.callId, call.arguments); if (call.tool === READ_TOOL_RESULT) { await this.#boundContext({ tool: call.tool }); return this.#readToolResult(call.arguments); @@ -1178,6 +1193,172 @@ export class PaperclipRunnerToolAuthority { } } + #imageGenerationAdvertised(workMode: string) { + return Boolean(this.binding.imageGeneration) && !this.binding.firstPartyOnly && !this.binding.museRuntime && !this.binding.dotRuntime + && !this.binding.nativeReview && workMode === "standard" && Boolean(this.binding.workspaceRoot) + && (this.binding.executionTargetKind !== "remote" || Boolean(this.binding.installRemoteArtifact)); + } + + /** + * Makes one image with the run's existing OpenAI API key. The key is read + * here, in the server, and sent only to OpenAI; it never reaches the + * provider process or the agent's shell. One receipt per provider call ID + * means a retried or recovered call replays instead of paying twice. + */ + async #generateImage(callId: string, value: unknown): Promise { + const input = generateImageSchema.parse(value); + const binding = this.binding.imageGeneration; + const initial = await this.#boundContext({ tool: GENERATE_IMAGE }); + if (!binding || !this.#imageGenerationAdvertised(initial.issue.workMode) || initial.issue.workMode !== "standard") { + throw forbidden("Image generation is unavailable to this run or provider"); + } + const key = toolCallKey(callId); + const digest = toolInputDigest(input); + const priorReceipt = (resultJson: unknown): { result: unknown } | { charged: Record } | undefined => { + const receipt = record(record(record(resultJson).imageToolReceipts)[key]); + if (!Object.keys(receipt).length) return undefined; + if (receipt.digest !== digest) throw badRequest("This tool call ID was already used for a different image request"); + if (receipt.state === "completed" && Object.hasOwn(receipt, "result")) return { result: receipt.result }; + if (receipt.state === "failed") throw badRequest(String(receipt.error ?? "Image generation failed")); + if (receipt.state === "charged") return { charged: receipt }; + throw new SemanticToolOutcomeUnknownError("An earlier attempt of this image call did not finish. Call generate_image again with a new request if the image is still needed."); + }; + const replay = priorReceipt(initial.run.resultJson); + if (replay && "charged" in replay) return this.#settleInterruptedImage(key, replay.charged); + if (replay) return replay.result; + const authorize = async () => { + const current = await this.#boundContext({ tool: GENERATE_IMAGE }); + if (!this.#imageGenerationAdvertised(current.issue.workMode)) throw forbidden("Image generation is unavailable to this run or provider"); + const decision = await authorizationService(this.db).decide({ actor: this.#privacyActor(current.run), action: "issue:mutate", + resource: { type: "issue", companyId: this.binding.companyId, issueId: current.issue.id } }); + if (!decision.allowed) throw forbidden(decision.explanation); + // #boundContext checks company and agent budgets; spending also needs the task's project. + if (current.issue.projectId && await budgetServiceInTransaction(this.db).getInvocationBlock(this.binding.companyId, this.binding.agentId, + { issueId: current.issue.id, projectId: current.issue.projectId })) throw forbidden("Company or budget authority unavailable"); + await assertNoCaseHandoffMutation(this.db, this.binding.runId, this.binding.issueId); + return current; + }; + const current = await authorize(); + const connections = aiConnectionService(this.db); + const connection = await connections.selectImageGeneration({ companyId: this.binding.companyId, agentId: this.binding.agentId, + connectionId: binding.connectionId, grantId: binding.grantId, userId: current.run.responsibleUserId }); + const apiKey = await connections.credential(connection, 0, { responsibleUserId: current.run.responsibleUserId, actorType: "agent", + actorId: this.binding.agentId, issueId: this.binding.issueId, heartbeatRunId: this.binding.runId }); + if (!apiKey) throw forbidden("Image generation needs an OpenAI API key"); + const setReceipt = (receipt: Record, claim = false) => this.db.update(heartbeatRuns).set({ + resultJson: sql`jsonb_set(coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb), '{imageToolReceipts}', + coalesce(${heartbeatRuns.resultJson} -> 'imageToolReceipts', '{}'::jsonb) || ${JSON.stringify({ [key]: { digest, ...receipt } })}::jsonb)`, + }).where(and(eq(heartbeatRuns.id, this.binding.runId), eq(heartbeatRuns.companyId, this.binding.companyId), + ...(claim ? [sql`not (coalesce(${heartbeatRuns.resultJson} -> 'imageToolReceipts', '{}'::jsonb) ? ${key})`] : []))) + .returning({ id: heartbeatRuns.id }); + // Claim the call before spending; a concurrent duplicate replays the winner. + if (!(await setReceipt({ state: "started" }, true)).length) { + const [row] = await this.db.select({ resultJson: heartbeatRuns.resultJson }).from(heartbeatRuns).where(eq(heartbeatRuns.id, this.binding.runId)); + const raced = priorReceipt(row?.resultJson); + if (raced && "charged" in raced) return this.#settleInterruptedImage(key, raced.charged); + if (raced) return raced.result; + throw new SemanticToolOutcomeUnknownError("This image call is already in progress."); + } + const details = { model: GENERATE_IMAGE_MODEL, size: input.size, quality: input.quality, promptChars: input.prompt.length }; + const audit = async (outcome: string, extra: Record = {}) => { await logActivity(this.db, { + companyId: this.binding.companyId, actorType: "agent", actorId: this.binding.agentId, runId: this.binding.runId, + action: "runner.image_generate", entityType: "issue", entityId: this.binding.issueId, details: { ...details, ...extra, outcome } }); }; + await audit("started"); + // Stop, reassignment or a budget stop can commit while the writes above + // wait. Recheck live authority as the last step before paying. + try { + const live = await authorize(); + // The grant can be revoked while the writes above wait; a revoked grant + // does not delete the key, so reload connection access too. + await connections.selectImageGeneration({ companyId: this.binding.companyId, agentId: this.binding.agentId, + connectionId: binding.connectionId, grantId: binding.grantId, userId: live.run.responsibleUserId }); + } catch (error) { + await setReceipt({ state: "failed", error: "Image generation was stopped before any request was sent" }).catch(() => undefined); + await audit("refused").catch(() => undefined); + throw error; + } + let image: Awaited>; + try { + image = await requestOpenAiImage({ ...input, apiKey }); + } catch (error) { + const message = error instanceof Error ? error.message.slice(0, 500) : "Image generation failed"; + if (!(error instanceof ImageGenerationProviderError && error.rejected)) { + // OpenAI may have generated and billed this image. Record unknown spend + // through the same settlement path; run accounting retries it. + const charge = uncertainImageCharge({ projectId: current.issue.projectId ?? null, reason: message }); + await setReceipt({ state: "charged", charge, error: message }).catch(() => undefined); + // Settle only after the spend is recorded; otherwise run accounting + // still finds the `charged` receipt and records it later. + const recorded = await this.#recordImageCharge(key, charge).then(() => true, () => false); + if (recorded) await setReceipt({ state: "failed", charge, error: `The image request may have been billed but returned no usable image: ${message}` }).catch(() => undefined); + await audit("unconfirmed", { error: message }).catch(() => undefined); + throw badRequest(`The image request may have been billed but returned no usable image: ${message}`); + } + await setReceipt({ state: "failed", error: message }).catch(() => undefined); + await audit("failed", { error: message }).catch(() => undefined); + throw badRequest(message); + } + const price = priceGeneratedImage(image); + const identity = imageIdentity(image.bytes); + const destination = generatedImageDestination(input.name); + const result = { outcome: "completed", path: destination, contentType: "image/png", ...identity, model: image.model, size: input.size, + costUsd: price.costUsd, guidance: "Saved in the workspace root. Move or copy it where the task needs it; use register_deliverable if the user should receive it." }; + // The provider has charged. Persist the charge before any other step, so a + // failed save, accounting write or process crash can still settle it. + const charge = { model: image.model, providerRequestId: image.providerRequestId, usage: image.usage, costCents: price.costCents, + pricingProvenance: price.pricingProvenance, projectId: current.issue.projectId ?? null, destination, ...identity, + chargedAt: new Date().toISOString() }; + try { + await setReceipt({ state: "charged", charge }); + } catch (error) { + // Without the receipt, record the spend now; the call cannot be confirmed. + await this.#recordImageCharge(key, charge).catch(() => undefined); + await audit("unconfirmed", { destination, ...identity, costCents: price.costCents }).catch(() => undefined); + throw new SemanticToolOutcomeUnknownError("The image was generated but its receipt could not be saved. Do not retry this call.", { cause: error }); + } + let saveError: unknown; + try { + await authorize(); + if (this.binding.executionTargetKind === "remote") { + await this.binding.installRemoteArtifact!({ destination, ...identity, bytes: image.bytes, authorize: async () => { await authorize(); } }); + } else { + await writeGeneratedImageLocally(this.binding.workspaceRoot!, destination, image.bytes); + } + } catch (error) { saveError = error; } + // Spend is recorded whether or not the image was saved. The budget + // service evaluates hard-stops here. + await this.#recordImageCharge(key, charge); + if (saveError !== undefined) { + if (!(saveError instanceof SemanticToolOutcomeUnknownError)) { + await setReceipt({ state: "failed", error: "The image was generated but could not be saved to the workspace" }).catch(() => undefined); + } + await audit("unconfirmed", { destination, ...identity, costCents: price.costCents }).catch(() => undefined); + throw saveError; + } + await setReceipt({ state: "completed", result }); + await audit("completed", { destination, ...identity, costCents: price.costCents }); + return result; + } + + async #recordImageCharge(key: string, charge: Record) { + await withOperationAccountingTransaction(this.db, this.binding.companyId, (tx, publications) => createCostEventInTransaction(tx, this.binding.companyId, + imageChargeCostEvent({ agentId: this.binding.agentId, issueId: this.binding.issueId, runId: this.binding.runId, key, charge }), publications)); + } + + /** An interrupted call was paid for but not confirmed saved: record the spend, then settle it as failed. */ + async #settleInterruptedImage(key: string, receipt: Record): Promise { + const charge = record(receipt.charge); + await this.#recordImageCharge(key, charge); + const message = typeof charge.destination === "string" + ? `An earlier attempt generated the image but did not confirm saving it. Check whether ${charge.destination} exists in the workspace before generating another.` + : "An earlier image request may have been billed but returned no usable image."; + await this.db.update(heartbeatRuns).set({ + resultJson: sql`jsonb_set(coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb), '{imageToolReceipts}', + coalesce(${heartbeatRuns.resultJson} -> 'imageToolReceipts', '{}'::jsonb) || ${JSON.stringify({ [key]: { ...receipt, state: "failed", error: message } })}::jsonb)`, + }).where(and(eq(heartbeatRuns.id, this.binding.runId), eq(heartbeatRuns.companyId, this.binding.companyId))); + throw badRequest(message); + } + async #readToolResult(value: unknown): Promise { if (!this.binding.assignedMcpTools) throw forbidden("Saved app-tool results are unavailable to this run"); const input = readToolResultSchema.parse(value); diff --git a/server/src/services/run-cost-accounting.ts b/server/src/services/run-cost-accounting.ts index 7afc306ab7..cb4244dc60 100644 --- a/server/src/services/run-cost-accounting.ts +++ b/server/src/services/run-cost-accounting.ts @@ -11,6 +11,7 @@ import { conflict } from "../errors.js"; import { logActivity } from "./activity-log.js"; import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "./native-runtime/native-runner-recovery-contract.js"; import { createCostEventInTransaction } from "./costs.js"; +import { settleChargedImageReceiptsInTransaction } from "./native-runtime/generate-image-tool.js"; import type { BudgetServiceHooks } from "./budgets.js"; import { queueBudgetEvaluation } from "./budget-evaluation-work.js"; import { ensureRuntimeAccountingBaseline } from "./runtime-accounting-totals.js"; @@ -216,6 +217,11 @@ export async function accountRunCost(db: Db, runId: string, hooks: BudgetService )).for("key share"); let [run] = await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update"); if (!run?.costAccountingPending || !terminalStatuses.includes(run.status)) return false; + // Tool-side image spend is separate from the provider receipt; settle any + // paid image whose own accounting write did not complete. + for (const projectId of new Set(await settleChargedImageReceiptsInTransaction(tx, run, publications))) { + await queueBudgetEvaluation(tx, { companyId: run.companyId, agentId: run.agentId, projectId }); + } if (run.costAccountedAt) { if (object(run.usageJson).accountingLateReceiptPending === true) { // Clearing late debt can release an admission hold too.