diff --git a/doc/project-repositories.md b/doc/project-repositories.md index e6eee59956..d6e2245d11 100644 --- a/doc/project-repositories.md +++ b/doc/project-repositories.md @@ -73,6 +73,14 @@ access; PAT connections use paginated `/user/repos`. Provider failures are repor without exposing provider error bodies or credential material. Successful connections remain selectable when another connection fails. +Task repository preparation distinguishes a temporary provider refusal from a +completed catalog that denies access. Before provider dispatch, known HTTP +availability failures and exhausted quotas use the existing bounded infrastructure +retry and provider deadline. Each successor discovers repositories again under the +same responsible user's current grants. Empty successful catalogs, revoked grants, +and permission denials remain terminal. A partial catalog can authorize a requested +repository only when that repository is present. + `ConnectionSetupFlow` owns provider setup in both Apps and project dialogs. Task intents retain their existing callback protocol. Standalone dialogs verify the saved connection through the API after the sign-in popup returns to the instance. Project diff --git a/server/src/__tests__/github-repository-catalog-failure.test.ts b/server/src/__tests__/github-repository-catalog-failure.test.ts new file mode 100644 index 0000000000..ae5ee02133 --- /dev/null +++ b/server/src/__tests__/github-repository-catalog-failure.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { readGitHubRepositoryCatalogFailure } from "../services/tool-access/github-repository-catalog-failure.js"; + +describe("GitHub repository discovery failure", () => { + const now = Date.UTC(2026, 0, 1); + it.each([401, 403, 404, 422])("keeps HTTP%s authorization failures terminal", status => { + expect(readGitHubRepositoryCatalogFailure(new Response(null, { status }), now)).toBeNull(); + }); + it.each([408, 425, 429, 500, 502, 503, 504])("recognizes HTTP%s availability without exposing provider content", status => { + expect(readGitHubRepositoryCatalogFailure(new Response("private provider detail", { status }), now)) + .toMatchObject({ code: "repository_catalog_unavailable", retryAt: undefined }); + }); + it("recognizes secondary rate limits without treating permission denials as temporary", () => { + expect(readGitHubRepositoryCatalogFailure(new Response(null, { status: 403, + headers: { "retry-after": "60", "x-ratelimit-remaining": "42" } }), now)?.retryAt).toBe(now + 60_000); + expect(readGitHubRepositoryCatalogFailure(new Response(null, { status: 403, + headers: { "retry-after": "invalid", "x-ratelimit-remaining": "42" } }), now)).toBeNull(); + }); + it("respects both exhausted quota deadlines", () => { + const response = new Response(null, { status: 403, headers: { + "x-ratelimit-remaining": "0", "retry-after": "5", "x-ratelimit-reset": String((now + 60_000) / 1000), + } }); + expect(readGitHubRepositoryCatalogFailure(response, now)?.retryAt).toBe(now + 60_000); + }); + it("bounds the provider deadline and ignores malformed reset headers", () => { + expect(readGitHubRepositoryCatalogFailure(new Response(null, { status: 429, + headers: { "retry-after": "999999999" } }), now)?.retryAt).toBe(now + 24 * 3600_000); + expect(readGitHubRepositoryCatalogFailure(new Response(null, { status: 403, + headers: { "x-ratelimit-remaining": "0", "x-ratelimit-reset": "not-a-date" } }), now)?.retryAt).toBeUndefined(); + }); +}); diff --git a/server/src/__tests__/heartbeat-repository-catalog-recovery.test.ts b/server/src/__tests__/heartbeat-repository-catalog-recovery.test.ts new file mode 100644 index 0000000000..249f721fa5 --- /dev/null +++ b/server/src/__tests__/heartbeat-repository-catalog-recovery.test.ts @@ -0,0 +1,235 @@ +import express from "express"; +import request from "supertest"; +import { taskWorkspaceRoutes } from "../routes/task-workspaces.js"; +import { errorHandler } from "../middleware/error-handler.js"; +import { randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, authUsers, companies, companyMemberships, connectionGrants, createDb, + executionWorkspaceRepositories, executionWorkspaces, heartbeatRuns, issues, toolApplications, toolConnections, +} from "@paperclipai/db"; +import { RepositoryCatalogUnavailableError } from "../services/repository-catalog-unavailable-error.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { taskRepositoryRelativePath } from "../services/execution-workspace-repositories.js"; +import { writeConnectionCredential } from "../services/connection-credentials.js"; +import { syncConnectionCredentialBindings } from "../services/connection-credential-bindings.js"; +import { toolAccessService } from "../services/tool-access.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; +import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js"; + +const execute = vi.hoisted(() => vi.fn()); +vi.mock("../adapters/index.js", () => ({ + getServerAdapter: () => ({ type: "codex_local", execute, supportsLocalAgentJwt: false }), + findActiveServerAdapter: () => ({ type: "codex_local", execute, supportsLocalAgentJwt: false }), + runningProcesses: new Map(), +})); + +describe.skipIf(!recoveryPostgresSupport.supported)("repository catalog bootstrap recovery", () => { + let temporary: Awaited>; + let db: ReturnType; + let heartbeat: ReturnType; + let root: string; + + beforeAll(async () => { + root = await realpath(await mkdtemp(path.join(tmpdir(), "repository-catalog-recovery-"))); + vi.stubEnv("PAPERCLIP_HOME", path.join(root, "home")); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "01234567890123456789012345678901"); + vi.stubEnv("PAPERCLIP_ISSUE_PRIVACY_MODE", "enforce"); + temporary = await startRecoveryPostgres("repository-catalog-recovery-"); + db = createDb(temporary.connectionString); + await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true }); + heartbeat = heartbeatService(db); + }, 60_000); + + afterEach(async () => { + await drainHeartbeatRunsToQuiescence(db, heartbeat); + vi.useRealTimers(); + vi.restoreAllMocks(); + execute.mockReset(); + }); + + afterAll(async () => { + await db?.$client.end(); + await temporary?.cleanup(); + vi.unstubAllEnvs(); + if (root) await rm(root, { recursive: true, force: true }); + }); + + async function fixture(status: number, genuineDenial = false) { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(); + const workspaceId = randomUUID(), repositoryId = randomUUID(), userId = randomUUID(); + await db.insert(authUsers).values({ id: userId, name: "Repository owner", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companies).values({ id: companyId, name: "Repository retry", issuePrefix: `C${companyId.slice(0, 6)}`, defaultResponsibleUserId: userId }); + await db.insert(companyMemberships).values({ companyId, principalId: userId, principalType: "user", membershipRole: "owner", status: "active" }); + const cwd = path.join(root, workspaceId), repoUrl = "https://github.com/catalog-fixture/repository"; + const relativePath = taskRepositoryRelativePath(repoUrl), checkout = path.join(cwd, relativePath); + await mkdir(checkout, { recursive: true }); + const git = (...args: string[]) => execFileSync("git", ["-C", checkout, ...args], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); + git("init", "--initial-branch=main"); + git("config", "user.name", "Catalog fixture"); + git("config", "user.email", "catalog@example.test"); + git("remote", "add", "origin", repoUrl); + await writeFile(path.join(checkout, "saved.txt"), "retained repository work\n"); + git("add", "saved.txt"); + git("commit", "-m", "Saved fixture"); + const pinnedCommit = git("rev-parse", "HEAD"); + await writeFile(path.join(checkout, ".git", "paperclip-workspace-owner.json"), JSON.stringify({ version: 1, repositoryId, pinnedCommit })); + await db.insert(agents).values({ id: agentId, companyId, name: "Catalog worker", status: "idle", adapterType: "codex_local", + adapterConfig: { cwd }, runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } }, permissions: {} }); + await db.insert(issues).values({ id: issueId, companyId, title: "Continue retained repository", status: "todo", assigneeAgentId: agentId, + executionWorkspacePreference: "reuse_existing", executionWorkspaceSettings: { mode: "shared_workspace" }, + workspaceSelection: { version: 1, source: "explicit", selection: { kind: "existing", workspaceId } } }); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issueId, name: "Retained task files", cwd, + mode: "shared_workspace", strategyType: "task_directory" }); + await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, issueId)); + await db.insert(executionWorkspaceRepositories).values({ id: repositoryId, companyId, executionWorkspaceId: workspaceId, + requestedByIssueId: issueId, repoUrl, repositoryIdentity: repoUrl, catalogRepositoryId: "201", relativePath, + requestedRef: "HEAD", pinnedCommit, branchName: "main", state: "ready", requestKey: "saved-repository", requestKeys: ["saved-repository"] }); + const [application] = await db.insert(toolApplications).values({ companyId, name: "Catalog GitHub", type: "mcp_http" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: application.id, name: "Catalog GitHub", uid: randomUUID(), + enabled: true, status: "active", transport: "mcp_remote", authKind: "api_key", credentialPolicy: "per_user", config: { sourceTemplateKey: "github" } }).returning(); + const { secret } = await writeConnectionCredential(db, { companyId, connectionName: connection.name, configPath: "credentials.authorization", + label: "Fixture credential", value: "synthetic-catalog-token", ownerUserId: userId, actor: { userId } }); + const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId: connection.id, kind: "user", subjectUserId: userId, + credentialSecretRefs: [{ configPath: "credentials.authorization", secretId: secret.id, versionSelector: "latest" }] }).returning(); + await syncConnectionCredentialBindings(db, connection); + expect(await toolAccessService(db).githubReadHeaders(companyId, connection.id, userId)).toEqual({ Authorization: "Bearer synthetic-catalog-token" }); + let available = false; + const retryAt = new Date(Math.ceil((Date.now() + 1000) / 1000) * 1000); + const provider = vi.spyOn(globalThis, "fetch").mockImplementation(async url => { + expect(new URL(String(url)).hostname).toBe("api.github.com"); + expect(new URL(String(url)).pathname).toBe("/user/repos"); + return available ? Response.json([{ id: 201, full_name: "catalog-fixture/repository", private: true }]) + : Response.json({ message: "synthetic provider body must not become an access denial" }, { status, + headers: genuineDenial ? {} : status === 403 ? { "x-ratelimit-remaining": "0", "x-ratelimit-reset": String(retryAt.getTime() / 1000) } : { "retry-after": "1" } }); + }); + execute.mockImplementation(async () => { + expect(available).toBe(true); + expect(await readFile(path.join(checkout, "saved.txt"), "utf8")).toBe("retained repository work\n"); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, issueId)); + return { exitCode: 0, signal: null, timedOut: false }; + }); + return { companyId, agentId, issueId, workspaceId, repositoryId, userId, applicationId: application.id, grant, provider, retryAt, recover: () => { available = true; } }; + } + + it("keeps the direct repository request HTTP contract outside bootstrap recovery", async () => { + const f = await fixture(503); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.actor = { type: "board", source: "session", userId: f.userId, companyIds: [f.companyId], + memberships: [{ companyId: f.companyId, membershipRole: "owner", status: "active" }], isInstanceAdmin: false }; + next(); + }); + app.use("/api", taskWorkspaceRoutes(db)); + app.use(errorHandler); + const response = await request(app).post(`/api/issues/${f.issueId}/workspace/repositories`) + .send({ repository: { kind: "catalog", id: "201" }, requestKey: "direct-repository-request" }); + expect(response.status).toBe(403); + expect(response.body.error).toBe("Repository access is no longer available; refresh the repository catalog"); + expect(f.provider).toHaveBeenCalled(); + expect(execute).not.toHaveBeenCalled(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(0); + }); + + it.each([true, false])("accepts a partial catalog only when the required repository is present (%s)", async present => { + const f = await fixture(503); + const [connection] = await db.insert(toolConnections).values({ companyId: f.companyId, applicationId: f.applicationId, + name: "Healthy GitHub", uid: randomUUID(), enabled: true, status: "active", transport: "mcp_remote", + authKind: "api_key", credentialPolicy: "per_user", config: { sourceTemplateKey: "github" } }).returning(); + const { secret } = await writeConnectionCredential(db, { companyId: f.companyId, connectionName: connection.name, + configPath: "credentials.authorization", label: "Healthy fixture", value: "synthetic-healthy-token", + ownerUserId: f.userId, actor: { userId: f.userId } }); + await db.insert(connectionGrants).values({ companyId: f.companyId, connectionId: connection.id, kind: "user", + subjectUserId: f.userId, credentialSecretRefs: [{ configPath: "credentials.authorization", secretId: secret.id, versionSelector: "latest" }] }); + await syncConnectionCredentialBindings(db, connection); + f.provider.mockImplementation(async (url, init) => { + expect(new URL(String(url)).hostname).toBe("api.github.com"); + if (new Headers(init?.headers).get("authorization") === "Bearer synthetic-healthy-token") { + return Response.json([{ id: present ? 201 : 202, full_name: present ? "catalog-fixture/repository" : "catalog-fixture/other", private: true }]); + } + return new Response(null, { status: 503 }); + }); + const discovered = toolAccessService(db).listProjectRepositories(f.companyId, f.userId, false, { requiredRepositoryIds: ["201"] }); + if (present) expect(await discovered).toMatchObject({ failedConnectionCount: 1, repositories: [expect.objectContaining({ id: "201" })] }); + else await expect(discovered).rejects.toMatchObject({ code: "repository_catalog_unavailable" }); + expect(f.provider).toHaveBeenCalledTimes(2); + expect(execute).not.toHaveBeenCalled(); + }); + + it("does not retry a genuine catalog permission denial or start provider work", async () => { + const f = await fixture(403, true); + const run = await heartbeat.wakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", + contextSnapshot: { issueId: f.issueId } }); + await vi.waitFor(async () => { expect(await heartbeat.getRun(run!.id)).toMatchObject({ status: "failed", errorCode: "setup_failed" }); }, { timeout: 15_000 }); + await drainHeartbeatRunsToQuiescence(db, heartbeat); + expect(f.provider).toHaveBeenCalled(); + expect(execute).not.toHaveBeenCalled(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run!.id))).toHaveLength(0); + expect(await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id))).toEqual([f.grant]); + }); + + it("does not classify a provider-started error as catalog bootstrap recovery", async () => { + const f = await fixture(503); + f.recover(); + execute.mockRejectedValue(new RepositoryCatalogUnavailableError()); + const run = await heartbeat.wakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", contextSnapshot: { issueId: f.issueId } }); + await vi.waitFor(async () => { expect(await heartbeat.getRun(run!.id)).toMatchObject({ status: "failed", errorCode: "adapter_failed" }); }, { timeout: 15_000 }); + await drainHeartbeatRunsToQuiescence(db, heartbeat); + expect(execute).toHaveBeenCalledTimes(1); + const failed = await heartbeat.getRun(run!.id); + expect(failed!.resultJson?.executionRecovery).not.toEqual(expect.objectContaining({ kind: "bootstrap", providerWorkStarted: false })); + expect(failed!.resultJson?.retryNotBefore).toBeUndefined(); + for (const retry of await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run!.id))) { + await heartbeat.cancelRun(retry.id, "Fixture cleanup", { expectedStatus: "scheduled_retry" }); + } + }); + + it("preserves an operator Stop before the automatic successor becomes due", async () => { + const f = await fixture(503); + const run = await heartbeat.wakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", contextSnapshot: { issueId: f.issueId } }); + await vi.waitFor(async () => { expect(await heartbeat.getRun(run!.id)).toMatchObject({ status: "failed" }); }, { timeout: 15_000 }); + await drainHeartbeatRunsToQuiescence(db, heartbeat); + const [successor] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run!.id)); + expect(successor).toBeDefined(); + await heartbeat.cancelRun(successor.id, "Operator stopped retry", { expectedStatus: "scheduled_retry", + resultJson: { cancelledByActorType: "user", cancelledByUserId: f.grant.subjectUserId } }); + f.recover(); + await heartbeat.promoteDueScheduledRetries(new Date(successor.scheduledRetryAt!.getTime() + 1)); + await heartbeat.resumeQueuedRunsForAgent(f.companyId, f.agentId); + expect(await heartbeat.getRun(successor.id)).toMatchObject({ status: "cancelled" }); + expect(execute).not.toHaveBeenCalled(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, successor.id))).toHaveLength(0); + }); + + it.each([403, 503])("retries a real HTTP%s catalog refusal before provider work and continues only after fresh discovery", async status => { + const f = await fixture(status); + const run = await heartbeat.wakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", + contextSnapshot: { issueId: f.issueId } }); + expect(run).not.toBeNull(); + await vi.waitFor(async () => { expect(await heartbeat.getRun(run!.id)).toMatchObject({ status: "failed" }); }, { timeout: 15_000 }); + await drainHeartbeatRunsToQuiescence(db, heartbeat); + expect(f.provider).toHaveBeenCalled(); + expect(execute).not.toHaveBeenCalled(); + const successors = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run!.id)); + expect(successors).toHaveLength(1); + expect(successors[0].contextSnapshot?.codexTransientFallbackMode).toBeUndefined(); + expect(successors[0].scheduledRetryAt!.getTime()).toBeGreaterThanOrEqual(f.retryAt.getTime()); + expect(await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id))).toEqual([f.grant]); + f.recover(); + const due = new Date(successors[0].scheduledRetryAt!.getTime() + 1); + await new Promise(resolve => setTimeout(resolve, Math.max(0, due.getTime() - Date.now()))); + await heartbeat.promoteDueScheduledRetries(due); + await heartbeat.resumeQueuedRunsForAgent(f.companyId, f.agentId); + await vi.waitFor(async () => { expect(await heartbeat.getRun(successors[0].id)).toMatchObject({ status: "succeeded" }); }, { timeout: 15_000 }); + expect(execute).toHaveBeenCalledTimes(1); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run!.id))).toHaveLength(1); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]).toMatchObject({ status: "done", executionWorkspaceId: f.workspaceId }); + }, 60_000); +}); diff --git a/server/src/services/execution-workspace-repositories.ts b/server/src/services/execution-workspace-repositories.ts index 3c8e4559b5..fcd2d73c4c 100644 --- a/server/src/services/execution-workspace-repositories.ts +++ b/server/src/services/execution-workspace-repositories.ts @@ -113,9 +113,9 @@ export function assertRepositoryRetryMatches(row: Pick - (await toolAccessService(db).listProjectRepositories(companyId, userId, localTrusted)).repositories); + (await toolAccessService(db).listProjectRepositories(companyId, userId, localTrusted, { requiredRepositoryIds })).repositories); } async function requiresAnonymousAcquisition(reader: Db | DbTransaction, input: { issue: typeof issues.$inferSelect; workspace: Pick | null; @@ -362,7 +362,8 @@ export function executionWorkspaceRepositoryService(db: Db) { } if (readRemoteWorkspaceAuthority(workspace.metadata)) { if (!input.remoteTarget || !input.remoteLease) throw conflict("Remote repository preparation requires its bound sandbox"); - const available = acquisitionRestricted ? [] : await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board"); + const available = acquisitionRestricted ? [] : await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board", + rows.flatMap(row => row.catalogRepositoryId ? [row.catalogRepositoryId] : [])); const resolveAuth = createGitRemoteAuthProvider(db, input.companyId, { issueId: input.issueId, heartbeatRunId: input.runId, agentId: input.agentId, responsibleUserId: input.responsibleUserId }); const prepared: Array<{ id: string; cwd: string; repoUrl: string; relativePath: string; pinnedCommit: string; branchName: string | null }> = []; @@ -380,7 +381,7 @@ export function executionWorkspaceRepositoryService(db: Db) { await publicationBinding(db, input); if (!acquisitionRestricted) await assertCurrentControllerAcquisition(input); if (source.catalogRepositoryId) { - const current = await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board"); + const current = await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board", [source.catalogRepositoryId]); const authorized = resolveAcquisitionSource({ kind: "catalog", id: source.catalogRepositoryId }, current, false); if (authorized.repositoryIdentity !== row.repositoryIdentity) throw conflict("Repository catalog authorization changed"); } @@ -430,7 +431,8 @@ export function executionWorkspaceRepositoryService(db: Db) { return prepared; } const root = await measureRunPhase("workspace.repository.paths", () => fs.realpath(input.cwd)); - const available = acquisitionRestricted ? [] : await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board"); + const available = acquisitionRestricted ? [] : await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board", + rows.flatMap(row => row.catalogRepositoryId ? [row.catalogRepositoryId] : [])); const resolveAuth = createGitRemoteAuthProvider(db, input.companyId, { issueId: input.issueId, heartbeatRunId: input.runId, agentId: input.agentId, responsibleUserId: input.responsibleUserId }); const sources = rows.map(row => { assertAcquisitionReceipt(row, acquisitionRestricted); @@ -535,7 +537,7 @@ export function executionWorkspaceRepositoryService(db: Db) { await assertCurrentControllerAcquisition(input); await assertPublicationAllowed(input); if (source.catalogRepositoryId) { - const currentCatalog = await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board"); + const currentCatalog = await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board", [source.catalogRepositoryId]); const currentSource = resolveAcquisitionSource({ kind: "catalog", id: source.catalogRepositoryId }, currentCatalog, false); if (currentSource.repositoryIdentity !== row.repositoryIdentity) throw conflict("Repository preparation receipt does not match its source"); } @@ -558,7 +560,7 @@ export function executionWorkspaceRepositoryService(db: Db) { if (!acquisitionRestricted) { await assertCurrentControllerAcquisition(input); if (source.catalogRepositoryId) { - const currentCatalog = await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board"); + const currentCatalog = await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board", [source.catalogRepositoryId]); const currentSource = resolveAcquisitionSource({ kind: "catalog", id: source.catalogRepositoryId }, currentCatalog, false); if (currentSource.repositoryIdentity !== row.repositoryIdentity) throw conflict("Repository preparation receipt does not match its source"); } diff --git a/server/src/services/heartbeat/retries.test.ts b/server/src/services/heartbeat/retries.test.ts index 5ee40c701a..c170776bb9 100644 --- a/server/src/services/heartbeat/retries.test.ts +++ b/server/src/services/heartbeat/retries.test.ts @@ -289,6 +289,25 @@ describePostgres("heartbeat retry module database wiring", () => { } }); + it("keeps the accepted interaction owner and its catalog provider deadline", async () => { + const { run, agent } = await fixture(); + const retryAt = new Date(now.getTime() + 3600_000); + const [failed] = await db.update(heartbeatRuns).set({ errorCode: "repository_catalog_unavailable", + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, + errorFamily: "transient_upstream", retryNotBefore: retryAt.toISOString() }, + contextSnapshot: { ...run.contextSnapshot, interactionId: "answered-question", interactionStatus: "answered", + interactionKind: "ask_user_questions", mutation: "interaction", wakeReason: "issue_commented" }, + }).where(eq(heartbeatRuns.id, run.id)).returning(); + expect(await boundRetries().retries.scheduleBoundedRetryForRun(failed!, agent, { + now, random: () => 0, retryReason: legacy.INTERACTION_CONTINUATION_INFRA_RETRY_REASON, maxAttempts: 2, + })).toMatchObject({ outcome: "scheduled" }); + const [successor] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run.id)); + expect(successor!.scheduledRetryAt!.getTime()).toBeGreaterThanOrEqual(retryAt.getTime()); + expect(successor).toMatchObject({ scheduledRetryReason: legacy.INTERACTION_CONTINUATION_INFRA_RETRY_REASON }); + expect(isResolvedInteractionContinuationWakeContext(successor!.contextSnapshot)).toBe(true); + expect(successor!.contextSnapshot?.codexTransientFallbackMode).toBeUndefined(); + }); + it("waits for a Runner update without spending failure retries and keeps an answered interaction", async () => { const { run, agent } = await fixture(); const snapshot = { ...run.contextSnapshot, interactionId: "interaction-1", interactionStatus: "answered", interactionKind: "ask_user_questions", diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 5f7c5dcde0..bee5fc219a 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -27,6 +27,7 @@ import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, REMOTE_WORKSPACE_RESTORE_FAI import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "../reusable-sandbox-resume-error.js"; import { PLUGIN_WORKER_UNAVAILABLE_CODE } from "../plugin-worker-unavailable-error.js"; import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "../environment-provider-unavailable-error.js"; +import { REPOSITORY_CATALOG_UNAVAILABLE_CODE } from "../repository-catalog-unavailable-error.js"; import { RUNNER_REVISION_MISMATCH_CODE, RUNNER_REVISION_MISMATCH_RETRY, RUNNER_REVISION_WAIT_RETRY_REASON } from "../runner-revision-mismatch.js"; import { CONTROLLER_LEASE_LOST_FAILURE_CODE } from "../legacy-controller-lease.js"; import { randomUUID } from "node:crypto"; @@ -141,7 +142,8 @@ export function isTransientWorkspacePreparationCode(code: string | null | undefi return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated || code === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE || code === REUSABLE_SANDBOX_RESUME_FAILED_CODE || code === REMOTE_WORKSPACE_RESTORE_FAILED_CODE || code === PLUGIN_WORKER_UNAVAILABLE_CODE - || code === RUNNER_REVISION_MISMATCH_CODE || code === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE; + || code === RUNNER_REVISION_MISMATCH_CODE || code === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE + || code === REPOSITORY_CATALOG_UNAVAILABLE_CODE; } export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = @@ -356,6 +358,7 @@ function isRetryableInteractionContinuationInfrastructureFailure( run.errorCode === CONTROLLER_LEASE_LOST_FAILURE_CODE || run.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE || run.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE || + run.errorCode === REPOSITORY_CATALOG_UNAVAILABLE_CODE || run.errorCode === RUNNER_REVISION_MISMATCH_CODE ) { return true; @@ -504,11 +507,12 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen ? { ...computedBaseSchedule, maxAttempts } : null; const transientRecovery = - retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON + (retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON + || retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON && run.errorCode === REPOSITORY_CATALOG_UNAVAILABLE_CODE) ? readTransientRecoveryContractFromRun(run) : null; const codexTransientFallbackMode = - agent.adapterType === "codex_local" && + agent.adapterType === "codex_local" && run.errorCode !== REPOSITORY_CATALOG_UNAVAILABLE_CODE && transientRecovery?.errorFamily === "transient_upstream" ? resolveCodexTransientFallbackMode(nextAttempt) : null; diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 4d70b78e2d..106e64777c 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -15,6 +15,7 @@ import { escalateRemoteWorkspaceRestoreFailure } from "../remote-workspace-resto import { findReusableSandboxResumeError } from "../reusable-sandbox-resume-error.js"; import { findPluginWorkerUnavailableError, PLUGIN_WORKER_UNAVAILABLE_CODE } from "../plugin-worker-unavailable-error.js"; import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE, findEnvironmentProviderUnavailableError } from "../environment-provider-unavailable-error.js"; +import { REPOSITORY_CATALOG_UNAVAILABLE_CODE, findRepositoryCatalogUnavailableError } from "../repository-catalog-unavailable-error.js"; import { findWorkspaceProviderFailure } from "../workspace-provider-failure.js"; import { isRunnerRevisionMismatch, RUNNER_REVISION_MISMATCH_CODE, RUNNER_REVISION_MISMATCH_RETRY } from "../runner-revision-mismatch.js"; import { type RunSessionOutcome, resolveNextSessionState, normalizeUsageTotals, resolveCacheAdjustedCostUsd, resolveLedgerBiller, resolveLedgerCostStatus, normalizeLedgerBillingType } from "./run-state.js"; @@ -1393,6 +1394,8 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC // stays with its creation record and the cleanup reconciler. const environmentProviderUnavailable = !providerDispatchStarted && run.runtimeMode === "legacy" && run.nativeSessionId === null && run.runnerInstanceId === null ? findEnvironmentProviderUnavailableError(outerErr) : null; + const repositoryCatalogUnavailable = !providerDispatchStarted && run.runtimeMode === "legacy" + && run.nativeSessionId === null && run.runnerInstanceId === null ? findRepositoryCatalogUnavailableError(outerErr) : null; const runnerRevisionMismatch = !providerDispatchStarted && run.runtimeMode === "legacy" && run.nativeSessionId === null && run.runnerInstanceId === null && isRunnerRevisionMismatch(outerErr); const setupFailureErrorCode = @@ -1402,6 +1405,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC remoteWorkspaceRestoreFailure?.code ?? pluginWorkerUnavailableFailure?.code ?? environmentProviderUnavailable?.code ?? + repositoryCatalogUnavailable?.code ?? (runnerRevisionMismatch ? RUNNER_REVISION_MISMATCH_CODE : null) ?? workspaceGitScanFailure?.code ?? workspaceValidationSetupFailure?.code ?? @@ -1451,6 +1455,9 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC const connectionFailure = readGitConnectionFailure(outerErr); const setupFailureResultJson = { ...setupFailureDetails, + ...(repositoryCatalogUnavailable ? { errorFamily: "transient_upstream", + ...(repositoryCatalogUnavailable.retryAt ? { retryNotBefore: new Date(repositoryCatalogUnavailable.retryAt).toISOString() } : {}), + } : {}), ...(providerFailure ? { providerFailure: { ...providerFailure, message: redactCurrentUserText(identityRedactor.redact(providerFailure.message), currentUserRedactionOptions) } } : {}), ...(connectionFailure ? { connectionFailure } : {}), @@ -1567,7 +1574,8 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC // An accepted interaction keeps its own continuation retry, which // carries the interaction and escalates on exhaustion. const interactionOwnsRetry = (livenessRun.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE - || livenessRun.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE) + || livenessRun.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE + || livenessRun.errorCode === REPOSITORY_CATALOG_UNAVAILABLE_CODE) && isResolvedInteractionContinuationWakeContext(livenessRun.contextSnapshot); await (isTransientWorkspacePreparationCode(livenessRun.errorCode) && !interactionOwnsRetry ? livenessRun.errorCode === RUNNER_REVISION_MISMATCH_CODE diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 32698e7d6a..978528a024 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -11,6 +11,7 @@ import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, REMOTE_WORKSPACE_RESTORE_FAI import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "./reusable-sandbox-resume-error.js"; import { PLUGIN_WORKER_UNAVAILABLE_CODE } from "./plugin-worker-unavailable-error.js"; import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "./environment-provider-unavailable-error.js"; +import { REPOSITORY_CATALOG_UNAVAILABLE_CODE } from "./repository-catalog-unavailable-error.js"; import { RUNNER_REVISION_MISMATCH_CODE } from "./runner-revision-mismatch.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { normalizeMaxTurnStopReason } from "./heartbeat-stop-metadata.js"; @@ -86,7 +87,8 @@ export function legacyExecutionNeedsReconciliation( if ((run.errorCode === "workspace_git_scan_timeout" || run.errorCode === "workspace_git_scan_saturated" || run.errorCode === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE || run.errorCode === REUSABLE_SANDBOX_RESUME_FAILED_CODE || run.errorCode === REMOTE_WORKSPACE_RESTORE_FAILED_CODE || run.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE - || run.errorCode === RUNNER_REVISION_MISMATCH_CODE || run.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE) && + || run.errorCode === RUNNER_REVISION_MISMATCH_CODE || run.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE + || run.errorCode === REPOSITORY_CATALOG_UNAVAILABLE_CODE) && evidence?.kind === "bootstrap" && evidence.providerWorkStarted === false) return false; if (executionFailureRetryCount(run) >= 2) return true; return !( diff --git a/server/src/services/repository-catalog-unavailable-error.ts b/server/src/services/repository-catalog-unavailable-error.ts new file mode 100644 index 0000000000..ec3faaca3d --- /dev/null +++ b/server/src/services/repository-catalog-unavailable-error.ts @@ -0,0 +1,17 @@ +export const REPOSITORY_CATALOG_UNAVAILABLE_CODE = "repository_catalog_unavailable"; + +export class RepositoryCatalogUnavailableError extends Error { + readonly code = REPOSITORY_CATALOG_UNAVAILABLE_CODE; + + constructor(readonly retryAt?: number) { + super("The repository catalog is temporarily unavailable. Paperclip will retry before starting agent work."); + this.name = "RepositoryCatalogUnavailableError"; + } +} + +export function findRepositoryCatalogUnavailableError(error: unknown): RepositoryCatalogUnavailableError | null { + for (let current = error, depth = 0; current && depth < 4; current = (current as { cause?: unknown }).cause, depth++) { + if (current instanceof RepositoryCatalogUnavailableError) return current; + } + return null; +} diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.ts index ae1682df28..2733fa83bd 100644 --- a/server/src/services/tool-access.ts +++ b/server/src/services/tool-access.ts @@ -1,4 +1,5 @@ import { notifyGitHubCredentialChange } from "./github-connection-notifications.js"; +import { RepositoryCatalogUnavailableError, findRepositoryCatalogUnavailableError } from "./repository-catalog-unavailable-error.js"; import { notifyVoiceChange } from "./voice/voice-recovery-notifications.js"; import { notifyToolConnectionHealth, writeToolConnectionHealthChange } from "./tool-connection-health-notifications.js"; import { connectionHealthSchedule } from "./tool-access/catalog/health-schedule.js"; @@ -4277,8 +4278,9 @@ export function toolAccessService( // Repository discovery uses credential audiences, not connection-management // visibility. An administrator cannot browse another user's private repos. - listProjectRepositories: async (companyId: string, userId: string | null, localTrusted = false) => - toolAccessService(db).listGitHubRepositories(companyId, userId, localTrusted), + listProjectRepositories: async (companyId: string, userId: string | null, localTrusted = false, + options?: { requiredRepositoryIds: readonly string[] }) => + toolAccessService(db).listGitHubRepositories(companyId, userId, localTrusted, options), // Return identifiers, never credentials, so each attempted read can recheck its grant. githubReadConnectionIds: async (companyId: string, userId: string | null, localTrusted = false): Promise => { @@ -4322,6 +4324,7 @@ export function toolAccessService( companyId: string, userId: string | null, localTrusted = false, + options?: { requiredRepositoryIds: readonly string[] }, ) => measureRunPhase("connectors.github.catalog", async () => { const [connections, grants, members, memberships] = await Promise.all([ measureRunPhase("connectors.github.connections", () => db @@ -4362,6 +4365,7 @@ export function toolAccessService( const usableConnections: Array<{ id: string; name: string }> = []; let connectionCount = 0; let failedConnectionCount = 0; + const unavailable: RepositoryCatalogUnavailableError[] = []; for (const connection of connections) { if ( connection.status !== "active" || @@ -4445,7 +4449,9 @@ export function toolAccessService( for (const row of rows) { mergeProjectRepository(repositories, row, connection.name, connection.id); } - } catch { + } catch (error) { + const catalogFailure = findRepositoryCatalogUnavailableError(error); + if (catalogFailure) unavailable.push(catalogFailure); // Credential/provider errors may contain secrets. Only expose an // aggregate failure; successful connections remain usable. failed = true; @@ -4453,6 +4459,10 @@ export function toolAccessService( } if (failed) failedConnectionCount += 1; } + const availableIds = new Set([...repositories.values()].map(repository => repository.id)); + if (unavailable.length && options?.requiredRepositoryIds.some(id => !availableIds.has(id))) { + throw unavailable.reduce((latest, failure) => (failure.retryAt ?? 0) > (latest.retryAt ?? 0) ? failure : latest); + } return { repositories: [...repositories.values()].sort((a, b) => a.fullName.localeCompare(b.fullName), diff --git a/server/src/services/tool-access/github-access-discovery.ts b/server/src/services/tool-access/github-access-discovery.ts index 2d45377ab6..8c07a6fc4f 100644 --- a/server/src/services/tool-access/github-access-discovery.ts +++ b/server/src/services/tool-access/github-access-discovery.ts @@ -1,6 +1,7 @@ import { providerRetryAt } from "../provider-retry-after.js"; import { randomUUID } from "node:crypto"; import { unprocessable } from "../../errors.js"; +import { readGitHubRepositoryCatalogFailure } from "./github-repository-catalog-failure.js"; function gitHubRepositoryOwnerType(repository: Record) { const type = recordValue(repository.owner) ? repository.owner.type : undefined; @@ -31,10 +32,13 @@ export async function loadGitHubTokenRepositories( signal: AbortSignal.timeout(15_000), }, ); - if (!response.ok) + if (!response.ok) { + const unavailable = readGitHubRepositoryCatalogFailure(response); + if (unavailable) throw unavailable; throw unprocessable( "Could not load GitHub repositories. Reconnect GitHub and try again.", ); + } const rows: unknown = await response.json(); if (!Array.isArray(rows)) throw unprocessable("GitHub returned invalid repositories"); @@ -104,16 +108,19 @@ export async function loadGitHubGrantMetadata( signal: AbortSignal.timeout(15_000), }); if (!response.ok) { - throw unprocessable( + const unavailable = readGitHubRepositoryCatalogFailure(response); + const failure = unprocessable( "GitHub could not verify this account's installation access", { - retryAt: providerRetryAt(response.headers.get("retry-after")), + retryAt: unavailable?.retryAt ?? providerRetryAt(response.headers.get("retry-after")), code: response.status === 401 ? "oauth_reauthorization_required" : "github_access_check_failed", }, ); + if (unavailable) failure.cause = unavailable; + throw failure; } const value = (await response.json()) as unknown; if (!recordValue(value)) diff --git a/server/src/services/tool-access/github-repository-catalog-failure.ts b/server/src/services/tool-access/github-repository-catalog-failure.ts new file mode 100644 index 0000000000..1d3b7b4ec9 --- /dev/null +++ b/server/src/services/tool-access/github-repository-catalog-failure.ts @@ -0,0 +1,16 @@ +import { providerRetryAt } from "../provider-retry-after.js"; +import { RepositoryCatalogUnavailableError } from "../repository-catalog-unavailable-error.js"; + +const transientStatuses = new Set([408, 425, 429, 500, 502, 503, 504]); + +export function readGitHubRepositoryCatalogFailure(response: Pick, now = Date.now()) { + const exhausted = response.status === 403 && response.headers.get("x-ratelimit-remaining") === "0"; + const retryAfter = providerRetryAt(response.headers.get("retry-after"), now); + if (!exhausted && !(response.status === 403 && retryAfter !== undefined) && !transientStatuses.has(response.status)) return null; + const rawReset = response.headers.get("x-ratelimit-reset"); + const reset = rawReset && /^\d{1,13}$/.test(rawReset) ? Number(rawReset) * 1000 : undefined; + const quotaReset = (exhausted || response.status === 429) && reset && reset > now + ? Math.min(reset, now + 24 * 3600_000) : undefined; + const retryAt = retryAfter === undefined ? quotaReset : Math.max(retryAfter, quotaReset ?? retryAfter); + return new RepositoryCatalogUnavailableError(retryAt); +}