mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-07 06:16:06 -10:00
Calibrate four chat oracles and qualify coherent candidate
This commit is contained in:
1 parent
2d8711e783
commit
b59e295c7f
12 files changed
+87
-11
No files matched your search
@@ -5731,7 +5731,7 @@ registry.registerPath({
|
||||
method: "post",
|
||||
path: "/api/approvals/{id}/resubmit",
|
||||
tags: ["approvals"],
|
||||
summary: "Revise and resubmit an approval or proposed hire",
|
||||
summary: "Revise and resubmit a changes-requested approval",
|
||||
description:
|
||||
"Replace the payload of an approval whose status is revision_requested, or resubmit its existing payload. " +
|
||||
"Use this operation to revise a changes-requested hire proposal, such as changing its agent configuration to a cheaper model, and send it back for board review. " +
|
||||
|
||||
@@ -175,7 +175,7 @@ describe("runner API catalog", () => {
|
||||
expect(previewBody.properties).not.toHaveProperty("adapterOverrides");
|
||||
});
|
||||
|
||||
it("discovers proposal revision without confusing it with approval decisions or reads", () => {
|
||||
it("discovers proposal revision without confusing it with hiring, approval decisions, or reads", () => {
|
||||
const resubmit = "POST /api/approvals/{id}/resubmit";
|
||||
for (const query of [
|
||||
"revise changes-requested hire proposal",
|
||||
@@ -184,6 +184,8 @@ describe("runner API catalog", () => {
|
||||
]) expect(searchRunnerApi({ query }).results.map(row => row.operationId), query).toContain(resubmit);
|
||||
for (const [query, expected] of [
|
||||
["GET /api/approvals/{id}", "GET /api/approvals/{id}"],
|
||||
["get approval", "GET /api/approvals/{id}"],
|
||||
["hire agent", "POST /api/companies/{companyId}/agent-hires"],
|
||||
["approve approval", "POST /api/approvals/{id}/approve"],
|
||||
["reject approval", "POST /api/approvals/{id}/reject"],
|
||||
[resubmit, resubmit],
|
||||
|
||||
@@ -564,3 +564,69 @@ with cap 2, zero automatic retries and original 480000ms/660000ms cells. Team re
|
||||
calibrated locally; the final uniform ten is still required on this same source.
|
||||
All original grades, historical conditions, unmerged core holds and the 196/244
|
||||
full-catalog gate remain intact.
|
||||
|
||||
|
||||
Prospective timeline and provider-routing conditions
|
||||
|
||||
Timeline v13 accepts a same-identified-task account that orders the first worker's
|
||||
real command failure and pause before the different saved repair worker's verified
|
||||
repair, without requiring an explicit assignment phrase. Independent same-task
|
||||
saved handoff, actor, command, chronology, link, privacy and read-only proofs remain
|
||||
mandatory. Launch 3376's original timeline FAILED result remains unchanged; its
|
||||
completed truthful account motivated this prospective calibration, not a regrade.
|
||||
|
||||
The reviewed OpenCode chat profile now supplies model-specific
|
||||
`OPENCODE_CONFIG_CONTENT` selecting OpenRouter `provider.sort: "throughput"` only
|
||||
for `deepseek/deepseek-v4.1-flash-1017`. This changes the profile environment and
|
||||
provider-routing condition: it is a PROSPECTIVE performance experiment, with
|
||||
benefit unknown. The exact model, default reasoning behavior, permissions, output
|
||||
options, auxiliary-model configuration and all original native/human bounds and
|
||||
clocks are unchanged. Previous default-routing failures are retained; no runtime
|
||||
latency repair, reliability improvement or new native pass is claimed.
|
||||
|
||||
Approval resubmission metadata describes the existing revision/resubmission
|
||||
operation and its unchanged authority boundary so ordinary API discovery can find
|
||||
it. This does not approve or execute a hire. Candidate core changes remain open
|
||||
and unmerged; the 196/244 full-catalog gate remains held. All 244 YAML identities,
|
||||
queries, contexts and outcomes remain unchanged. Combined qualification and any
|
||||
next native measurement remain pending the coherent release decision.
|
||||
|
||||
Personal secret v17 accepts an asserted current personal value, key, token or
|
||||
credential that worked or works. This is a narrative calibration, not permission
|
||||
to count save-only, planned, negated, unrelated-task or another person's success.
|
||||
The original launch 3376 failure remains FAILED even though its plain-comment
|
||||
path reached secure current-owner save, actual worker delivery with a 200 receipt
|
||||
and a separate CEO acknowledgment. Independent consumer delivery, actor/owner
|
||||
isolation, receipt, privacy and no-leak proofs remain mandatory. No new native
|
||||
pass or latency improvement is claimed; the original three native/two human inputs
|
||||
and 660000ms cell remain unchanged. Timeline and Personal are two of the four oracle version
|
||||
leaves changed in this prospective batch.
|
||||
|
||||
Skill authoring v6 observes the four required checklist points within their
|
||||
actual heading-and-list sections rather than requiring every term in one text
|
||||
block. The launch 3376 native authored and saved the skill, but its original
|
||||
FAILED observer stopped before UI reload/readback qualification. This repair
|
||||
provides no native pass credit: actual file, author, version, editable Studio,
|
||||
reload and matching readback proofs remain required. Unrelated sections, missing
|
||||
requirements and waivers remain rejected. This is the third prospective oracle
|
||||
version leaf; skill review/update is the fourth calibration below.
|
||||
|
||||
Any next throughput-routing measurement first uses a small representative
|
||||
announcement case before expanding to the uniform ten. That experiment remains
|
||||
unrun and requires a separate release decision; neither local projection nor
|
||||
these assertion calibrations demonstrate a performance benefit.
|
||||
|
||||
|
||||
Skill review/update v3 preserves a complete persisted preview on its original
|
||||
path. Only an incomplete persisted preview may fall back to the current rendered
|
||||
final response, anchored after the actual human review request and corroborated
|
||||
by the same committed successful native run. Hidden, stale, foreign, reasoning,
|
||||
tool-output and API-only summaries remain excluded. Hydration waits consume the
|
||||
original observation deadline without renewing it. The later human apply request,
|
||||
actual upstream version change and unchanged local fork remain mandatory. Launch
|
||||
3376's original FAILED review/update result stays unchanged; its later apply
|
||||
phase was never reached, so this calibration supplies no completed native pass.
|
||||
Together the four prospective oracle leaves are Timeline v13, Personal v17,
|
||||
Skill authoring v6 and Integrator v3; every other descriptor field and sourceFiles
|
||||
array remains unchanged. The routing experiment remains unrun and its performance
|
||||
benefit is unproved. No main pass credit or release acceptance is claimed.
|
||||
@@ -259,7 +259,7 @@
|
||||
"../../server/src/services/work-timeline.ts",
|
||||
"../../server/src/services/native-runtime/runner-api-catalog.ts"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.work-timeline.v12",
|
||||
"oracleVersion": "paperclip.chat-catalog.work-timeline.v13",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 3,
|
||||
"minimumRunCount": 3,
|
||||
@@ -5009,7 +5009,7 @@
|
||||
"../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx",
|
||||
"chat-scenarios/secrets.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.personal-secret.v16",
|
||||
"oracleVersion": "paperclip.chat-catalog.personal-secret.v17",
|
||||
"expectedRunCount": 3,
|
||||
"expectedHumanMessageCount": 2,
|
||||
"minimumRunCount": 3,
|
||||
@@ -5113,7 +5113,7 @@
|
||||
"../../ui/src/pages/CompanySkills.tsx",
|
||||
"../../ui/src/lib/company-skill-routes.ts"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.skill-authoring.v5",
|
||||
"oracleVersion": "paperclip.chat-catalog.skill-authoring.v6",
|
||||
"expectedRunCount": 1,
|
||||
"expectedHumanMessageCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
@@ -5231,7 +5231,7 @@
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/skills.yaml"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.v2",
|
||||
"oracleVersion": "paperclip.chat-catalog.integrator.v3",
|
||||
"expectedRunCount": 2,
|
||||
"expectedHumanMessageCount": 2,
|
||||
"turnTimeoutMs": 120000,
|
||||
|
||||
@@ -3,6 +3,7 @@ import { questionSetToAskUserQuestionsPayload } from "../../../packages/shared/s
|
||||
import { expect, it } from "vitest";
|
||||
import {
|
||||
gradePersonalSecret,
|
||||
personalSecretExternalCases,
|
||||
personalSecretGuidance,
|
||||
personalSecretConfirmation,
|
||||
personalSecretQuestionHandoff,
|
||||
@@ -316,3 +317,8 @@ it("keeps successful worked wording dependent on actual worker delivery",()=>{
|
||||
o.state.workerReceiptBound=false;
|
||||
expect(gradePersonalSecret(o).passed).toBe(false);
|
||||
});
|
||||
|
||||
it("declares the prospective personal acknowledgment calibration without changing its bounds", () => {
|
||||
expect(personalSecretExternalCases[0]).toMatchObject({ oracleVersion: "paperclip.chat-catalog.personal-secret.v17",
|
||||
expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 660000 });
|
||||
});
|
||||
@@ -461,7 +461,7 @@ export const personalSecretExternalCases = [
|
||||
if (!o.state.initialQuestionId) return;
|
||||
return { interactionId: String(o.state.initialQuestionId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId) };
|
||||
},
|
||||
oracleVersion: "paperclip.chat-catalog.personal-secret.v16",
|
||||
oracleVersion: "paperclip.chat-catalog.personal-secret.v17",
|
||||
setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] },
|
||||
expectedRunCount: 3,
|
||||
minimumRunCount: 3,
|
||||
|
||||
@@ -62,6 +62,7 @@ it("rejects invented inspection receipts and keeps other skill cases on their or
|
||||
f.state.authoredSkillInspection={companyId:"company",skillId:"other",versionId:"revision",libraryVisible:true,studioVisible:true,editable:true,checklistVisible:true,reloaded:true,readbackMatched:true};
|
||||
expect(gradeIntegratorSkills(f.o).passed).toBe(false);
|
||||
expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.explicit-restriction")!.oracleVersion).toBe("paperclip.chat-catalog.integrator.v1");
|
||||
expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.author-skill")!.oracleVersion).toBe("paperclip.chat-catalog.skill-authoring.v6");
|
||||
});
|
||||
it("cannot extend an exhausted original case budget to inspect a finished artifact",async()=>{
|
||||
const f=fixture();Object.assign(f.ctx,{observationDeadlineAt:Date.now()-1});
|
||||
|
||||
@@ -130,6 +130,7 @@ it("actual review-update setup keeps the real prerequisite and then prepares the
|
||||
vi.stubGlobal("fetch", vi.fn(async (url: string) => new Response(url.includes(pins.oldCommit) ? bytes.old : bytes.next)));
|
||||
try {
|
||||
const f = setupFixture(false); const result = await integratorSkillUpdateCases[0]!.setup(f.ctx);
|
||||
expect(integratorSkillUpdateCases[0]!.oracleVersion).toBe("paperclip.chat-catalog.integrator.v3");
|
||||
expect(f.prerequisite).toHaveBeenCalledOnce(); expect(saveSkillConversationContext).toHaveBeenCalledOnce(); expect(result.source).toMatchObject({ trackingRef: pins.newCommit, lastScanCommit: pins.oldCommit });
|
||||
} finally { vi.unstubAllGlobals(); }
|
||||
});
|
||||
@@ -163,7 +163,7 @@ export const integratorSkillUpdateCases = [integratorDefinition("skills", "skill
|
||||
const selected = await saveSkillConversationContext(ctx, { description: `Selected upstream accessibility skill: ${upstream.name} (${upstream.id}), tracked source ${source.id}. Installed source revision ${upstream.sourceRef}; tracking reference ${source.trackingRef}. Separate editable local copy: ${localFork.name} (${localFork.id}).` });
|
||||
return { upstream, localFork, source, factualContextAttachment: selected.factualContextAttachment, localVersions: await readSkillVersionHistory(ctx.input.api, ctx.input.fixtures.company.id, localFork.id, localFork.currentVersionId) };
|
||||
}, observeUpdate, gradeIntegratorSkillUpdate, async (ctx, state) => { const skill = state.upstream as CompanySkillDetail; await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/skills/studio/${skill.id}`, skill.name); },
|
||||
{ oracleVersion: "paperclip.chat-catalog.integrator.v2", authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md"), expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 540_000,
|
||||
{ oracleVersion: "paperclip.chat-catalog.integrator.v3", authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md"), expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 540_000,
|
||||
sourceFiles: ["chat-catalog-domains/identity-support.ts", "chat-catalog-domains/integrator-skill-versions.ts", "chat-catalog-domains/integrator-skill-update.ts", "chat-catalog-domains/integrator-skill-source.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/skills.yaml"],
|
||||
async act(ctx, state) {
|
||||
updateReceipts.delete(ctx);
|
||||
|
||||
@@ -201,7 +201,7 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor
|
||||
const define = (id: string, setup: ChatCatalogCase["setup"]): ChatCatalogCase => {
|
||||
const base = integratorDefinition("skills", id, setup, observeSkills, gradeIntegratorSkills, verifySkill);
|
||||
base.sourceFiles = [...base.sourceFiles, "chat-catalog-domains/integrator-skill-versions.ts"];
|
||||
return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v5", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", "../../ui/src/pages/SkillStudio.tsx", "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"),
|
||||
return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v6", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", "../../ui/src/pages/SkillStudio.tsx", "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"),
|
||||
async act(ctx, state) { remainingIdentityObservationBudget(ctx); await waitForReply(ctx, ctx.observationDeadlineAt); await inspectAuthoredSkill(ctx, state, await observeSkills(ctx, state)); },
|
||||
async verifyResult(ctx, state, value) { const o = observation(value); if (!o) throw new Error("Missing authored skill inspection evidence"); await inspectAuthoredSkill(ctx, state, o); },
|
||||
} : base;
|
||||
|
||||
@@ -269,7 +269,7 @@ it("admits the selected live timeline metadata without invoking fixture callback
|
||||
const spies = callbacks.map(key => vi.spyOn(implementation, key));
|
||||
try {
|
||||
expect(await validateChatCatalogConsistency([descriptor])).toEqual({ descriptorCount: 1, fixtureCallbacksInvoked: 0 });
|
||||
expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v12");
|
||||
expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v13");
|
||||
expect(descriptor.sourceFiles).toEqual(implementation.sourceFiles);
|
||||
expect(descriptor).toMatchObject({ expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000 });
|
||||
for (const spy of spies) expect(spy).not.toHaveBeenCalled();
|
||||
|
||||
@@ -92,7 +92,7 @@ function timeline(): ChatCatalogCase {
|
||||
const state = { worker, repairWorker, project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id, personName: session.user.name };
|
||||
await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state;
|
||||
}, gradeWorkTimeline);
|
||||
return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v12", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000,
|
||||
return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v13", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000,
|
||||
sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts","../../server/src/services/native-runtime/runner-api-catalog.ts"],
|
||||
async observe(ctx, state): Promise<Timeline> { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; },
|
||||
async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } },
|
||||
|
||||
Reference in new issue
Block a user