chore(evals): declare calibrated secret acknowledgments and skill caveats

Co-Authored-By: Paperclip <[email protected]>
This commit is contained in:
DottaandPaperclip committed 2026-11-06 03:23:24 -06:00
1 parent f60ad9c6ae
commit d68c00a3d7
5 files changed
+36 -6

No files matched your search

+30
View File
@@ -534,3 +534,33 @@ permissions, human/native bounds and clocks remain unchanged. Cap 2 and zero
retries remain the controller policy. Existing candidate core overlays stay
unmerged, and the held full-catalog gate remains 196 of 244. The next native
measurement is still required; none of these local controls regrades an original.
The next secret-pair measurement retains launch 3310's one original PASS and three
original FAILURES. Multipart question v6 was independently verified on the same
form without dismissal or replacement, with exact real answers and one continuation
in 241214ms of its original 300000ms cell. This does not prove reliability or isolate
the fixture change as the sole cause.
Company secret v17 recognizes an ordered secure-save acknowledgment across bound
explanatory clauses, while rejecting a declined save or acknowledgment. Its original
initial native succeeded in 344242ms; only 86070ms remained at the false pre-human
gate. Measured handlers do not explain that latency, and no avoidable latency bug
was proved. Human entry and later native turns remain unqualified.
Personal secret v16 preserves the canonical saved confirmation policy through real
secure entry, acceptance and independent accounting. Both supported wake policies
must match the original saved card exactly. The actual wake-on-accept/After Saving
card was rejected before private entry; canceled or negated acknowledgments remain
rejected. Team install v10 accepts a bounded qualifier on the exact required-skill
subject for truthful partial-installation reporting. Its local calibration does
not regrade original 5242 or provide runtime-migration credit; the fifteen other
saved-state and phase checks remain required.
Only Company, Personal and Team versions change in this batch. All 244 YAML
queries, contexts, outcomes, permissions, human/native bounds, model and clocks
remain fixed. The next measurement is only the Company and Personal secret pair,
with cap 2, zero automatic retries and original 480000ms/660000ms cells. Team remains
calibrated locally; the final uniform ten is still required on this same source.
All original grades, historical conditions, unmerged core holds and the 196/244
full-catalog gate remain intact.
@@ -4923,7 +4923,7 @@
"chat-flow.ts",
"chat-scenarios/secrets.yaml"
],
"oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v16",
"oracleVersion": "paperclip.chat-catalog.company-secret-controller-baseline.v17",
"expectedRunCount": 3,
"minimumRunCount": 2,
"expectedHumanMessageCount": 2,
@@ -5009,7 +5009,7 @@
"../../ui/src/pages/secrets/SetMyUserSecretDialog.tsx",
"chat-scenarios/secrets.yaml"
],
"oracleVersion": "paperclip.chat-catalog.personal-secret.v15",
"oracleVersion": "paperclip.chat-catalog.personal-secret.v16",
"expectedRunCount": 3,
"expectedHumanMessageCount": 2,
"minimumRunCount": 3,
@@ -6046,7 +6046,7 @@
"chat-flow.ts",
"chat-scenarios/teams.yaml"
],
"oracleVersion": "paperclip.chat-catalog.team-install.v9",
"oracleVersion": "paperclip.chat-catalog.team-install.v10",
"expectedRunCount": 2,
"expectedHumanMessageCount": 2,
"turnTimeoutMs": 120000,
@@ -215,7 +215,7 @@ export const companySecretExternalCases=[definition("secrets.configure-company-s
},
optionalCompanySecretBindingApproval:true,
expectedCompanySecretBindingApproval(value){if(!gradeCompanySecret(value).passed)return;return companySecretApprovalProof(value);},
oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v16",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000,
oracleVersion:"paperclip.chat-catalog.company-secret-controller-baseline.v17",expectedRunCount:3,minimumRunCount:2,expectedHumanMessageCount:2,attemptTimeoutMs:480000,
sourceFiles:["../../server/src/routes/secrets.ts","../../packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts","../../packages/paperclip-runner/src/cli/opencode-proxy-finish-diagnostics.ts","../../packages/paperclip-runner/src/protocol-actions/request-human-input.ts","chat-catalog-domains/external-company-secret.ts","chat-catalog-domains/external-company-secret-confirmation.ts","chat-catalog-domains/external-secret-proposal.ts","chat-catalog-domains/external-secret-question.ts","chat-catalog-domains/external-secret-route.ts","chat-catalog-domains/external-company-secret-menu.ts","chat-catalog-contract.ts","chat-catalog-flow.ts","chat-catalog-domains/external-private-browser.ts","chat-catalog-domains/external-private-browser-worker.ts","chat-catalog-domains/external-credential-language.ts","chat-catalog-domains/external-secret-scan.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","live-fixtures.ts","../../server/src/services/secrets.ts","../../server/src/services/native-runtime/runner-api-catalog.ts","../../server/src/routes/issues.ts","run-observations.ts","chat-flow.ts","chat-scenarios/secrets.yaml"],
async act(ctx,state){
const deadline=chatCatalogOriginalPhaseDeadline(ctx),remaining=()=>remainingIdentityObservationBudget(ctx);
@@ -462,7 +462,7 @@ export const personalSecretExternalCases = [
if (!o.state.initialQuestionId) return;
return { interactionId: String(o.state.initialQuestionId), sourceRunId: String(o.state.initialRunId), userId: String(o.state.currentPersonId) };
},
oracleVersion: "paperclip.chat-catalog.personal-secret.v15",
oracleVersion: "paperclip.chat-catalog.personal-secret.v16",
setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] },
expectedRunCount: 3,
minimumRunCount: 3,
@@ -346,7 +346,7 @@ export const teamInstallExternalCases = [
gradeTeamInstall,
{
authentication: "human-session",
oracleVersion: "paperclip.chat-catalog.team-install.v9",
oracleVersion: "paperclip.chat-catalog.team-install.v10",
expectedRunCount: 2,
expectedHumanMessageCount: 2,
attemptTimeoutMs: 480_000,