mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-07 17:10:43 -10:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 esengine
|
95da89e081 |
No files matched your search
@@ -26,8 +26,6 @@
|
||||
*.xml text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.yml text eol=lf
|
||||
Makefile text eol=lf
|
||||
.githooks/** text eol=lf
|
||||
benchmarks/**/*.sh text eol=lf
|
||||
internal/runtime/agent/testdata/perseveration/** -text
|
||||
desktop/frontend-next/perf/media/delivery.pdf binary
|
||||
@@ -1,7 +0,0 @@
|
||||
# Logins allowed to open pull requests, one per line, matched case-insensitively.
|
||||
# Repository owners, members and collaborators, and bots, are always allowed.
|
||||
lancy69
|
||||
chuiwenwei
|
||||
xxoingr
|
||||
kevin9327
|
||||
imcreazy
|
||||
@@ -2,9 +2,6 @@
|
||||
Target branch: 2.x features and fixes go to `studio`. 1.x takes bug fixes,
|
||||
provider/API compatibility, release/updater, security and platform stability
|
||||
only, on `main-v2`.
|
||||
|
||||
Pull requests are limited to maintainers and invited contributors; one from
|
||||
anyone else is closed automatically. Open an issue instead.
|
||||
-->
|
||||
|
||||
## Summary
|
||||
|
||||
@@ -1,293 +0,0 @@
|
||||
name: Android
|
||||
|
||||
# The Android shell under `android/` ships from the same tag the desktop line
|
||||
# publishes, and lands on the same release page.
|
||||
#
|
||||
# It is a separate workflow rather than a job inside release-studio.yml for two
|
||||
# reasons. The toolchains share nothing — Go, the Android SDK, a Gradle wrapper —
|
||||
# so a runner carrying both would install three toolchains to build one artifact.
|
||||
# And `signing-contract` pins which workflows may reach the SignPath credentials;
|
||||
# this one never touches them, and keeping it out of that graph is what keeps the
|
||||
# fingerprint stable.
|
||||
#
|
||||
# A failure here does not hold back the desktop release: the two are independent
|
||||
# by construction, and an APK is worth less than the desktop builds it sits
|
||||
# beside.
|
||||
on:
|
||||
push:
|
||||
tags: ['studio-v*']
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Existing studio tag to build for and attach to (for example studio-v2.51.0)"
|
||||
required: false
|
||||
type: string
|
||||
ref:
|
||||
description: "A branch to build as a trial run of this workflow; nothing is attached to any release"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write # upload the APK to the release the tag belongs to
|
||||
|
||||
concurrency:
|
||||
group: android-${{ inputs.tag || inputs.ref || github.ref_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
resolve:
|
||||
name: resolve the tag
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
tag: ${{ steps.release.outputs.tag }}
|
||||
version: ${{ steps.release.outputs.version }}
|
||||
sha: ${{ steps.release.outputs.sha }}
|
||||
candidate: ${{ steps.release.outputs.candidate }}
|
||||
publish: ${{ steps.release.outputs.publish }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
# The same tag grammar release-studio.yml enforces, spelled the same way:
|
||||
# two workflows that disagree about which tags name a release would publish
|
||||
# one artifact for a tag the other refused.
|
||||
- name: Resolve tag and version
|
||||
id: release
|
||||
env:
|
||||
IN_TAG: ${{ inputs.tag }}
|
||||
IN_REF: ${{ inputs.ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
run: |
|
||||
# A trial run builds a branch as a throwaway version and publishes nothing.
|
||||
if [ -n "$IN_REF" ] && [ -z "$IN_TAG" ]; then
|
||||
sha="$(git rev-parse --verify "origin/$IN_REF^{commit}")"
|
||||
{ echo "tag="; echo "version=v0.0.0-dev.${sha:0:7}"; echo "sha=$sha"
|
||||
echo "candidate=false"; echo "publish=false"; } >> "$GITHUB_OUTPUT"
|
||||
echo "trial build of $IN_REF at $sha"
|
||||
exit 0
|
||||
fi
|
||||
tag="${IN_TAG:-$REF_NAME}"
|
||||
semver='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-([0-9A-Za-z-]+)(\.[0-9A-Za-z-]+)*)?$'
|
||||
if [[ ! "$tag" =~ ^studio-v(.+)$ ]] || [[ ! "${BASH_REMATCH[1]}" =~ $semver ]]; then
|
||||
echo "::error::studio release tag must be studio-vMAJOR.MINOR.PATCH[-PRERELEASE], got: $tag"
|
||||
exit 1
|
||||
fi
|
||||
sha="$(git rev-parse --verify "refs/tags/$tag^{commit}")"
|
||||
if ! git merge-base --is-ancestor "$sha" origin/studio; then
|
||||
echo "::error::$tag is not on studio history"
|
||||
exit 1
|
||||
fi
|
||||
version="${tag#studio-}"
|
||||
candidate=false
|
||||
if [[ "${version#v}" =~ $semver ]] && [ -n "${BASH_REMATCH[4]}" ]; then
|
||||
candidate=true
|
||||
fi
|
||||
{ echo "tag=$tag"; echo "version=$version"; echo "sha=$sha"
|
||||
echo "candidate=$candidate"; echo "publish=true"; } >> "$GITHUB_OUTPUT"
|
||||
echo "resolved $tag -> $version at $sha (candidate=$candidate)"
|
||||
|
||||
build:
|
||||
name: build the APK
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
# `publish` decides whether to attach anything, and the only honest answer to
|
||||
# "will this APK install" is whether a key was restored.
|
||||
outputs:
|
||||
signed: ${{ steps.key.outputs.signed }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
cache-dependency-path: go.sum
|
||||
|
||||
# pnpm first: setup-node's pnpm cache needs the executable to exist.
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v4
|
||||
with:
|
||||
version: 10
|
||||
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: desktop/frontend-next/pnpm-lock.yaml
|
||||
|
||||
- uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: "21"
|
||||
cache: gradle
|
||||
|
||||
# The SDK is the one thing nothing sets for you, per android/README.md. The
|
||||
# runner image carries one; this asserts the two pieces the build names
|
||||
# rather than re-installing them, so a missing package fails here by name
|
||||
# instead of surfacing later as an unresolved compileSdk.
|
||||
- name: Provide the Android SDK
|
||||
run: |
|
||||
if [ -z "${ANDROID_HOME:-}" ]; then
|
||||
echo "::error::ANDROID_HOME is unset; the runner image is expected to provide the Android SDK"
|
||||
exit 1
|
||||
fi
|
||||
for piece in "platforms/android-35" "build-tools/35.0.0"; do
|
||||
if [ ! -d "$ANDROID_HOME/$piece" ]; then
|
||||
echo "::error::the Android SDK at $ANDROID_HOME has no $piece; android/README.md names it as a requirement"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "ANDROID_SDK_ROOT=$ANDROID_HOME" >> "$GITHUB_ENV"
|
||||
|
||||
# A key that exists signs; a key that does not leaves the artifact
|
||||
# unsigned, which is what a release with no key has always produced. The
|
||||
# decision is the repository's, not this workflow's: it reads the four
|
||||
# secrets android/README.md already names.
|
||||
- name: Restore the signing key
|
||||
id: key
|
||||
env:
|
||||
KEYSTORE_BASE64: ${{ secrets.REASONIX_KEYSTORE_BASE64 }}
|
||||
STORE_PASSWORD: ${{ secrets.REASONIX_KEYSTORE_PASSWORD }}
|
||||
KEY_PASSWORD: ${{ secrets.REASONIX_KEY_PASSWORD }}
|
||||
KEY_ALIAS: ${{ secrets.REASONIX_KEY_ALIAS }}
|
||||
run: |
|
||||
if [ -z "$KEYSTORE_BASE64" ] || [ -z "$STORE_PASSWORD" ] || [ -z "$KEY_ALIAS" ]; then
|
||||
echo "signed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "no signing key configured: the APK will be unsigned"
|
||||
exit 0
|
||||
fi
|
||||
mkdir -p android/keystore
|
||||
printf '%s' "$KEYSTORE_BASE64" | base64 --decode > android/keystore/release.jks
|
||||
{
|
||||
echo "storeFile=keystore/release.jks"
|
||||
echo "storePassword=${STORE_PASSWORD}"
|
||||
echo "keyAlias=${KEY_ALIAS}"
|
||||
# PKCS#12 keystores use one password for both; only write the second
|
||||
# when it is genuinely a different fact.
|
||||
[ -n "$KEY_PASSWORD" ] && echo "keyPassword=${KEY_PASSWORD}"
|
||||
} > android/keystore/keystore.properties
|
||||
chmod 600 android/keystore/*
|
||||
echo "signed=true" >> "$GITHUB_OUTPUT"
|
||||
echo "signing key restored"
|
||||
|
||||
- name: Build the kernel and the page
|
||||
working-directory: android
|
||||
env:
|
||||
VERSION: ${{ needs.resolve.outputs.version }}
|
||||
run: |
|
||||
export REASONIX_APP_VERSION="${VERSION#v}"
|
||||
bash ./scripts/build-kernel.sh
|
||||
bash ./scripts/sync-page.sh
|
||||
|
||||
# The one end-to-end check that needs no phone: the kernel the shell starts,
|
||||
# serving the page the APK carries. Running it here means an APK that would
|
||||
# open blank fails the build instead of the install.
|
||||
- name: Verify the packaged page
|
||||
working-directory: android
|
||||
run: |
|
||||
(cd .. && go build -o reasonix-verify ./cmd/reasonix)
|
||||
KERNEL_BIN="$PWD/../reasonix-verify" bash ./scripts/verify-page.sh
|
||||
|
||||
- name: Assemble
|
||||
working-directory: android
|
||||
env:
|
||||
VERSION: ${{ needs.resolve.outputs.version }}
|
||||
run: |
|
||||
# The version comes from the tag, so the About screen and the update
|
||||
# check name this release. versionCode keeps its existing shape, which
|
||||
# is why the build digit stays 1: it is a single decimal place, and a
|
||||
# number that grows past it would collide with the next patch.
|
||||
./gradlew assembleRelease --no-daemon "-PreasonixVersion=${VERSION#v}"
|
||||
|
||||
- name: Name the artifact
|
||||
id: apk
|
||||
working-directory: android/app/build/outputs/apk/release
|
||||
run: |
|
||||
src="$(ls ./*.apk | head -n 1)"
|
||||
if [ -z "$src" ]; then
|
||||
echo "::error::assembleRelease produced no APK"
|
||||
exit 1
|
||||
fi
|
||||
case "$src" in
|
||||
*-unsigned.apk) name="ReasonixMobile-${{ needs.resolve.outputs.version }}-unsigned.apk" ;;
|
||||
*) name="ReasonixMobile-${{ needs.resolve.outputs.version }}.apk" ;;
|
||||
esac
|
||||
mv "$src" "$name"
|
||||
sha256sum "$name" > "$name.sha256"
|
||||
{
|
||||
echo "path=$PWD/$name"
|
||||
echo "name=$name"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
ls -l "$name" "$name.sha256"
|
||||
|
||||
# An unsigned package is not a lesser APK, it is an unusable one: Android's
|
||||
# package manager rejects it outright rather than reporting a signature
|
||||
# mismatch. Say so where the run summary can be read, instead of letting a
|
||||
# green build imply a shippable artifact.
|
||||
- name: Say what an unsigned build means
|
||||
if: steps.key.outputs.signed != 'true'
|
||||
run: |
|
||||
echo "::warning::no signing key configured, so this APK is unsigned. Android refuses to install an unsigned package, so the release will carry no APK; the file stays a workflow artifact for whoever signs it."
|
||||
{
|
||||
echo "### No APK on this release"
|
||||
echo ""
|
||||
echo "The build produced an unsigned APK, which Android refuses to install."
|
||||
echo "It is attached to this run as a workflow artifact only. Add the four"
|
||||
echo "signing secrets to publish one."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v8
|
||||
with:
|
||||
name: android-apk
|
||||
path: |
|
||||
android/app/build/outputs/apk/release/*.apk
|
||||
android/app/build/outputs/apk/release/*.sha256
|
||||
if-no-files-found: error
|
||||
|
||||
publish:
|
||||
name: attach the APK
|
||||
needs: [resolve, build]
|
||||
# Nothing to attach unless the build was signed. Skipping is the honest
|
||||
# outcome: a release page with an APK nobody can install is worse than a
|
||||
# release page without one.
|
||||
if: needs.build.outputs.signed == 'true' && needs.resolve.outputs.publish == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: android-apk
|
||||
path: dist
|
||||
|
||||
# The release itself belongs to release-studio.yml, which creates it and
|
||||
# owns its notes. This waits for it rather than creating a second one: two
|
||||
# jobs racing to define a release is how a page ends up with the wrong body.
|
||||
- name: Attach to the release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ needs.resolve.outputs.tag }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
# 90 minutes: on the last Studio release the page appeared about 40
|
||||
# minutes after the tag run started, and the APK build spends part of
|
||||
# that before this wait begins.
|
||||
for _ in $(seq 1 360); do
|
||||
if gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1; then
|
||||
gh release upload "$TAG" dist/*.apk dist/*.sha256 --repo "$REPO" --clobber
|
||||
echo "attached to $TAG"
|
||||
exit 0
|
||||
fi
|
||||
sleep 15
|
||||
done
|
||||
echo "::error::release $TAG did not appear within 90 minutes; the APK was built but not published"
|
||||
exit 1
|
||||
+44
-676
@@ -15,7 +15,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.ref == 'refs/heads/studio' }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
# Cheap path gate for pull requests. PRs confined to docs/release-notes
|
||||
@@ -179,27 +179,9 @@ jobs:
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
run: go test ./...
|
||||
|
||||
- name: test (Windows composition links)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(CaptureDirectoryMaterializesContainedLinks|CaptureDirectoryRefusesUnsafeLinks|CaptureHookFileFlattensContainedLink|MaterializeStillRejectsLinkedRetainedFilesAndRoot|WindowsCompositionJunctions|WindowsLiveSkillJunctions|CompositionRetainsLinkedSkillAcrossRestart|CompositionLinkedSkillRejectsLinkedResources|CompositionLinkedExtensionResources|CompositionLinkedHookResources)$' ./internal/state/agentpreset ./internal/ext/skill ./internal/assembly/boot | tee "$RUNNER_TEMP/composition-links.jsonl"
|
||||
jq -se '
|
||||
["TestCaptureDirectoryMaterializesContainedLinks", "TestCaptureDirectoryRefusesUnsafeLinks",
|
||||
"TestCaptureHookFileFlattensContainedLink", "TestMaterializeStillRejectsLinkedRetainedFilesAndRoot",
|
||||
"TestWindowsCompositionJunctions", "TestWindowsLiveSkillJunctions", "TestCompositionRetainsLinkedSkillAcrossRestart",
|
||||
"TestCompositionLinkedSkillRejectsLinkedResources", "TestCompositionLinkedExtensionResources",
|
||||
"TestCompositionLinkedHookResources"] as $required |
|
||||
[.[] | select(.Action == "pass") | .Test] as $passed |
|
||||
all(.[]; .Action != "skip") and (($required - $passed) | length == 0)
|
||||
' "$RUNNER_TEMP/composition-links.jsonl"
|
||||
|
||||
# Pull requests run a Windows smoke suite: the packages that actually
|
||||
# carry *_windows.go code, the startup/checkpoint packages with portable
|
||||
# filesystem contracts, plus cmd/. Isolation/worktree/mutation have a
|
||||
# complete JSON inventory below. The full ./... sweep stays on pushes
|
||||
# filesystem contracts, plus cmd/. The full ./... sweep stays on pushes
|
||||
# to main-v2; the Unix legs always run the full suite. Keep the job-level
|
||||
# budget above normal 7-9 minute runner variance; each package test binary
|
||||
# remains independently bounded by Go's per-package timeout below.
|
||||
@@ -223,542 +205,11 @@ jobs:
|
||||
WINDOWS_SANDBOX_WAIT_MS: "20000"
|
||||
run: go test -count=1 -p 4 -timeout=18m ./internal/runtime/agent/... ./internal/platform/appidentity/... ./internal/assembly/boot/... ./internal/state/checkpoint/... ./internal/frontend/cli/... ./internal/session/control/... ./internal/base/filelock/... ./internal/base/fileutil/... ./internal/ext/hook/... ./internal/state/instruction/... ./internal/ext/mcplaunch/... ./internal/platform/notify/... ./internal/base/proc/... ./internal/platform/remote/... ./internal/platform/repair/... ./internal/safety/sandbox/... ./internal/base/sysproxy/... ./internal/state/workspacelease/... ./cmd/...
|
||||
|
||||
- name: test (Windows published command job owner)
|
||||
# Worktree rollback is absent from the broad Windows PR smoke list.
|
||||
- name: test (Windows worktree rollback)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
evidence="$RUNNER_TEMP/published-job-owner"
|
||||
mkdir -p "$evidence"
|
||||
go test -count=1 -timeout=3m -json -run '^TestPublishedJob' ./internal/base/proc | tee "$evidence/positive.jsonl"
|
||||
node scripts/published-job-evidence.mjs positive "$evidence/positive.jsonl"
|
||||
node scripts/published-job-evidence.mjs overlay "$evidence"
|
||||
set +e
|
||||
go test -count=1 -timeout=3m -json -overlay="$evidence/overlay.json" -run '^TestPublishedJob' ./internal/base/proc > "$evidence/negative.jsonl"
|
||||
old_result=$?
|
||||
set -e
|
||||
test "$old_result" -eq 1
|
||||
node scripts/published-job-evidence.mjs negative "$evidence/negative.jsonl"
|
||||
|
||||
- name: upload Windows published command job owner evidence
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: published-job-owner-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/published-job-owner/
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows isolation owners)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 25
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -p 4 -timeout=18m -json ./internal/platform/worktree/... ./internal/state/mutation/... | tee "$RUNNER_TEMP/isolation-owners.jsonl" |
|
||||
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $passed |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
(["reasonix/internal/platform/worktree", "reasonix/internal/state/mutation"] - $passed | length) == 0 and
|
||||
(["TestHeldPathsCoverOnlyTheSameFileAcrossCaseSpellings",
|
||||
"TestLegacyWaitDoesNotConsumeExecutionCapacity"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/isolation-owners.jsonl"
|
||||
jq -se '
|
||||
. as $events |
|
||||
{"reasonix/internal/platform/worktree": [
|
||||
"TestWriteChangesRejectsRedirectedDeletion",
|
||||
"TestWriteChangesUnlinksLeafDirectoryLink"
|
||||
]} |
|
||||
to_entries | all(.[]; . as $package |
|
||||
([ $events[] | select(.Package == $package.key and .Test == null and
|
||||
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
|
||||
length == 1 and .[0].Action == "pass") and
|
||||
all(.value[]; . as $test |
|
||||
[ $events[] | select(.Package == $package.key and .Test == $test and
|
||||
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
|
||||
length == 1 and .[0].Action == "pass"))
|
||||
' "$RUNNER_TEMP/isolation-owners.jsonl"
|
||||
|
||||
- name: upload Windows isolation test inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: isolation-owners-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/isolation-owners.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows delegated Goal usage scope)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(UsageScope.*)$' ./internal/contract/event ./internal/session/control ./internal/assembly/boot | tee "$RUNNER_TEMP/goal-usage-scope.jsonl"
|
||||
jq -se '
|
||||
["TestUsageScopeKeepsEarlierEventOwner",
|
||||
"TestUsageScopeWithoutCapability" ] as $required |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
all(.[]; .Action != "fail" and .Action != "skip") and
|
||||
(($required - $passed) | length == 0) and
|
||||
((["reasonix/internal/contract/event", "reasonix/internal/session/control", "reasonix/internal/assembly/boot"] - $packages) | length == 0)
|
||||
' "$RUNNER_TEMP/goal-usage-scope.jsonl"
|
||||
|
||||
- name: upload Windows delegated Goal usage scope inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: goal-usage-scope-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/goal-usage-scope.jsonl
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows Controller routing root budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(SemanticRootUsage)$' ./internal/runtime/capability | tee "$RUNNER_TEMP/controller-routing-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/capability"] - $packages | length) == 0 and
|
||||
([
|
||||
"TestSemanticRootUsage",
|
||||
"TestSemanticRootUsage/billed_error",
|
||||
"TestSemanticRootUsage/cache",
|
||||
"TestSemanticRootUsage/estimated",
|
||||
"TestSemanticRootUsage/invalid_answer",
|
||||
"TestSemanticRootUsage/missing",
|
||||
"TestSemanticRootUsage/nil_sink",
|
||||
"TestSemanticRootUsage/no_observer",
|
||||
"TestSemanticRootUsage/reported"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/controller-routing-budget.jsonl"
|
||||
|
||||
- name: upload Windows Controller routing root budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: controller-routing-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/controller-routing-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows Coordinator root budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(PreparedRootBudget|CoordinatorRootBudget|EffectCoordinatorRoot)' ./internal/runtime/agent ./internal/runtime/coordinator | tee "$RUNNER_TEMP/coordinator-root-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/agent", "reasonix/internal/runtime/coordinator"] - $packages | length) == 0 and
|
||||
(["TestCoordinatorRootBudgetRoutes",
|
||||
"TestCoordinatorRootBudgetRoutes/raw", "TestCoordinatorRootBudgetRoutes/tool_planner",
|
||||
"TestCoordinatorRootBudgetRoutes/failure_fallback", "TestCoordinatorRootBudgetRoutes/plan_only_continuation",
|
||||
"TestCoordinatorRootBudgetRoutes/approval_declined_continuation", "TestCoordinatorRootBudgetRoutes/approval_allowed"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/coordinator-root-budget.jsonl"
|
||||
|
||||
- name: upload Windows Coordinator root budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: coordinator-root-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/coordinator-root-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows resumed Goal planner budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(GoalResumeRenewsPlannerBudget|GoalNonBudgetResumeKeepsPlannerBudget|GoalTokenBudgetPausesAndResumes|EffectGoalResumeRenewsPlannerBudget)$' ./internal/session/control ./internal/assembly/boot | tee "$RUNNER_TEMP/goal-planner-resume.jsonl"
|
||||
jq -se '
|
||||
["TestGoalResumeRenewsPlannerBudget",
|
||||
"TestGoalNonBudgetResumeKeepsPlannerBudget",
|
||||
"TestGoalNonBudgetResumeKeepsPlannerBudget/manual_pause",
|
||||
"TestGoalNonBudgetResumeKeepsPlannerBudget/not_resumable",
|
||||
"TestGoalTokenBudgetPausesAndResumes",
|
||||
"TestEffectGoalResumeRenewsPlannerBudget"] as $required |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
all(.[]; .Action != "fail" and .Action != "skip") and
|
||||
(($required - $passed) | length == 0) and
|
||||
((["reasonix/internal/session/control", "reasonix/internal/assembly/boot"] - $packages) | length == 0)
|
||||
' "$RUNNER_TEMP/goal-planner-resume.jsonl"
|
||||
|
||||
- name: upload Windows resumed Goal planner budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: goal-planner-resume-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/goal-planner-resume.jsonl
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows root wall budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootWall|EffectRootWall)' ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/root-wall-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestRootWallBudgetReachesDelegatedAgents", "TestRootWallBudgetReachesDelegatedAgents/foreground",
|
||||
"TestRootWallBudgetReachesDelegatedAgents/nested", "TestRootWallBudgetReachesDelegatedAgents/siblings",
|
||||
"TestRootWallBudgetReachesDelegatedAgents/context", "TestRootWallBudgetReachesDelegatedAgents/unconfigured",
|
||||
"TestRootWallBudgetReachesDelegatedAgents/explicit_zero", "TestRootWallBudgetContinuationAndReset",
|
||||
"TestRootWallBudgetContinuationAndReset/continuation", "TestRootWallBudgetContinuationAndReset/new_task",
|
||||
"TestRootWallBudgetContinuationAndReset/reset", "TestRootWallBudgetContinuationAndReset/override"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-wall-budget.jsonl"
|
||||
|
||||
- name: upload Windows root wall budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: root-wall-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/root-wall-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows root token budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootToken|EffectRootToken|BudgetCountsKnownReportedTokens|ReportedToken)' ./internal/runtime/agent | tee "$RUNNER_TEMP/root-token-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/agent"] - $packages | length) == 0 and
|
||||
(["TestRootTokenBudgetIncludesChildProviderUsage", "TestRootTokenBudgetIncludesChildProviderUsage/configured",
|
||||
"TestRootTokenBudgetIncludesChildProviderUsage/context", "TestRootTokenBudgetIncludesChildProviderUsage/unconfigured",
|
||||
"TestRootTokenBudgetIncludesChildProviderUsage/explicit_zero", "TestRootTokenBudgetSurvivesNestedAndConcurrentChildren",
|
||||
"TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/nested", "TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/siblings",
|
||||
"TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/child_override", "TestRootTokenBudgetContinuationAndReset",
|
||||
"TestRootTokenBudgetContinuationAndReset/continuation", "TestRootTokenBudgetContinuationAndReset/new_task",
|
||||
"TestRootTokenBudgetContinuationAndReset/reset" ] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-token-budget.jsonl"
|
||||
jq -se '
|
||||
. as $events |
|
||||
{"reasonix/internal/runtime/agent": [
|
||||
"TestBudgetCountsKnownReportedTokens",
|
||||
"TestBudgetCountsKnownReportedTokens/split",
|
||||
"TestBudgetCountsKnownReportedTokens/total_only",
|
||||
"TestBudgetCountsKnownReportedTokens/partial_split",
|
||||
"TestBudgetCountsKnownReportedTokens/lower_total",
|
||||
"TestBudgetCountsKnownReportedTokens/missing_total",
|
||||
"TestBudgetCountsKnownReportedTokens/negative_split",
|
||||
"TestBudgetCountsKnownReportedTokens/zero",
|
||||
"TestBudgetCountsKnownReportedTokens/negative",
|
||||
"TestReportedTokenCountersSaturate"]} |
|
||||
to_entries | all(.[]; . as $package |
|
||||
([ $events[] | select(.Package == $package.key and .Test == null and
|
||||
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
|
||||
length == 1 and .[0].Action == "pass") and
|
||||
all(.value[]; . as $test |
|
||||
[ $events[] | select(.Package == $package.key and .Test == $test and
|
||||
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
|
||||
length == 1 and .[0].Action == "pass"))
|
||||
' "$RUNNER_TEMP/root-token-budget.jsonl"
|
||||
|
||||
- name: upload Windows root token budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: root-token-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/root-token-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows budget usage completeness)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RequestOnlyUsage|RunBudgetMissingUsage|TaskBudgetKnownCost|RunBudgetExplicitZero|TaskBudgetUsageCompleteness|TaskBudgetCostUses|EffectBudgetCompleteness|TaskBudgetGateIgnoresCostWhenUnpriced)' ./internal/contract/provider ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/budget-usage.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/contract/provider", "reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestRequestOnlyUsageDoesNotCertifyTokenAccounting", "TestRunBudgetMissingUsageKeepsCostIncomplete",
|
||||
"TestRunBudgetMissingUsageKeepsCostIncomplete/complete", "TestRunBudgetMissingUsageKeepsCostIncomplete/missing_first",
|
||||
"TestRunBudgetMissingUsageKeepsCostIncomplete/missing_last", "TestRunBudgetMissingUsageKeepsCostIncomplete/all_missing",
|
||||
"TestTaskBudgetKnownCostStillLandsAfterMissingUsage", "TestRunBudgetExplicitZeroUsageIsPriced",
|
||||
"TestTaskBudgetUsageCompletenessFollowsItsLifetime", "TestTaskBudgetUsageCompletenessFollowsItsLifetime/continuation",
|
||||
"TestTaskBudgetUsageCompletenessFollowsItsLifetime/new_task", "TestTaskBudgetUsageCompletenessFollowsItsLifetime/explicit_reset",
|
||||
"TestTaskBudgetCostUsesOnlyTheReportedSubtotal", "TestTaskBudgetGateIgnoresCostWhenUnpriced",
|
||||
"TestEffectBudgetCompletenessFromOpenAIStream", "TestEffectBudgetCompletenessFromOpenAIStream/missing_0",
|
||||
"TestEffectBudgetCompletenessFromOpenAIStream/missing_1", "TestEffectBudgetCompletenessFromOpenAIStream/missing_2"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/budget-usage.jsonl"
|
||||
|
||||
- name: upload Windows budget usage inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: budget-usage-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/budget-usage.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
|
||||
- name: test (Windows Git storage environment)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(HostStorageEnvironment|HostObjectStorage|HostExplicitStorageControls|DetachedStorageEnvironment)$' ./internal/platform/gitcmd ./internal/platform/worktree ./internal/assembly/boot | tee "$RUNNER_TEMP/git-storage-environment.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/platform/gitcmd", "reasonix/internal/platform/worktree", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
([
|
||||
"TestDetachedStorageEnvironment",
|
||||
"TestDetachedStorageEnvironment/explicit_env",
|
||||
"TestDetachedStorageEnvironment/inherited_env",
|
||||
"TestHostExplicitStorageControls",
|
||||
"TestHostExplicitStorageControls/explicit_object_directory",
|
||||
"TestHostExplicitStorageControls/private_index",
|
||||
"TestHostExplicitStorageControls/repository_declared_alternate",
|
||||
"TestHostObjectStorage",
|
||||
"TestHostObjectStorage/ambient_alternate_not_read",
|
||||
"TestHostObjectStorage/writes_stay_in_repository",
|
||||
"TestHostStorageEnvironment",
|
||||
"TestHostStorageEnvironment/GIT_ALTERNATE_OBJECT_DIRECTORIES",
|
||||
"TestHostStorageEnvironment/GIT_INDEX_FILE",
|
||||
"TestHostStorageEnvironment/GIT_OBJECT_DIRECTORY"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/git-storage-environment.jsonl"
|
||||
|
||||
- name: upload Windows Git storage environment inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: git-storage-environment-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/git-storage-environment.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows Anthropic interrupted usage)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(InterruptedUsage|EffectAnthropicPartialUsageRecovery)' ./internal/model/anthropic ./internal/assembly/boot | tee "$RUNNER_TEMP/anthropic-interrupted-usage.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/model/anthropic", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestInterruptedUsageTerminal", "TestInterruptedUsageTerminal/upstream_error",
|
||||
"TestInterruptedUsageTerminal/payload_error", "TestInterruptedUsageTerminal/premature_eof",
|
||||
"TestInterruptedUsageTerminal/decode_error", "TestInterruptedUsageTerminal/reported_zero",
|
||||
"TestInterruptedUsageTerminal/missing_control", "TestInterruptedUsageTerminal/complete_control",
|
||||
"TestInterruptedUsageBoundedConsumer", "TestInterruptedUsageReadErrorAndCancellation",
|
||||
"TestInterruptedUsageReadErrorAndCancellation/read_error", "TestInterruptedUsageReadErrorAndCancellation/cancelled_buffered_reader",
|
||||
"TestInterruptedUsageAbandonedConsumer", "TestInterruptedUsageIdleTimeout",
|
||||
"TestEffectAnthropicPartialUsageRecovery", "TestEffectAnthropicPartialUsageRecovery/upstream_error",
|
||||
"TestEffectAnthropicPartialUsageRecovery/premature_eof", "TestEffectAnthropicPartialUsageRecovery/complete_control"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/anthropic-interrupted-usage.jsonl"
|
||||
|
||||
- name: upload Windows Anthropic interrupted usage inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: anthropic-interrupted-usage-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/anthropic-interrupted-usage.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows root websearch budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootWebSearch|EffectRootWebSearch)' ./internal/tools/websearch | tee "$RUNNER_TEMP/root-websearch-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/tools/websearch"] - $packages | length) == 0 and
|
||||
(["TestRootWebSearchUsageTerminal", "TestRootWebSearchUsageTerminal/reported",
|
||||
"TestRootWebSearchUsageTerminal/no_reporter", "TestRootWebSearchUsageTerminal/missing",
|
||||
"TestRootWebSearchUsageTerminal/no_observer", "TestRootWebSearchUsageTerminal/input_refused",
|
||||
"TestRootWebSearchUsageTerminal/no_factory", "TestRootWebSearchUsageTerminal/factory_error",
|
||||
"TestRootWebSearchUsageTerminal/billed_failure", "TestRootWebSearchUsageTerminal/cancel_after_usage",
|
||||
"TestRootWebSearchUsageTerminal/cancel_before_start", "TestRootWebSearchUsageTerminal/stream_error"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-websearch-budget.jsonl"
|
||||
|
||||
- name: upload Windows root websearch budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: root-websearch-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/root-websearch-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows explicit skill root budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^(TestProfileRootBudgetLifetime|TestProfileBudgetPreparationUsesTurnAdmission|TestEffectExplicitSkillRootBudget)$' ./internal/session/control | tee "$RUNNER_TEMP/explicit-skill-root-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/session/control"] - $packages | length) == 0 and
|
||||
(["TestProfileBudgetPreparationUsesTurnAdmission" ] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/explicit-skill-root-budget.jsonl"
|
||||
|
||||
- name: upload Windows explicit skill root budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: explicit-skill-root-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/explicit-skill-root-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows root triage budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootTriage|EffectRootTriage)' ./internal/runtime/agent | tee "$RUNNER_TEMP/root-triage-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/agent"] - $packages | length) == 0 and
|
||||
(["TestRootTriageTokenBudget", "TestRootTriageTokenBudget/classifier", "TestRootTriageTokenBudget/screen",
|
||||
"TestRootTriageTokenBudget/billed_failure", "TestRootTriageTokenBudget/missing", "TestRootTriageTokenBudget/disabled",
|
||||
"TestRootTriageTokenBudget/no_sink", "TestRootTriageTokenBudget/counts_without_total",
|
||||
"TestRootTriageOutsideRunDoesNotCreateScope" ] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-triage-budget.jsonl"
|
||||
|
||||
- name: upload Windows root triage budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: root-triage-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/root-triage-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows local cost quote)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(LocalBudgetUsesOccurrenceQuote|LocalCost.*|UsageQuote.*)$' ./internal/contract/event ./internal/runtime/agent | tee "$RUNNER_TEMP/local-cost-quote.jsonl"
|
||||
jq -se '
|
||||
["TestLocalBudgetUsesOccurrenceQuote", "TestLocalBudgetUsesOccurrenceQuote/peak", "TestLocalBudgetUsesOccurrenceQuote/offpeak",
|
||||
"TestLocalCostKeepsOriginalCurrencyBuckets", "TestLocalCostFixedPointThresholdAndUnknown",
|
||||
"TestUsageQuoteUsesOneOwnerAndOccurrence", "TestUsageQuoteLegacyAndNilSinks"
|
||||
] as $required |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
all(.[]; .Action != "fail" and .Action != "skip") and
|
||||
(($required - $passed) | length == 0) and
|
||||
((["reasonix/internal/contract/event", "reasonix/internal/runtime/agent"] - $packages) | length == 0)
|
||||
' "$RUNNER_TEMP/local-cost-quote.jsonl"
|
||||
|
||||
- name: upload Windows local cost quote inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: local-cost-quote-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/local-cost-quote.jsonl
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows estimated usage quotes)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(EstimatedUsage|EffectEstimatedUsage|ToWireUsagePayloadJSON$|StatusExtensionTracksMultipleSessionsAndUsage$|CacheSplitProvenanceIsRecorded$|RecordedPlainCallIsUnchanged$)' ./internal/contract/pricing ./internal/contract/event ./internal/contract/config ./internal/assembly/boot ./internal/contract/eventwire ./internal/frontend/acp ./internal/state/stats | tee "$RUNNER_TEMP/estimated-quote.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/contract/pricing", "reasonix/internal/contract/event", "reasonix/internal/contract/config", "reasonix/internal/assembly/boot",
|
||||
"reasonix/internal/contract/eventwire", "reasonix/internal/frontend/acp", "reasonix/internal/state/stats"] - $packages | length) == 0 and
|
||||
(["TestEstimatedUsageQuotePreservesAmountsWithoutCompleteFacts", "TestEstimatedUsageQuotePreservesAmountsWithoutCompleteFacts/CNY",
|
||||
"TestEstimatedUsageQuotePreservesAmountsWithoutCompleteFacts/USD", "TestEstimatedUsageLedgerRetainsPartialCoverage",
|
||||
"TestEstimatedUsageLedgerRetainsPartialCoverage/separate_keys", "TestEstimatedUsageLedgerRetainsPartialCoverage/same_key",
|
||||
"TestEstimatedUsageSavedQuoteReader", "TestEstimatedUsageQuoteSinkCoversTurnAndGuardian",
|
||||
"TestEstimatedUsageQuoteSinkCoversTurnAndGuardian/turn", "TestEstimatedUsageQuoteSinkCoversTurnAndGuardian/guardian",
|
||||
"TestEstimatedUsageConfiguredQuote", "TestEffectEstimatedUsageQuoteAfterStreamRecovery",
|
||||
"TestEffectEstimatedUsageQuoteAfterStreamRecovery/reported_control", "TestEffectEstimatedUsageQuoteAfterStreamRecovery/interrupted",
|
||||
"TestToWireUsagePayloadJSON", "TestStatusExtensionTracksMultipleSessionsAndUsage", "TestCacheSplitProvenanceIsRecorded", "TestRecordedPlainCallIsUnchanged"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/estimated-quote.jsonl"
|
||||
|
||||
- name: upload Windows estimated quote inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: estimated-quote-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/estimated-quote.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows bounded root budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(AuxiliaryUsageObserver|BoundedUsageObserver|RootBounded|EffectRootBounded)' ./internal/contract/provider ./internal/model/boundedllm ./internal/runtime/agent | tee "$RUNNER_TEMP/bounded-root-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/contract/provider", "reasonix/internal/model/boundedllm",
|
||||
"reasonix/internal/runtime/agent"] - $packages | length) == 0 and
|
||||
(["TestAuxiliaryUsageObserverReplacesAndCopies",
|
||||
"TestBoundedUsageObserverOwnsOneTerminal", "TestBoundedUsageObserverOwnsOneTerminal/reported",
|
||||
"TestBoundedUsageObserverOwnsOneTerminal/billed_failure", "TestBoundedUsageObserverOwnsOneTerminal/without_sink",
|
||||
"TestBoundedUsageObserverOwnsOneTerminal/missing", "TestBoundedUsageObserverOwnsOneTerminal/input_refused"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/bounded-root-budget.jsonl"
|
||||
|
||||
- name: upload Windows bounded root budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: bounded-root-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/bounded-root-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
timeout-minutes: 3
|
||||
run: go test -count=1 -timeout=2m -run '^TestDiscardUnpublished.*OnWindows$|^TestCreateFromDetachedHEADAndOccupiedBranch$' ./internal/platform/worktree
|
||||
|
||||
# The general Windows PR smoke list intentionally omits internal/contract/tool.
|
||||
# Keep the session-temp portability contract covered without widening the
|
||||
@@ -777,6 +228,40 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
run: go test -count=1 -timeout=3m -run '^TestBashPowerShellExecuteDetailedContract$|^TestBashPowerShell51PreflightRejectsAndAndDetailed$|^TestBashPowerShellOutputIsUTF8$|^TestBashPowerShellSurfacesNonZeroExit$|^TestBashPowerShellRejectsChaining$' ./internal/tools/builtin
|
||||
|
||||
# Full push suite uses the same 18m package timeout as Windows smoke and a
|
||||
# 35m step budget above it. -count=1 as at the smoke step. A failure prints
|
||||
# the failed tests' output and the 15 slowest boot tests.
|
||||
- name: test (full)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name != 'pull_request'
|
||||
timeout-minutes: 35
|
||||
env:
|
||||
# Run the prompt-cache prefix-stability guard (TestCacheHit*) in CI: a
|
||||
# regression there silently tanks the cache hit rate the project is
|
||||
# built around.
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
# Bound sandbox helper children in Windows tests so a failed OS-level
|
||||
# launch cannot pin the Actions step after Go's package timeout fires.
|
||||
WINDOWS_SANDBOX_WAIT_MS: "20000"
|
||||
shell: bash
|
||||
run: |
|
||||
go test -count=1 -p 4 -timeout=18m -json ./... | tee "$RUNNER_TEMP/full.jsonl" |
|
||||
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
|
||||
|
||||
- name: test (full) failure detail
|
||||
if: failure() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name != 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
f="$RUNNER_TEMP/full.jsonl"
|
||||
[ -s "$f" ] || exit 0
|
||||
echo "::group::output of failed tests"
|
||||
jq -rn '[inputs | select(.Test != null)] | group_by(.Package + "/" + .Test)
|
||||
| map(select(any(.[]; .Action == "fail")))[]
|
||||
| map(select(.Action == "output") | .Output) | join("")' "$f" || true
|
||||
echo "::endgroup::"
|
||||
echo "slowest internal/assembly/boot tests (seconds, name):"
|
||||
jq -rn '[inputs | select((.Package | endswith("/internal/assembly/boot")) and .Test != null and .Action == "pass")]
|
||||
| sort_by(-.Elapsed)[:15][] | "\(.Elapsed)\t\(.Test)"' "$f" || true
|
||||
|
||||
race:
|
||||
needs: changes
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.code != 'false')
|
||||
@@ -811,130 +296,13 @@ jobs:
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
run: go test -race ./internal/runtime/agent/... ./internal/runtime/delegation/... ./internal/platform/worktree/... ./internal/state/mutation/... ./internal/ext/plugin/... ./internal/tools/jobs/... ./internal/base/proc/... ./internal/safety/sandbox/... ./internal/base/filelock/... ./internal/contract/eventwire/... ./internal/platform/remote/... ./internal/ext/extension/... ./internal/assembly/boot/... ./internal/session/control/... ./internal/contract/tool/...
|
||||
run: go test -race ./internal/runtime/agent/... ./internal/ext/plugin/... ./internal/tools/jobs/... ./internal/base/proc/... ./internal/safety/sandbox/... ./internal/base/filelock/... ./internal/contract/eventwire/... ./internal/platform/remote/... ./internal/ext/extension/... ./internal/assembly/boot/... ./internal/session/control/... ./internal/contract/tool/...
|
||||
|
||||
# Push-only full suites. internal/frontend/serve and internal/assembly/boot run
|
||||
# hundreds of serial tests and set the wall time of one whole-tree `go test`,
|
||||
# so each is cut by test name into shards that run on their own runners. The
|
||||
# -run / -skip patterns partition the names exactly: every test lands in one
|
||||
# shard. `rest` is every other package. Pull requests keep the focused steps
|
||||
# in the `test` and `race` jobs above.
|
||||
test-windows-shards:
|
||||
if: github.event_name != 'pull_request'
|
||||
name: test (windows-latest, ${{ matrix.shard }})
|
||||
runs-on: windows-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { shard: rest }
|
||||
- { shard: serve-1, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[A-E]' }
|
||||
- { shard: serve-2, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[F-R]' }
|
||||
- { shard: serve-3, pkg: ./internal/frontend/serve, flag: -skip, filter: '^Test[A-R]' }
|
||||
- { shard: boot-1, pkg: ./internal/assembly/boot, flag: -skip, filter: '^TestEffect([^0-9A-C]|$)' }
|
||||
- { shard: boot-2, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect[D-O]' }
|
||||
- { shard: boot-3, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect([^0-9A-O]|$)' }
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: test (shard)
|
||||
timeout-minutes: 40
|
||||
env:
|
||||
# Run the prompt-cache prefix-stability guard (TestCacheHit*) in CI: a
|
||||
# regression there silently tanks the cache hit rate the project is
|
||||
# built around.
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
# Bound sandbox helper children in Windows tests so a failed OS-level
|
||||
# launch cannot pin the Actions step after Go's package timeout fires.
|
||||
WINDOWS_SANDBOX_WAIT_MS: "20000"
|
||||
SHARD: ${{ matrix.shard }}
|
||||
PKG: ${{ matrix.pkg }}
|
||||
FLAG: ${{ matrix.flag }}
|
||||
FILTER: ${{ matrix.filter }}
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "$SHARD" = rest ]; then
|
||||
mapfile -t pkgs < <(go list ./... | tr -d '\r' | grep -v -E '/internal/(frontend/serve|assembly/boot)$')
|
||||
else
|
||||
pkgs=("$PKG")
|
||||
fi
|
||||
filter=()
|
||||
if [ -n "$FILTER" ]; then filter=("$FLAG" "$FILTER"); fi
|
||||
go test -count=1 -p 4 -timeout=36m -json "${filter[@]}" "${pkgs[@]}" | tee "$RUNNER_TEMP/shard.jsonl" |
|
||||
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
|
||||
|
||||
- name: test (shard) failure detail
|
||||
if: failure()
|
||||
shell: bash
|
||||
run: |
|
||||
f="$RUNNER_TEMP/shard.jsonl"
|
||||
[ -s "$f" ] || exit 0
|
||||
echo "::group::output of failed tests"
|
||||
jq -rn '[inputs | select(.Test != null)] | group_by(.Package + "/" + .Test)
|
||||
| map(select(any(.[]; .Action == "fail")))[]
|
||||
| map(select(.Action == "output") | .Output) | join("")' "$f" || true
|
||||
echo "::endgroup::"
|
||||
echo "slowest tests (seconds, package, name):"
|
||||
jq -rn '[inputs | select(.Test != null and .Action == "pass")]
|
||||
| sort_by(-.Elapsed)[:15][] | "\(.Elapsed) \(.Package) \(.Test)"' "$f" || true
|
||||
|
||||
race-shards:
|
||||
if: github.event_name != 'pull_request'
|
||||
name: race (${{ matrix.shard }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { shard: rest }
|
||||
- { shard: serve-1, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[A-E]' }
|
||||
- { shard: serve-2, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[F-R]' }
|
||||
- { shard: serve-3, pkg: ./internal/frontend/serve, flag: -skip, filter: '^Test[A-R]' }
|
||||
- { shard: boot-1, pkg: ./internal/assembly/boot, flag: -skip, filter: '^TestEffect([^0-9A-C]|$)' }
|
||||
- { shard: boot-2, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect[D-O]' }
|
||||
- { shard: boot-3, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect([^0-9A-O]|$)' }
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Install and verify Linux sandbox backend
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y bubblewrap
|
||||
if sysctl -n kernel.unprivileged_userns_clone >/dev/null 2>&1; then
|
||||
sudo sysctl -w kernel.unprivileged_userns_clone=1
|
||||
fi
|
||||
if sysctl -n kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then
|
||||
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
||||
fi
|
||||
bwrap --ro-bind / / --dev /dev --proc /proc -- true || echo "::warning::bubblewrap is installed but this runner denies user namespaces; unavailable-backend tests will run fail-closed"
|
||||
|
||||
- name: test -race (shard)
|
||||
timeout-minutes: 30
|
||||
- name: test -race (full)
|
||||
if: github.event_name != 'pull_request'
|
||||
env:
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
SHARD: ${{ matrix.shard }}
|
||||
PKG: ${{ matrix.pkg }}
|
||||
FLAG: ${{ matrix.flag }}
|
||||
FILTER: ${{ matrix.filter }}
|
||||
run: |
|
||||
if [ "$SHARD" = rest ]; then
|
||||
mapfile -t pkgs < <(go list ./... | grep -v -E '/internal/(frontend/serve|assembly/boot)$')
|
||||
else
|
||||
pkgs=("$PKG")
|
||||
fi
|
||||
filter=()
|
||||
if [ -n "$FILTER" ]; then filter=("$FLAG" "$FILTER"); fi
|
||||
go test -race -timeout=25m "${filter[@]}" "${pkgs[@]}"
|
||||
run: go test -race ./...
|
||||
|
||||
# sdk/go is a nested stdlib-only module invisible to root `go test ./...`.
|
||||
# Its DTOs are generated from internal/ext/extension/protocol, and the
|
||||
|
||||
@@ -43,20 +43,20 @@ jobs:
|
||||
if: matrix.branch != ''
|
||||
run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
||||
uses: github/codeql-action/init@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.language == 'go' && 'autobuild' || 'none' }}
|
||||
- name: Perform CodeQL Analysis
|
||||
if: matrix.branch == ''
|
||||
uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
||||
uses: github/codeql-action/analyze@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
# Results from a checked-out line are filed under that line's ref, not the
|
||||
# default branch the schedule happened to run from.
|
||||
- name: Perform CodeQL Analysis (${{ matrix.branch }})
|
||||
if: matrix.branch != ''
|
||||
uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
||||
uses: github/codeql-action/analyze@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
ref: refs/heads/${{ matrix.branch }}
|
||||
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
name: PR author gate
|
||||
|
||||
# Pull requests are limited to maintainers and the logins in .github/pr-authors.txt.
|
||||
# This runs in the base repository's context, checks out the base branch only, and
|
||||
# never executes pull-request code; the allowlist it reads is the base branch's.
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: pr-author-gate-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
gate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- run: node scripts/pr-author-gate.mjs
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
@@ -16,7 +16,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: studio-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.ref == 'refs/heads/studio' }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
browser-guards:
|
||||
name: browser guards
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
timeout-minutes: 15
|
||||
defaults:
|
||||
run:
|
||||
working-directory: desktop/frontend-next
|
||||
@@ -155,22 +155,14 @@ jobs:
|
||||
node perf/browser-full.mjs
|
||||
node perf/mod-ui.mjs
|
||||
node perf/codeeditor-size.mjs
|
||||
node perf/composer-reflow.mjs
|
||||
node perf/composer-toolbar-row.mjs
|
||||
node perf/action-icons.mjs
|
||||
node perf/rowhit.mjs
|
||||
node perf/prefs-wide-centered.mjs
|
||||
node perf/tab-pip.mjs
|
||||
node perf/syntax-tokens.mjs
|
||||
node perf/tab-close-inside.mjs
|
||||
node perf/context-card-hover.mjs
|
||||
node perf/sessquick.mjs
|
||||
node perf/sessionrows.mjs
|
||||
node perf/session-row-marker.mjs
|
||||
node perf/session-selected-row.mjs
|
||||
node perf/wheel.mjs
|
||||
node perf/branch-tooltip.mjs
|
||||
node perf/branch-alignment.mjs
|
||||
node perf/branch-menu.mjs
|
||||
node perf/image-preview.mjs
|
||||
node perf/user-images.mjs
|
||||
@@ -178,7 +170,6 @@ jobs:
|
||||
node perf/addfolder-rest.mjs
|
||||
node perf/scroll-jump.mjs
|
||||
node perf/add-project-selects.mjs
|
||||
node perf/project-header.mjs
|
||||
node perf/setup-defer.mjs
|
||||
node perf/narrow.mjs
|
||||
node perf/settings-rows.mjs
|
||||
@@ -188,13 +179,6 @@ jobs:
|
||||
node perf/web-access.mjs
|
||||
node perf/peers-settings.mjs
|
||||
node perf/output-cap.mjs
|
||||
node perf/tool-output-clip.mjs
|
||||
node perf/agent-run-tab.mjs
|
||||
node perf/changes-cap.mjs
|
||||
node perf/session-menu-fit.mjs
|
||||
node perf/rail-rows-typography.mjs
|
||||
node perf/settings-nav-fit.mjs
|
||||
node perf/rail-rows-align.mjs
|
||||
node perf/progress-tail.mjs
|
||||
node perf/trust-wait.mjs
|
||||
node perf/nest-nodes.mjs
|
||||
|
||||
@@ -117,22 +117,6 @@ go test ./internal/runtime/agent/ -v # verbose, one package
|
||||
go test ./internal/tools/builtin/ -run TestGrep # one test
|
||||
```
|
||||
|
||||
`testenv.IsolateUserState` installs a disposable user home and config-edit lock
|
||||
scope. Default-path tests can clear `REASONIX_HOME` without losing the test
|
||||
registry, while explicitly isolated Reasonix homes retain their own locks.
|
||||
|
||||
`REASONIX_TEST_HOME` carries the scope to helper subprocesses. A helper that
|
||||
skips the isolation guard must install the inherited home with
|
||||
`testscope.InstallHome` before running tests.
|
||||
|
||||
The environment variable alone cannot activate a test scope. Production
|
||||
executables still use the OS account's home for shared config locks.
|
||||
|
||||
The guard removes its scratch home on normal completion. Scratch homes are
|
||||
created under the original home, so an interrupted run can still leave
|
||||
`.reasonix-test-home-*` directories; this is separate from persistent
|
||||
config-edit locks in the real `.reasonix/locks` directory.
|
||||
|
||||
### Code style
|
||||
|
||||
- `gofmt` is enforced by CI — format before committing
|
||||
|
||||
@@ -169,19 +169,6 @@ Every package ships with a `.minisig` signature, and the release carries
|
||||
The same release also carries the 2.x `reasonix` CLI as archives for
|
||||
`darwin|linux|windows × amd64|arm64`.
|
||||
|
||||
The Android build ships on that release as `ReasonixMobile-<version>.apk`
|
||||
(`arm64`, Android 8.0 or newer). The kernel and the page run on the phone, and
|
||||
the agent reaches your servers over SSH; pairing with a computer is done by
|
||||
scanning the code it shows.
|
||||
|
||||
The APK is signed with the Android release key rather than minisign, so it
|
||||
carries a `.sha256` and no `.minisig`.
|
||||
|
||||
When that key is absent the build is named
|
||||
`ReasonixMobile-<version>-unsigned.apk`. Android refuses to install an unsigned
|
||||
package, so those builds stay a workflow artifact for whoever wants to sign one,
|
||||
and the release carries no APK at all.
|
||||
|
||||
### Reasonix 1.x (stable)
|
||||
|
||||
The 1.x CLI installs through npm on any supported platform, or Homebrew on
|
||||
@@ -301,11 +288,11 @@ A small list of folks whose work has shaped Reasonix the most — the current to
|
||||
<!-- reasonix-top-contributors:start -->
|
||||
| Contributor | Contributor | Contributor | Contributor |
|
||||
| --- | --- | --- | --- |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**KHG420**](https://github.com/KHG420) | [**ttmouse**](https://github.com/ttmouse) | [**chuiwenwei**](https://github.com/chuiwenwei) |
|
||||
| [**lifu963**](https://github.com/lifu963) | [**XTLine**](https://github.com/XTLine) | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**xxoingr**](https://github.com/xxoingr) |
|
||||
| [**Harbor404**](https://github.com/Harbor404) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) | [**Li-Charles-One**](https://github.com/Li-Charles-One) |
|
||||
| [**light-front-theory**](https://github.com/light-front-theory) | [**imcreazy**](https://github.com/imcreazy) | [**eghrhegpe**](https://github.com/eghrhegpe) | [**noreply**](https://github.com/noreply) |
|
||||
| [**par73e**](https://github.com/par73e) | [**Bernardxu123**](https://github.com/Bernardxu123) | [**lancy69**](https://github.com/lancy69) | [**Guan0923**](https://github.com/Guan0923) |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**KHG420**](https://github.com/KHG420) | [**ttmouse**](https://github.com/ttmouse) | [**lifu963**](https://github.com/lifu963) |
|
||||
| [**XTLine**](https://github.com/XTLine) | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**chuiwenwei**](https://github.com/chuiwenwei) | [**Harbor404**](https://github.com/Harbor404) |
|
||||
| [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) | [**Li-Charles-One**](https://github.com/Li-Charles-One) | [**light-front-theory**](https://github.com/light-front-theory) |
|
||||
| [**xxoingr**](https://github.com/xxoingr) | [**eghrhegpe**](https://github.com/eghrhegpe) | [**noreply**](https://github.com/noreply) | [**par73e**](https://github.com/par73e) |
|
||||
| [**Bernardxu123**](https://github.com/Bernardxu123) | [**Guan0923**](https://github.com/Guan0923) | [**HaoyueQin**](https://github.com/HaoyueQin) | [**BuGlessRB**](https://github.com/BuGlessRB) |
|
||||
<!-- reasonix-top-contributors:end -->
|
||||
|
||||
Special thanks to [**Bernardxu123**](https://github.com/Bernardxu123) for designing the project logo and intro video.
|
||||
|
||||
+5
-14
@@ -156,15 +156,6 @@ Reasonix 分为两条版本线，调整的原因见[版本路线公告](https://
|
||||
|
||||
同一个发布页也提供 2.x 的 `reasonix` CLI 归档（`darwin|linux|windows × amd64|arm64`）。
|
||||
|
||||
安卓版在同一个发布页上，文件名是 `ReasonixMobile-<版本>.apk`（`arm64`，Android 8.0 及以上）。
|
||||
|
||||
这个 APK 用 Android 发布密钥签名，不走 minisign，因此只有一个 `.sha256`，没有 `.minisig`。
|
||||
|
||||
该密钥缺失时产物名为 `ReasonixMobile-<版本>-unsigned.apk`。Android 不接受未签名的包，
|
||||
这类构建只留作 workflow 产物供自行签名，发布页上不会挂 APK。
|
||||
|
||||
内核与界面都跑在手机上，Agent 通过 SSH 连接你自己的服务器；与电脑配对时扫它显示的二维码。
|
||||
|
||||
### Reasonix 1.x（稳定版）
|
||||
|
||||
1.x CLI 在任意支持的平台上都可以通过 npm 安装，macOS 也可以使用 Homebrew：
|
||||
@@ -278,11 +269,11 @@ CLI 进阶用法和详细配置见 **[CLI 命令参考](./docs/CLI.zh-CN.md)**
|
||||
<!-- reasonix-top-contributors:start -->
|
||||
| Contributor | Contributor | Contributor | Contributor |
|
||||
| --- | --- | --- | --- |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**KHG420**](https://github.com/KHG420) | [**ttmouse**](https://github.com/ttmouse) | [**chuiwenwei**](https://github.com/chuiwenwei) |
|
||||
| [**lifu963**](https://github.com/lifu963) | [**XTLine**](https://github.com/XTLine) | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**xxoingr**](https://github.com/xxoingr) |
|
||||
| [**Harbor404**](https://github.com/Harbor404) | [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) | [**Li-Charles-One**](https://github.com/Li-Charles-One) |
|
||||
| [**light-front-theory**](https://github.com/light-front-theory) | [**imcreazy**](https://github.com/imcreazy) | [**eghrhegpe**](https://github.com/eghrhegpe) | [**noreply**](https://github.com/noreply) |
|
||||
| [**par73e**](https://github.com/par73e) | [**Bernardxu123**](https://github.com/Bernardxu123) | [**lancy69**](https://github.com/lancy69) | [**Guan0923**](https://github.com/Guan0923) |
|
||||
| [**SivanCola**](https://github.com/SivanCola) | [**KHG420**](https://github.com/KHG420) | [**ttmouse**](https://github.com/ttmouse) | [**lifu963**](https://github.com/lifu963) |
|
||||
| [**XTLine**](https://github.com/XTLine) | [**HUQIANTAO**](https://github.com/HUQIANTAO) | [**chuiwenwei**](https://github.com/chuiwenwei) | [**Harbor404**](https://github.com/Harbor404) |
|
||||
| [**GTC2080**](https://github.com/GTC2080) | [**mchenziyi**](https://github.com/mchenziyi) | [**Li-Charles-One**](https://github.com/Li-Charles-One) | [**light-front-theory**](https://github.com/light-front-theory) |
|
||||
| [**xxoingr**](https://github.com/xxoingr) | [**eghrhegpe**](https://github.com/eghrhegpe) | [**noreply**](https://github.com/noreply) | [**par73e**](https://github.com/par73e) |
|
||||
| [**Bernardxu123**](https://github.com/Bernardxu123) | [**Guan0923**](https://github.com/Guan0923) | [**HaoyueQin**](https://github.com/HaoyueQin) | [**BuGlessRB**](https://github.com/BuGlessRB) |
|
||||
<!-- reasonix-top-contributors:end -->
|
||||
|
||||
特别感谢 [**Bernardxu123**](https://github.com/Bernardxu123) 设计的项目 logo和开场视频。
|
||||
|
||||
@@ -20,25 +20,10 @@ plugins {
|
||||
}
|
||||
|
||||
// One version, read by the About screen and the update check.
|
||||
//
|
||||
// A published build takes its version from the tag it was cut for, so the About
|
||||
// screen and the update check name that release rather than the number this file
|
||||
// was last edited to. Without the property — a local build — the numbers below
|
||||
// are the answer.
|
||||
def publishedVersion = findProperty("reasonixVersion") as String
|
||||
def versionMajor = 2
|
||||
def versionMinor = 0
|
||||
def versionPatch = 0
|
||||
def versionBuild = 1
|
||||
if (publishedVersion) {
|
||||
def parsed = publishedVersion =~ /^(\d+)\.(\d+)\.(\d+)/
|
||||
if (!parsed.find()) {
|
||||
throw new GradleException("reasonixVersion must start with MAJOR.MINOR.PATCH, got: ${publishedVersion}")
|
||||
}
|
||||
versionMajor = parsed.group(1) as int
|
||||
versionMinor = parsed.group(2) as int
|
||||
versionPatch = parsed.group(3) as int
|
||||
}
|
||||
|
||||
// Where the two build inputs are expected. Both are produced from the checkout
|
||||
// beside this module, by the scripts named in each message.
|
||||
@@ -68,12 +53,6 @@ android {
|
||||
// what `MainActivity` feeds real inset values for, since a WebView does
|
||||
// not implement `env(safe-area-inset-*)`.
|
||||
targetSdk 35
|
||||
// versionCode must strictly increase, because Android refuses an update
|
||||
// whose versionCode is not higher. The build digit is a single decimal
|
||||
// place, so the scheme collides once a patch reaches 10: 2.55.10 and
|
||||
// 2.56.0 both come out as 25601. Patch releases above 9 are not expected
|
||||
// on Studio, and widening the digit now would renumber every build, so
|
||||
// the constraint is recorded rather than fixed.
|
||||
versionCode versionMajor * 10000 + versionMinor * 100 + versionPatch * 10 + versionBuild
|
||||
versionName "${versionMajor}.${versionMinor}.${versionPatch}"
|
||||
|
||||
|
||||
+2806
-2917
File diff suppressed because it is too large.
Load diff
@@ -47,7 +47,7 @@ app_gradle="$android/app/build.gradle"
|
||||
version_field() {
|
||||
sed -n "s/^def $1 *= *\([0-9]*\).*/\1/p" "$app_gradle" | head -n1
|
||||
}
|
||||
app_version="${REASONIX_APP_VERSION:-$(version_field versionMajor).$(version_field versionMinor).$(version_field versionPatch)}"
|
||||
app_version="$(version_field versionMajor).$(version_field versionMinor).$(version_field versionPatch)"
|
||||
build_time_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
ldflags="-s -w"
|
||||
ldflags="$ldflags -X main.version=v$app_version"
|
||||
|
||||
@@ -32,8 +32,7 @@ fi
|
||||
|
||||
# 32 random bytes as hex — the same shape `Kernel.mintToken` writes.
|
||||
token=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n')
|
||||
# The kernel refuses a token file other users can read.
|
||||
(umask 077; printf '%s\n' "$token" > "$work/home/remote/launch.token")
|
||||
printf '%s\n' "$token" > "$work/home/remote/launch.token"
|
||||
|
||||
export REASONIX_HOME="$work/home"
|
||||
export HOME="$work/home"
|
||||
|
||||
+4
-64
@@ -1,6 +1,6 @@
|
||||
# Reasonix Benchmarks
|
||||
|
||||
Six harnesses live under `benchmarks/`; `cmd/e2ebench` also exposes a
|
||||
Five harnesses live under `benchmarks/`; `cmd/e2ebench` also exposes a
|
||||
SWE-bench Verified mode:
|
||||
|
||||
- `e2e/` — the committed end-to-end task suite, driven by
|
||||
@@ -19,10 +19,6 @@ SWE-bench Verified mode:
|
||||
- `fanout-width/` — a shape-instructed fleet whose members share nothing, for
|
||||
pricing what running them at the same time is worth. See
|
||||
[Pricing the run graph](#pricing-the-run-graph) below.
|
||||
- `realrepo/` — long tasks over a pinned snapshot of this repository, for
|
||||
measuring changes that cut model round trips (13–31 requests per task on
|
||||
`deepseek-flash`). See
|
||||
[its README](realrepo/README.md).
|
||||
|
||||
## Directory layout
|
||||
|
||||
@@ -420,7 +416,7 @@ own outcome).
|
||||
|
||||
| Flag | Default | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `-mode` | `suite` | `suite` \| `diff` \| `swebench` \| `compare` \| `traj` (`diff` generates tests for the PR diff; `swebench` runs the official per-instance evaluation; `compare` renders KPI/Pareto readouts from 2+ `-json` reports; `traj` re-digests recorded trajectory files without spending tokens; `audit` checks a batch's published figures against its retained rows). |
|
||||
| `-mode` | `suite` | `suite` \| `diff` \| `swebench` \| `compare` \| `traj` (`diff` generates tests for the PR diff; `swebench` runs the official per-instance evaluation; `compare` renders KPI/Pareto readouts from 2+ `-json` reports; `traj` re-digests recorded trajectory files without spending tokens). |
|
||||
| `-suite` | `benchmarks/e2e` | Suite root (must contain `tasks/<id>/`). |
|
||||
| `-task` | *(all)* | Suite mode: run only these comma-separated task IDs (e.g. `-task fix-add-bug`); unknown IDs fail with the available list. |
|
||||
| `-attempts` | `1` | Suite and diff modes: retry a task until an attempt passes, up to N; enables the `Pass@≤N` KPI, and TTCS charges a retried solve with its failed attempts' wall. |
|
||||
@@ -433,7 +429,6 @@ own outcome).
|
||||
| `-ablate` | *(none)* | Ablation arm: comma-separated subsystems to switch off — `evidence`, `planner`, `subagent`, `retrieval`, `compaction`, `upstream` and `recall-search`; `none` \| `all`. |
|
||||
| `-out` | *(stdout)* | Write the markdown report here. |
|
||||
| `-json` | *(none)* | Write the JSON report here (optional). |
|
||||
| `-batch` | *(none)* | Suite mode: write `results.jsonl` (one row per attempt) and `batch.json` (manifest v1) here, so the report's figures can be audited later. Audit mode: read the batch from here. See [Auditing a batch](#auditing-a-batch). |
|
||||
| `-trajectories` | *(none)* | Suite mode: write one `<task-id>.trajectory.jsonl` per task into this directory (the agent's full event stream with timestamps — see `reasonix run --trajectory`). The report gains a time-attribution line (tools vs. model) and each JSON result a `trajectory` digest. |
|
||||
| `-force-planner` | `false` | Suite mode: prefix each prompt with a plan-first directive so the two-model turn engages regardless of the planner gate. Use for the "with planner" arm of an A/B; results carry `plan_forced` so arms are only comparable with equal forcing. |
|
||||
| `-anchor` | `blind` | Suite mode: which hypothesis the agent holds before it looks at anything — `blind` (none, the control) \| `correct` \| `wrong`. The seeded arms prefix each prompt with the task's authored seed and **skip** tasks that have none, so an unseeded control run never lands in a seeded denominator. Results carry `anchor`. See [Anchor resistance](#anchor-resistance). |
|
||||
@@ -456,65 +451,10 @@ Diff-mode flags:
|
||||
| `-timeout` | `1200` | Agent timeout in seconds (diff mode). |
|
||||
| `-attempts` | `1` | Diff mode: retry up to N times until a run passes (stochastic agent). |
|
||||
|
||||
## Auditing a batch
|
||||
|
||||
`-batch DIR` writes two files from a suite run: `results.jsonl` (one row per
|
||||
attempt — the raw record every figure is computed from) and `batch.json`
|
||||
(manifest v1). `-mode audit -batch DIR` re-derives the report's headline figures
|
||||
from `results.jsonl`, prints the manifest's claim beside the recomputed value,
|
||||
and exits non-zero naming any figure that moved. It also refuses a batch whose
|
||||
manifest is missing, records no commit, or whose rows are truncated.
|
||||
|
||||
This is an **aggregation audit, not a replay**: it proves the published figures
|
||||
follow from the retained rows. It does not re-execute the model, tools or
|
||||
environment, and cannot show the experiment would reproduce. A real replay would
|
||||
need the commit, a corpus fingerprint, the model and provider configuration, the
|
||||
seed, the flags, the raw results and the runtime version; it is not implemented.
|
||||
|
||||
`schema_version` is top level. The rest is three groups:
|
||||
|
||||
| Group | Fields | Answers |
|
||||
| --- | --- | --- |
|
||||
| `over` | `command.mode` and `command.axes` (every flag that changes what a figure means; paths omitted); `tasks.count` and `tasks.digest` (sha256 over sorted, NUL-separated task ids — the corpus identity, never the task text); `filters`; `denominator` (`unit`, `definition`, `value`) | what the figures are over |
|
||||
| `producer` | `commit` (or an explicit `unknown`, never a guess), `commit_source`, `dirty`, `model`, `provider`, `seed`, `started_at`, `finished_at`, `harness_version`, `go_version` | which run produced them |
|
||||
| `trust` | `results_sha256`, `claimed` (the headline figures), `ledger`, `retention` | why they can still be checked |
|
||||
|
||||
The ledger accounts for every planned run, and the audit checks that the counts
|
||||
form a funnel (`scored ≤ completed ≤ attempted`, `failed ≤ scored`,
|
||||
`scored = pass1 + failed`, `rows = attempted + skipped`). A row count is never a
|
||||
denominator: `denominator.value` is a count of scored tasks, and `rows` is
|
||||
published only to show what was retained.
|
||||
|
||||
| Count | Meaning |
|
||||
| --- | --- |
|
||||
| `planned` | graded runs the batch set out to attempt: filtered tasks × trials |
|
||||
| `attempted` | retained rows for runs that started (skipped rows are not attempts) |
|
||||
| `completed` | attempts that ended and left a row; this harness writes a row only after an attempt ends, so it equals `attempted` and the equality is checked, not assumed |
|
||||
| `scored` | completed first attempts of solvable, non-skipped tasks — the pass@1 denominator |
|
||||
| `failed` | scored attempts the grader did not pass |
|
||||
| `unmeasured` | attempts whose cost and tokens are unknown because no metrics landed; kept out of spend totals, never counted as zero |
|
||||
| `excluded` | rows kept out of the accuracy denominator: skipped, `no_solution`, and retry entries |
|
||||
|
||||
Spend and cache fields are omitted from `claimed` when nothing was measured, so
|
||||
an absent figure reads as unknown rather than as a measured zero. `seed` is
|
||||
omitted for the same reason: the harness does not set a sampling seed.
|
||||
`results_sha256` covers the values and the presence of every row; line endings,
|
||||
indentation and key order are normalised away because none of them is evidence,
|
||||
while number spelling is kept (a rewritten `1` is not `1.0`).
|
||||
A successful audit records the verifier version (`audit.verifier_version`) and
|
||||
the digest it checked in `batch.json`, so a re-verification can tell which code
|
||||
checked the batch.
|
||||
|
||||
`results.jsonl` is retained unredacted beside the manifest — task prompts and
|
||||
agent notes included — and is not public. The manifest itself carries a task-id
|
||||
digest, no prompt text, no absolute paths, no endpoints or keys and no model
|
||||
output. A claim whose rows are gone cannot be audited: there is nothing to
|
||||
recompute from, and this tool will not certify it.
|
||||
|
||||
## Dataset retention
|
||||
|
||||
Keep every `-json` report, `-trajectories` directory and `-batch` directory from
|
||||
real runs: they are the accumulating corpus — per-task contracts-to-be, full event
|
||||
Keep every `-json` report and `-trajectories` directory from real runs: they
|
||||
are the accumulating corpus — per-task contracts-to-be, full event
|
||||
trajectories, checkpoint oracle verdicts, stop curves and phase traces — that
|
||||
any future offline learning (routing, stop policies, budgets) would train
|
||||
and evaluate on. The control plane stays deterministic and interpretable
|
||||
|
||||
@@ -103,6 +103,10 @@ func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorde
|
||||
ContextWindow: window,
|
||||
ArchiveDir: filepath.Join(t.dir, "archive"),
|
||||
SessionPath: path,
|
||||
RecentKeep: 4,
|
||||
// boot's default when cfg.Agent.Keep is unset; without it the bench
|
||||
// would measure a configuration no real session runs.
|
||||
KeepPolicy: agent.KeepErrors,
|
||||
}, rec.sink())
|
||||
return &harness{sess: sess, agentA: a, path: path, calls: rec}
|
||||
}
|
||||
|
||||
@@ -108,6 +108,7 @@ func newAgent(p provider.Provider, sess *sessionstore.Session, path string, sink
|
||||
return agent.New(p, tool.NewRegistry(), sess, agent.Options{
|
||||
ContextWindow: windowTokens,
|
||||
CompactRatio: compactRatio,
|
||||
RecentKeep: 2,
|
||||
SessionPath: path,
|
||||
WorkspaceID: "cm-e2e",
|
||||
ModelRef: "deepseek/" + model,
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
# Context retrieval bench
|
||||
|
||||
Two questions about long-horizon memory, and the harness that makes their
|
||||
answers trustworthy.
|
||||
|
||||
-mode=preflight build all 18 tasks under every arm and check the contract
|
||||
-mode=snippet does a rank-1 hit hand over the whole answer?
|
||||
-mode=adversarial send a model after the answer through every host surface
|
||||
-mode=run-search Experiment R: is searchable canonical a real capability?
|
||||
-mode=run-boundary Experiment I, paired: the 6 efficiency tasks across their boundary
|
||||
-mode=run-index Experiment I, full: the 6 efficiency tasks x 4 budgets
|
||||
-mode=calibrate find each index task's PlantAfterGen for its cue tier
|
||||
|
||||
`-dry` drives any run mode with a scripted provider, for nothing. Real runs need
|
||||
`DEEPSEEK_API_KEY`.
|
||||
|
||||
## Status
|
||||
|
||||
**Searchable canonical recall: proven.** Six of six recovered with search on,
|
||||
three of six with it off, and the successes without it came from enumerating
|
||||
addresses by hand at roughly nine times the page-in cost (139 vs 1292
|
||||
tokens/task). Measured after three separate leaks were closed, so the numbers
|
||||
are from a clean environment.
|
||||
|
||||
**Fold index marginal utility: direction reproduced, size still conditional.** On the v2
|
||||
substrate the direction holds in six of six tasks and in the pooled paired batch: with the
|
||||
cue visible the index addresses the target, so the same task is searched less and pages in
|
||||
less (0.67 searches and 70 recall tokens per task, against 1.50 and 120 with the cue
|
||||
absent).
|
||||
|
||||
A size is visible but is not yet quotable. On the four tasks whose absent-cue arm stayed
|
||||
out of a loop, the index cost 64,952 prompt tokens against 115,551 without, 44% less
|
||||
(`i106-budgetseal` -35,147, `i70-ingestfloor` -8,222, `i37-dispatchlag` -7,599, and
|
||||
`i112-lagquota` +369, the prefix rent a task that never searches pays). That is a
|
||||
**post-hoc subset**: `i118-fencelag` and `i47-tenantlease` are excluded because their
|
||||
absent-cue arm looped in this batch (five and three retrieval rounds, three escapes each;
|
||||
together 80% of the absent-cue total), and the qualification above does not screen for
|
||||
stopping behaviour — holding on both boundary arms across three passes shows a stable cue
|
||||
boundary, not a stopping arm. Quoting 44% as the index's effect would mean choosing the
|
||||
sample after seeing the result; a batch screened on stopping stability is what would make
|
||||
a size quotable, and one arm also reverses either way (`i70-ingestfloor` default, +11,135
|
||||
prompt tokens over three escapes).
|
||||
|
||||
Batch facts, because the counts differ: 36 runs were executed (12 paired, 24 index), and
|
||||
the token totals here cover the 24-run index batch — the paired batch ran before the
|
||||
per-run usage line existed.
|
||||
|
||||
**Things that turned out not to be problems**, each after being measured rather
|
||||
than argued about:
|
||||
|
||||
| Suspected | Measured |
|
||||
| --- | --- |
|
||||
| Model writes poor queries (2.17 searches/task) | 22/23 found the target on query 1; the extra searches are one round's parallel fan-out |
|
||||
| Model reaches for the workspace before memory (7/11) | Linear event order misread; by model round it is MemoryFirst 4/6 |
|
||||
| Snippets too short for multi-value answers | 240 runes covers 12/12 tasks at 100%; records are 30-179 runes |
|
||||
| A general stopping failure | 37/44 runs issued no search after the answer was in hand; all 7 that did belong to 2 tasks |
|
||||
|
||||
## Measurement distinctions this bench had to learn
|
||||
|
||||
Each of these reversed a conclusion once. They are enforced in the schema now,
|
||||
not left to whoever reads the numbers next.
|
||||
|
||||
- **TargetHit is not EvidenceSufficient.** A rank-1 hit can return a window too
|
||||
short to answer with. `FirstHitCoverage` records how much of the scored answer
|
||||
the first hit actually handed over.
|
||||
- **Linear event order is not model-round order.** Two calls in one round are a
|
||||
parallel fan-out, not a preference. Routing compares rounds.
|
||||
- **An isolated workdir is not an isolated host.** The sandbox mounts the host
|
||||
read-only by design. Three leaks were found and closed: the corpus source, the
|
||||
fixture transcript, and its event log sidecar.
|
||||
- **A count is not its evidence.** Query text, snippet contents and trajectories
|
||||
are all persisted now, because three analyses in a row needed a paid re-run to
|
||||
ask a question of data already collected.
|
||||
|
||||
## How a run stays honest
|
||||
|
||||
The corpus holds templates, never answers: every scored literal is a `{{var}}`
|
||||
instantiated per run, so grepping this directory reveals the question and not
|
||||
the answer. `TestNoAnswerLiteralExistsInTheRepository` asserts it with
|
||||
`git grep`. Preflight rebuilds every arm and checks the answer is unreadable in
|
||||
the real `provider.Request`, the probe query ranks the target within five, and
|
||||
an index task's cue sits at exactly the scales its tier names. The fixture is
|
||||
removed from disk once the agent holds it in memory, and the directory is
|
||||
scanned for answer literals before the first provider request and again after
|
||||
the turn. Any answer reaching the model through a tool that is not `recall`
|
||||
marks the run contaminated and takes it out of every statistic.
|
||||
|
||||
## Index corpus v2
|
||||
|
||||
The efficiency substrate is six tasks that hold on both arms: `i37-dispatchlag`
|
||||
and `i118-fencelag` (quarter), `i106-budgetseal` and `i70-ingestfloor` (half),
|
||||
`i47-tenantlease` and `i112-lagquota` (default).
|
||||
|
||||
They came out of five batches of candidates, each screened the same way: one
|
||||
`-mode=run-index` pass over the batch, then two more only for the candidates that
|
||||
were clean on both boundary arms, and only the candidates that held on both arms
|
||||
in all three passes stayed. A candidate that does not hold is dropped, not
|
||||
promoted. The numbers are candidates, per tier:
|
||||
|
||||
| batch | quarter | half | default |
|
||||
| --- | --- | --- | --- |
|
||||
| first pool, `i31`-`i54` | 1 of 8 | 0 of 8 | 2 of 8 |
|
||||
| second pool, `i61`-`i84` | 1 of 4 screened | 4 of 8 | 0 of 8 |
|
||||
| third pool, `i85`-`i108` | 2 of 8 | 8 of 16 | none authored |
|
||||
| replacement pool, `i109`-`i116` | - | - | 4 of 8 |
|
||||
| replacement pool, `i92`, `i117`-`i124` | 3 of 9 | - | - |
|
||||
|
||||
That table counts candidates that held on both arms in all three passes, which is
|
||||
not the size of the substrate. 89 candidates were authored, 77 were screened (the
|
||||
second pool's quarter tier stopped at 4 of 8 and its default tier was never run),
|
||||
and 25 held: 7 quarter, 12 half, 6 default. The substrate keeps two per tier, so
|
||||
the other 19 survivors are spares rather than corpus members. Candidates from the
|
||||
earlier round are not counted here.
|
||||
|
||||
| task | tier | cue present | cue absent |
|
||||
| --- | --- | ---: | ---: |
|
||||
| `i37-dispatchlag` | quarter | 3/3 | 3/3 |
|
||||
| `i118-fencelag` | quarter | 3/3 | 3/3 |
|
||||
| `i106-budgetseal` | half | 3/3 | 3/3 |
|
||||
| `i70-ingestfloor` | half | 3/3 | 3/3 |
|
||||
| `i47-tenantlease` | default | 3/3 | 3/3 |
|
||||
| `i112-lagquota` | default | 3/3 | 3/3 |
|
||||
|
||||
Across their 36 judged cells: `SnippetStop` 23, `SnippetThenRead` 10 and
|
||||
`DefensiveRead` 3 - every cell clean, nothing post-sufficient and nothing
|
||||
`NeverSufficient`.
|
||||
|
||||
Four of the first six were flagged in review for sharing answer values, probe
|
||||
queries and prompt wording within a tier. Separating them and re-screening those
|
||||
four changed two of them: `i37` and `i47` held, `i62` and `i51` did not, so they
|
||||
were replaced - `i90-recoverylag` was tried first and did not hold either, and
|
||||
`i118-fencelag` and `i112-lagquota` are what did. The separation was not
|
||||
cosmetic: the shared wording had been carrying two of the six.
|
||||
|
||||
Nothing in the task shape separates the survivors from the rest: within the half
|
||||
tier, plain-number answers survived 0 of 8 in the first pool and 4 of 8 in the
|
||||
second, and codename answers survived 8 of 16 in the third. An earlier claim here
|
||||
that the answer kind decides whether a candidate holds does not survive this
|
||||
table and is withdrawn. The dirty class was `PostSufficientRetrieval` in most
|
||||
failures - the model re-checking a value it already had - but which tasks hit it
|
||||
was not predictable from their shape at this sample.
|
||||
|
||||
`i01`-`i06` are the frozen study's six tasks, kept as `roleStopping`. Measured
|
||||
once each with `-mode=run-index -task <id>`, per task and per arm:
|
||||
|
||||
| task | tier | cue present | cue absent |
|
||||
| --- | --- | --- | --- |
|
||||
| `i01-transport-fallback` | quarter | SnippetStop | DefensiveRead |
|
||||
| `i02-scheduler-handoff` | quarter | PostSufficientRetrieval | PostSufficientRetrieval |
|
||||
| `i03-coalescing` | half | PostSufficientRetrieval | PostSufficientRunaway |
|
||||
| `i04-recovery-fence` | half | DefensiveRead | PostSufficientRunaway |
|
||||
| `i05-cache-probe` | default | SnippetThenRead | PostSufficientRetrieval |
|
||||
| `i06-dispatch-handoff` | default | SnippetStop | PostSufficientRetrieval |
|
||||
|
||||
Five of the six show post-sufficient behaviour in at least one arm (`i02`-`i06`),
|
||||
and `i01` is clean on both. The cue-present arm is clean for four of them
|
||||
(`i01`, `i04`, `i05`, `i06`), which is why they stay in the corpus as the
|
||||
stopping side rather than being retitled. All six are excluded from `run-index`
|
||||
and `run-boundary` batches and stay measurable one at a time with `-task <id>`. A
|
||||
task added to the corpus declares a role, so it cannot arrive without one.
|
||||
|
||||
A task qualifies only if it is clean on both of its boundary arms - the
|
||||
cue-present run and the cue-absent run - in every run of a screening batch. Clean
|
||||
means a `stopping_class` other than `PostSufficientRetrieval` or
|
||||
`PostSufficientRunaway`: `SnippetStop`, `SnippetThenRead` and `DefensiveRead`
|
||||
all qualify, and a defensive read is deliberately not counted as waste. The
|
||||
class comes from the three counts in that report — `searches_after_sufficient`,
|
||||
`reads_after_sufficient` and `escapes_after_sufficient` — together with how
|
||||
sufficiency was reached and how many rounds followed it. The remaining
|
||||
requirements still apply — successful tool call, answer in the output rather
|
||||
than the cue, rank 1, `FirstHitCoverage` complete, one or two simple values,
|
||||
nothing in the workspace.
|
||||
|
||||
`-mode=calibrate` is not a screening tool: it builds and reads back the
|
||||
fixtures, and reads nothing else, so its cue-visibility profile is the same for
|
||||
every task.
|
||||
|
||||
Then repeats rather than levels: cue-present against cue-absent on each task's
|
||||
own boundary, three runs per cell. What is unknown is the variance, and four
|
||||
budgets sampled once each cannot show it.
|
||||
@@ -0,0 +1,86 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// runBlockedBatch runs one dry index batch where every arm's runOne fails
|
||||
// before any model turn: the task's own directory is a regular file, so the
|
||||
// fixture cannot be built. In production the same early return happens when the
|
||||
// fixture build, the session load, or the answer-on-disk guard fails.
|
||||
func runBlockedBatch(t *testing.T) string {
|
||||
t.Helper()
|
||||
tk := indexTasks()[0]
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, tk.ID), []byte("not a directory\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code := runExperiment("run-index", experimentIndex, root, true, []contextTask{tk}); code != 0 {
|
||||
t.Fatalf("runExperiment exit = %d, want 0 (every run failed, but the batch finished)", code)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func readResultLines(t *testing.T, root string) ([]contextMetrics, []string) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(root, "results.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var runs []contextMetrics
|
||||
var lines []string
|
||||
for text := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") {
|
||||
var m contextMetrics
|
||||
if err := json.Unmarshal([]byte(text), &m); err != nil {
|
||||
t.Fatalf("results.jsonl: %v", err)
|
||||
}
|
||||
runs = append(runs, m)
|
||||
lines = append(lines, text)
|
||||
}
|
||||
return runs, lines
|
||||
}
|
||||
|
||||
// A run that failed before the model turn produced no measurement. Recording it
|
||||
// as a zero-valued contextMetrics puts it in the funnel's Scored denominator,
|
||||
// in "Where it broke", and in the dose curve as a real zero.
|
||||
func TestBatchDoesNotRecordAFailedRunAsAResult(t *testing.T) {
|
||||
root := runBlockedBatch(t)
|
||||
runs, lines := readResultLines(t, root)
|
||||
if len(runs) == 0 {
|
||||
t.Fatal("no runs recorded at all")
|
||||
}
|
||||
for i, m := range runs {
|
||||
if m.Task == "" || m.Arm == "" || m.FailureStage == "" {
|
||||
t.Fatalf("results.jsonl line %d records a run that never ran as a result "+
|
||||
"(task=%q arm=%q failure_stage=%q):\n %s", i+1, m.Task, m.Arm, m.FailureStage, lines[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The manifest is the artefact that says which runs counted, so a failed run
|
||||
// must not be added to CompletedRuns or to the task id list.
|
||||
func TestBatchManifestDoesNotReportFailedRunsAsCompleted(t *testing.T) {
|
||||
root := runBlockedBatch(t)
|
||||
var manifest batchManifest
|
||||
data, err := os.ReadFile(filepath.Join(root, "batch.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range manifest.TaskIDs {
|
||||
if id == "" {
|
||||
t.Fatalf("manifest lists an empty task id: task_ids=%q (completed=%d planned=%d)",
|
||||
manifest.TaskIDs, manifest.CompletedRuns, manifest.PlannedRuns)
|
||||
}
|
||||
}
|
||||
if manifest.CompletedRuns == manifest.PlannedRuns {
|
||||
t.Fatalf("manifest reports every planned run completed although none produced evidence: "+
|
||||
"completed=%d planned=%d task_ids=%q", manifest.CompletedRuns, manifest.PlannedRuns, manifest.TaskIDs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// batchInfo is what the mode itself knows before any run happens. Everything
|
||||
// else in the manifest is derived from the runs that were written, so a figure
|
||||
// in it cannot drift from the artifact beside it.
|
||||
type batchInfo struct {
|
||||
Mode string
|
||||
Planned int
|
||||
Started time.Time
|
||||
Dry bool
|
||||
// Repo is the checkout the batch ran from, captured before the harness
|
||||
// isolates its workspace: git cannot describe a directory that is gone.
|
||||
Repo string
|
||||
}
|
||||
|
||||
// batchManifest is written beside results.jsonl so a later reader does not have
|
||||
// to reconstruct what the batch was from a terminal scrollback.
|
||||
type batchManifest struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Mode string `json:"mode"`
|
||||
StartedAt string `json:"started_at"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
GitCommit string `json:"git_commit"`
|
||||
GitDirty *bool `json:"git_dirty,omitempty"`
|
||||
Model string `json:"model"`
|
||||
Window int `json:"window"`
|
||||
Generations int `json:"generations"`
|
||||
TaskIDs []string `json:"task_ids"`
|
||||
Arms []string `json:"arms"`
|
||||
PlannedRuns int `json:"planned_runs"`
|
||||
CompletedRuns int `json:"completed_runs"`
|
||||
// FailedRuns counts the planned runs that ended before a transcript existed.
|
||||
// They are rows in results.jsonl, never part of CompletedRuns, and the field
|
||||
// is absent when there were none so an intact batch is unchanged.
|
||||
FailedRuns int `json:"failed_runs,omitempty"`
|
||||
// Usage totals are absent when nothing reported any: an unmeasured cost must
|
||||
// not read as a measured zero. UsageReportedRuns stays, because zero there
|
||||
// is a fact about the batch.
|
||||
UsageReportedRuns int `json:"usage_reported_runs"`
|
||||
UsageReportedRequests int `json:"usage_reported_requests,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
CacheHitTokens int `json:"cache_hit_tokens,omitempty"`
|
||||
CacheMissTokens int `json:"cache_miss_tokens,omitempty"`
|
||||
UsageEstimated bool `json:"usage_estimated,omitempty"`
|
||||
ResultsSHA256 string `json:"results_sha256"`
|
||||
TrajectoriesSHA256 string `json:"trajectories_sha256"`
|
||||
}
|
||||
|
||||
// gitCommitAndDirty reports the checkout the batch ran from. Unknown is written
|
||||
// rather than omitted, so an absent field is never mistaken for a clean tree.
|
||||
func gitCommitAndDirty(repo string) (commit string, dirty *bool) {
|
||||
commit = "unknown"
|
||||
if repo == "" {
|
||||
return commit, nil
|
||||
}
|
||||
in := func(args ...string) ([]byte, error) {
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = repo
|
||||
return cmd.Output()
|
||||
}
|
||||
if out, err := in("rev-parse", "--short", "HEAD"); err == nil {
|
||||
commit = strings.TrimSpace(string(out))
|
||||
}
|
||||
if out, err := in("status", "--porcelain"); err == nil {
|
||||
state := strings.TrimSpace(string(out)) != ""
|
||||
dirty = &state
|
||||
}
|
||||
return commit, dirty
|
||||
}
|
||||
|
||||
func fileSHA256(path string) (string, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%x", sha256.Sum256(data)), nil
|
||||
}
|
||||
|
||||
func sortedKeys(set map[string]bool) []string {
|
||||
out := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// writeBatchManifest records the batch beside the runs it describes. It runs
|
||||
// last, so its digests are of what is already on disk.
|
||||
func writeBatchManifest(root string, info batchInfo, all []contextMetrics) error {
|
||||
ids, arms := map[string]bool{}, map[string]bool{}
|
||||
var prompt, completion, cacheHit, cacheMiss, requests, reported, failed int
|
||||
estimated := false
|
||||
for _, m := range all {
|
||||
// A failed run has no task, arm or evidence to contribute: it is counted
|
||||
// as a failure below and is not part of what the batch measured.
|
||||
if m.Failed {
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
ids[m.Task] = true
|
||||
arms[m.Arm] = true
|
||||
if m.UsageReportedRequests > 0 {
|
||||
reported++
|
||||
requests += m.UsageReportedRequests
|
||||
prompt += m.PromptTokens
|
||||
completion += m.CompletionTokens
|
||||
cacheHit += m.CacheHitTokens
|
||||
cacheMiss += m.CacheMissTokens
|
||||
estimated = estimated || m.UsageEstimated
|
||||
}
|
||||
}
|
||||
commit, dirty := gitCommitAndDirty(info.Repo)
|
||||
model := runModel
|
||||
if info.Dry {
|
||||
model = "scripted"
|
||||
}
|
||||
manifest := batchManifest{
|
||||
SchemaVersion: 1, Mode: info.Mode,
|
||||
StartedAt: info.Started.UTC().Format(time.RFC3339),
|
||||
FinishedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
GitCommit: commit, GitDirty: dirty,
|
||||
Model: model, Window: fixtureWindow, Generations: fixtureGenerations,
|
||||
TaskIDs: sortedKeys(ids), Arms: sortedKeys(arms),
|
||||
PlannedRuns: info.Planned, CompletedRuns: len(all) - failed, FailedRuns: failed,
|
||||
UsageReportedRuns: reported, UsageReportedRequests: requests,
|
||||
PromptTokens: prompt, CompletionTokens: completion,
|
||||
CacheHitTokens: cacheHit, CacheMissTokens: cacheMiss, UsageEstimated: estimated,
|
||||
}
|
||||
var err error
|
||||
if manifest.ResultsSHA256, err = fileSHA256(filepath.Join(root, "results.jsonl")); err != nil {
|
||||
return err
|
||||
}
|
||||
if manifest.TrajectoriesSHA256, err = fileSHA256(filepath.Join(root, "trajectories.jsonl")); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(filepath.Join(root, "batch.json"), append(data, '\n'), 0o644)
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func seedArtifacts(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
for _, name := range []string{"results.jsonl", "trajectories.jsonl"} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte("{}\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readManifest(t *testing.T, root string) (batchManifest, string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(root, "batch.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got batchManifest
|
||||
if err := json.Unmarshal(data, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got, string(data)
|
||||
}
|
||||
|
||||
func TestBatchManifestRecordsWhatTheBatchWas(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
seedArtifacts(t, root)
|
||||
all := []contextMetrics{
|
||||
{Task: "b", Arm: "index-off"},
|
||||
{Task: "a", Arm: "index-off"},
|
||||
{Task: "a", Arm: "index-half", UsageReportedRequests: 2, PromptTokens: 100,
|
||||
CompletionTokens: 10, CacheHitTokens: 60, CacheMissTokens: 40},
|
||||
}
|
||||
if err := writeBatchManifest(root, batchInfo{Mode: "run-index", Planned: 4, Started: time.Now().Add(-time.Minute)}, all); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := readManifest(t, root)
|
||||
if got.SchemaVersion != 1 || got.Mode != "run-index" || got.PlannedRuns != 4 || got.CompletedRuns != 3 {
|
||||
t.Fatalf("manifest header = %+v", got)
|
||||
}
|
||||
if strings.Join(got.TaskIDs, ",") != "a,b" || strings.Join(got.Arms, ",") != "index-half,index-off" {
|
||||
t.Fatalf("ids/arms = %v / %v, want sorted and unique", got.TaskIDs, got.Arms)
|
||||
}
|
||||
if got.UsageReportedRuns != 1 || got.UsageReportedRequests != 2 || got.PromptTokens != 100 ||
|
||||
got.CompletionTokens != 10 || got.CacheHitTokens != 60 || got.CacheMissTokens != 40 {
|
||||
t.Fatalf("usage totals = %+v", got)
|
||||
}
|
||||
// No repo means git could not describe the checkout: unknown commit, and no
|
||||
// dirty flag at all, because "clean" is a claim this batch cannot make.
|
||||
if got.GitCommit != "unknown" || got.GitDirty != nil {
|
||||
t.Fatalf("git state = %q / %v, want unknown and no dirty flag", got.GitCommit, got.GitDirty)
|
||||
}
|
||||
want := fmt.Sprintf("%x", sha256.Sum256([]byte("{}\n")))
|
||||
if got.ResultsSHA256 != want || got.TrajectoriesSHA256 != want {
|
||||
t.Fatalf("digests = %q / %q, want %q", got.ResultsSHA256, got.TrajectoriesSHA256, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A dry batch pays nothing, so its cost fields must be absent rather than zero:
|
||||
// an unmeasured spend must not read as a measured one.
|
||||
func TestBatchManifestLeavesUnreportedUsageAbsent(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
seedArtifacts(t, root)
|
||||
all := []contextMetrics{{Task: "a", Arm: "index-off"}, {Task: "a", Arm: "index-half"}}
|
||||
if err := writeBatchManifest(root, batchInfo{Mode: "run-index", Planned: 2, Started: time.Now(), Dry: true}, all); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, raw := readManifest(t, root)
|
||||
if got.UsageReportedRuns != 0 {
|
||||
t.Fatalf("usage_reported_runs = %d, want 0 reported", got.UsageReportedRuns)
|
||||
}
|
||||
if got.Model != "scripted" {
|
||||
t.Fatalf("dry batch model = %q, want scripted", got.Model)
|
||||
}
|
||||
for _, absent := range []string{"usage_reported_requests", "prompt_tokens", "completion_tokens", "cache_hit_tokens"} {
|
||||
if strings.Contains(raw, absent) {
|
||||
t.Fatalf("a dry batch wrote %q:\n%s", absent, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// boundaryRuns builds the input reportBoundaries reads: one scored run for an
|
||||
// index task's cue side, and one failed run for another index task's cue side.
|
||||
// The runs are constructed because the report is a pure function of the slice;
|
||||
// the batch path that produces them is covered by the failed-batch tests.
|
||||
func boundaryRuns(t *testing.T) (all []contextMetrics, scoredTask, failedTask string) {
|
||||
t.Helper()
|
||||
index := indexTasks()
|
||||
if len(index) < 2 {
|
||||
t.Fatalf("need at least two index tasks, have %d", len(index))
|
||||
}
|
||||
scored, failed := index[0], index[1]
|
||||
cue, noCue := boundaryPair(scored.CueTier)
|
||||
if cue == "" || noCue == "" {
|
||||
t.Fatalf("task %s has an empty boundary pair (%q, %q)", scored.ID, cue, noCue)
|
||||
}
|
||||
failedCue, _ := boundaryPair(failed.CueTier)
|
||||
if failedCue == "" {
|
||||
t.Fatalf("task %s has no cue side", failed.ID)
|
||||
}
|
||||
all = []contextMetrics{
|
||||
{Task: scored.ID, Arm: "index-" + cue, AnswerRecovered: true, SearchCalls: 1, RecallReturnedTokens: 318},
|
||||
{Task: scored.ID, Arm: "index-" + noCue, AnswerRecovered: false},
|
||||
{Task: failed.ID, Arm: "index-" + failedCue, Failed: true, FailureStage: stageNoEvidence},
|
||||
}
|
||||
return all, scored.ID, failed.ID
|
||||
}
|
||||
|
||||
// TestBoundaryReportExcludesFailedRuns: a run that never produced a transcript
|
||||
// must not appear in the cue-boundary table, which was the last reporting exit
|
||||
// that still folded it in as a zero measurement.
|
||||
func TestBoundaryReportExcludesFailedRuns(t *testing.T) {
|
||||
all, scoredTask, failedTask := boundaryRuns(t)
|
||||
out := captureStdout(t, func() { reportBoundaries(all) })
|
||||
if strings.Contains(out, failedTask) {
|
||||
t.Fatalf("the boundary table printed the failed task %q as a measurement:\n%s", failedTask, out)
|
||||
}
|
||||
if !strings.Contains(out, scoredTask) {
|
||||
t.Fatalf("the boundary table dropped the scored task %q, so the exclusion is not specific to failures:\n%s", scoredTask, out)
|
||||
}
|
||||
if !strings.Contains(out, "cue-present") || !strings.Contains(out, "cue-absent") {
|
||||
t.Fatalf("the pooled comparison disappeared:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBoundaryReportSaysNothingToCompareWhenEveryRunFailed: with no scored run
|
||||
// the report must say so, rather than printing a comparison of zeros.
|
||||
func TestBoundaryReportSaysNothingToCompareWhenEveryRunFailed(t *testing.T) {
|
||||
all, _, _ := boundaryRuns(t)
|
||||
failedOnly := make([]contextMetrics, 0, len(all))
|
||||
for _, m := range all {
|
||||
m.Failed = true
|
||||
m.FailureStage = stageNoEvidence
|
||||
failedOnly = append(failedOnly, m)
|
||||
}
|
||||
out := captureStdout(t, func() { reportBoundaries(failedOnly) })
|
||||
if !strings.Contains(out, "scored=0") || !strings.Contains(out, "failed=") {
|
||||
t.Fatalf("an all-failed batch did not report scored=0 with a failed count:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "cue-present") || strings.Contains(out, "cue-absent") {
|
||||
t.Fatalf("an all-failed batch still printed a pooled comparison of zeros:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// capture.go — what the agent actually hands a provider.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/assembly/boot"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/runtime/agent"
|
||||
_ "reasonix/internal/tools/builtin"
|
||||
)
|
||||
|
||||
// A projection is what the agent believes; a provider.Request is what the model
|
||||
// is told. Rendering, sanitization and tool-schema construction sit between
|
||||
// them, so the arm's contract is checked on the request.
|
||||
|
||||
type captureProvider struct {
|
||||
requests []provider.Request
|
||||
}
|
||||
|
||||
func (p *captureProvider) Name() string { return "contextbench-capture" }
|
||||
|
||||
// Stream records the request and answers with a clean terminal response. An
|
||||
// empty answer would buy an empty-final retry and a second request, which is
|
||||
// not the one round this is here to inspect.
|
||||
func (p *captureProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.requests = append(p.requests, cloneRequest(req))
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "contextbench preflight complete"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func cloneRequest(req provider.Request) provider.Request {
|
||||
out := req
|
||||
out.Messages = append([]provider.Message(nil), req.Messages...)
|
||||
out.Tools = append([]provider.ToolSchema(nil), req.Tools...)
|
||||
return out
|
||||
}
|
||||
|
||||
// captureRequest rebuilds the session the way a resume does — transcript from
|
||||
// disk, projection from its sidecar, the shipped tool surface at this arm —
|
||||
// and returns the request the first round hands the provider.
|
||||
func captureRequest(f builtFixture, arm ablation.Set, prompt string) (provider.Request, error) {
|
||||
sess, err := sessionstore.LoadSession(f.Path)
|
||||
if err != nil {
|
||||
return provider.Request{}, fmt.Errorf("load session: %w", err)
|
||||
}
|
||||
reg := tool.NewRegistry()
|
||||
for _, t := range tool.Builtins() {
|
||||
reg.Add(t)
|
||||
}
|
||||
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
|
||||
|
||||
cap := &captureProvider{}
|
||||
a := agent.New(cap, reg, sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, event.Discard)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
|
||||
if err := a.Run(context.Background(), prompt); err != nil {
|
||||
return provider.Request{}, fmt.Errorf("run: %w", err)
|
||||
}
|
||||
// The turn's own request is the one carrying tools. A fold on the way in
|
||||
// asks the same provider for a digest, and that request has none — taking
|
||||
// the first request blindly would inspect the summarizer's prompt.
|
||||
for _, req := range cap.requests {
|
||||
if len(req.Tools) > 0 {
|
||||
return req, nil
|
||||
}
|
||||
}
|
||||
return provider.Request{}, fmt.Errorf("no request carried tools (%d captured)", len(cap.requests))
|
||||
}
|
||||
|
||||
// surfaceField is one place a model could read a string, named so a leak
|
||||
// report says where rather than only that.
|
||||
type surfaceField struct {
|
||||
Path string
|
||||
Text string
|
||||
}
|
||||
|
||||
// providerSurface enumerates every provider-visible string in a request.
|
||||
// Enumerated rather than marshalled whole: a Message also carries local-only
|
||||
// fields, and scanning those would fail on bytes no model ever sees.
|
||||
func providerSurface(req provider.Request) []surfaceField {
|
||||
var out []surfaceField
|
||||
add := func(path, text string) {
|
||||
if strings.TrimSpace(text) != "" {
|
||||
out = append(out, surfaceField{Path: path, Text: text})
|
||||
}
|
||||
}
|
||||
for i, m := range req.Messages {
|
||||
add(fmt.Sprintf("messages[%d].content", i), m.Content)
|
||||
add(fmt.Sprintf("messages[%d].reasoning_content", i), m.ReasoningContent)
|
||||
add(fmt.Sprintf("messages[%d].name", i), m.Name)
|
||||
for j, tc := range m.ToolCalls {
|
||||
add(fmt.Sprintf("messages[%d].tool_calls[%d].name", i, j), tc.Name)
|
||||
add(fmt.Sprintf("messages[%d].tool_calls[%d].arguments", i, j), tc.Arguments)
|
||||
}
|
||||
for j, item := range m.ResponsesItems {
|
||||
add(fmt.Sprintf("messages[%d].responses_items[%d]", i, j), string(item))
|
||||
}
|
||||
}
|
||||
for i, t := range req.Tools {
|
||||
add(fmt.Sprintf("tools[%d].name", i), t.Name)
|
||||
add(fmt.Sprintf("tools[%d].description", i), t.Description)
|
||||
if len(t.Parameters) > 0 {
|
||||
b, err := json.Marshal(t.Parameters)
|
||||
if err == nil {
|
||||
add(fmt.Sprintf("tools[%d].parameters", i), string(b))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// findInSurface reports every place a marker is readable.
|
||||
func findInSurface(req provider.Request, marker string) []string {
|
||||
var hits []string
|
||||
for _, f := range providerSurface(req) {
|
||||
if strings.Contains(f.Text, marker) {
|
||||
hits = append(hits, f.Path)
|
||||
}
|
||||
}
|
||||
return hits
|
||||
}
|
||||
|
||||
// recallToolSchema returns the recall tool as the provider sees it.
|
||||
func recallToolSchema(req provider.Request) (provider.ToolSchema, bool) {
|
||||
for _, t := range req.Tools {
|
||||
if t.Name == "recall" {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return provider.ToolSchema{}, false
|
||||
}
|
||||
|
||||
// localOnlyLeaks reports host-side fields that should never survive the
|
||||
// provider boundary. A hit here is a boundary regression, marker or no marker.
|
||||
func localOnlyLeaks(req provider.Request) []string {
|
||||
var out []string
|
||||
for i, m := range req.Messages {
|
||||
switch {
|
||||
case m.RawContent != "":
|
||||
out = append(out, fmt.Sprintf("messages[%d].raw_content", i))
|
||||
case m.LocalOnly:
|
||||
out = append(out, fmt.Sprintf("messages[%d].local_only", i))
|
||||
case m.DecisionReceipt != nil || len(m.DecisionReceipts) > 0:
|
||||
out = append(out, fmt.Sprintf("messages[%d].decision_receipt", i))
|
||||
case m.ToolExecution != nil:
|
||||
out = append(out, fmt.Sprintf("messages[%d].tool_execution", i))
|
||||
case len(m.MemoryCitations) > 0:
|
||||
out = append(out, fmt.Sprintf("messages[%d].memory_citations", i))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// searchWording is what a search-off arm must never be told. The arm exists to
|
||||
// reproduce the runtime before search; an arm offered a capability it does not
|
||||
// have is a different experiment.
|
||||
var searchWording = []string{"recall with a query", "search the whole folded region", "query still finds"}
|
||||
@@ -0,0 +1,671 @@
|
||||
// contextbench.go — what the model did with the affordance, read off structure.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/tokencount"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// Every judgement here reads a tool call's name and its JSON arguments, or a
|
||||
// canonical position, and never the model's prose. "I will search the history"
|
||||
// is not a search; a recall call with a query is.
|
||||
|
||||
// failure stages, in funnel order. Exactly one is reported per run, except for
|
||||
// NoEvidence: that stage belongs to a run with no measurement at all.
|
||||
const (
|
||||
stageReadMissed = "NoSearch" // never asked, and never read the target either
|
||||
stageSearchMiss = "SearchMiss" // searched, target never came back
|
||||
stageHitNotRead = "HitNotRead" // target was returned and never opened
|
||||
stageAnswerWrong = "ReadButAnswerWrong" // opened it and still answered wrong
|
||||
stageRecovered = "Recovered"
|
||||
stageContaminated = "Contaminated"
|
||||
stageNoEvidence = "NoEvidence"
|
||||
|
||||
// Routing is judged by model round, not by the linear order of tool events:
|
||||
// two calls in one round are one parallel fan-out, and calling that
|
||||
// "workspace first" would report a choice the model never made.
|
||||
routeMemoryFirst = "MemoryFirst"
|
||||
routeWorkspaceFirst = "WorkspaceFirst"
|
||||
routeParallel = "Parallel"
|
||||
routeNeither = "Neither"
|
||||
stageCueRead = "DirectRead" // read the target from an index address, no search needed
|
||||
stageNoRetrieval = "NoRetrieval" // answered without touching recall at all
|
||||
toolRecall = "recall"
|
||||
)
|
||||
|
||||
// searchAttempt is one query the model wrote, and what it got back. Kept per
|
||||
// attempt rather than aggregated: the question is why a retrieval that the host
|
||||
// probe ranks first takes the model more than one try.
|
||||
type searchAttempt struct {
|
||||
Ordinal int `json:"ordinal"`
|
||||
Round int `json:"round"`
|
||||
Query string `json:"query"`
|
||||
TargetRank int `json:"target_rank"` // 0 = the target did not come back
|
||||
ResultCount int `json:"result_count"`
|
||||
// Reformulation classifies this query against the one before it, and
|
||||
// MissingProbeTerms are the host probe's terms this query left out. Both
|
||||
// are derived from token sets: no judge, and nothing read out of prose.
|
||||
Reformulation string `json:"reformulation,omitempty"`
|
||||
MissingProbeTerms []string `json:"missing_probe_terms,omitempty"`
|
||||
}
|
||||
|
||||
// contextMetrics is one run's behaviour.
|
||||
type contextMetrics struct {
|
||||
Task string `json:"task"`
|
||||
Arm string `json:"arm"`
|
||||
|
||||
SearchCalls int `json:"search_calls"`
|
||||
// SearchHits counts searches that returned anything; TargetSearchHits
|
||||
// counts searches that returned the target. The gap is query quality.
|
||||
SearchHits int `json:"search_hits"`
|
||||
TargetSearchHits int `json:"target_search_hits"`
|
||||
FirstTargetRank int `json:"first_target_rank"` // 0 = never returned
|
||||
// FirstHitCoverage: how much of the answer the first hit handed over.
|
||||
// rank=1 with coverage 1/3 is a window too short to answer with, not a
|
||||
// model that would not stop.
|
||||
FirstHitCoverage int `json:"first_hit_coverage"`
|
||||
AnswerValues int `json:"answer_values"`
|
||||
|
||||
ReadCalls int `json:"read_calls"`
|
||||
TargetRead bool `json:"target_read"`
|
||||
ReadAfterHit bool `json:"read_after_hit"`
|
||||
// RecallReadWithoutSearch: the index addressed the target, so no search was
|
||||
// needed. Not "direct read" — read_file is a read too, and in an audit the
|
||||
// two mean opposite things.
|
||||
RecallReadWithoutSearch bool `json:"recall_read_without_search"`
|
||||
SearchThenRecallRead bool `json:"search_then_recall_read"`
|
||||
|
||||
FirstSearchRound int `json:"first_search_round"`
|
||||
FirstReadRound int `json:"first_read_round"`
|
||||
RetrievalRounds int `json:"retrieval_rounds"`
|
||||
|
||||
// RecallReturnedTokens is what recall paged back in. The index saves
|
||||
// resident prefix; this is what is paid dynamically instead.
|
||||
RecallReturnedTokens int `json:"recall_returned_tokens"`
|
||||
|
||||
// Provider usage as reported on the stream, or empty for a dry run. The count
|
||||
// is completions that reported usage, so it is not a count of attempts.
|
||||
UsageReportedRequests int `json:"usage_reported_requests,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
CacheHitTokens int `json:"cache_hit_tokens,omitempty"`
|
||||
CacheMissTokens int `json:"cache_miss_tokens,omitempty"`
|
||||
UsageEstimated bool `json:"usage_estimated,omitempty"`
|
||||
|
||||
Searches []searchAttempt `json:"searches,omitempty"`
|
||||
// PostHitSearches are queries issued after the target had already been
|
||||
// returned: not a query problem, a result-to-read problem.
|
||||
PostHitSearches int `json:"post_hit_searches"`
|
||||
|
||||
AnswerRecovered bool `json:"answer_recovered"`
|
||||
// AnswerAmbiguous marks an answer that offered more than one candidate for
|
||||
// a value. Markers matching proves the right fact appears; it does not
|
||||
// prove it was submitted as the answer. Recorded, never acted on.
|
||||
AnswerAmbiguous bool `json:"answer_ambiguous,omitempty"`
|
||||
AmbiguousVar string `json:"ambiguous_var,omitempty"`
|
||||
MissingMarkers []string `json:"missing_markers,omitempty"`
|
||||
// FinalAnswer is kept bounded so a surprising run can be read back. A
|
||||
// metric nobody can audit is a claim, not a measurement.
|
||||
FinalAnswer string `json:"final_answer,omitempty"`
|
||||
FailureStage string `json:"failure_stage"`
|
||||
TimeoutReason string `json:"timeout_reason,omitempty"`
|
||||
|
||||
// Failed marks a run that ended before a transcript existed: it has no score
|
||||
// and no place in the statistics. The row stays in results.jsonl with its
|
||||
// task, arm and stage, so a failure is counted rather than dropped.
|
||||
Failed bool `json:"failed,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
|
||||
// UnexpectedWorkTools are investigation tools used instead of recall.
|
||||
// Reaching for the workspace is behaviour, not noise: how a model routes
|
||||
// its uncertainty when memory is missing is part of what is measured.
|
||||
UnexpectedWorkTools []string `json:"unexpected_work_tools,omitempty"`
|
||||
EscapeCalls int `json:"escape_calls"`
|
||||
EscapeBeforeFirstRecall int `json:"escape_before_first_recall"`
|
||||
EscapeAfterFirstRecall int `json:"escape_after_first_recall"`
|
||||
// Contaminated: the answer was reachable through something other than
|
||||
// recall. Asserting this beats trusting the isolation, and a run that trips
|
||||
// it leaves the statistics rather than inflating one.
|
||||
Contaminated bool `json:"contaminated,omitempty"`
|
||||
LeakedVia string `json:"leaked_via,omitempty"`
|
||||
LeakedMarker string `json:"leaked_marker,omitempty"`
|
||||
LeakedArgs string `json:"leaked_args,omitempty"`
|
||||
LeakedOutput string `json:"leaked_output,omitempty"`
|
||||
// EscapeFoundAnswer separates "looked at the workspace" from "found the
|
||||
// answer there": different facts about the same call.
|
||||
EscapeFoundAnswer bool `json:"escape_found_answer,omitempty"`
|
||||
|
||||
CueVisible bool `json:"cue_visible"`
|
||||
|
||||
// trajectory is kept out of the metric line and written beside it, so a
|
||||
// future question can be asked of this batch instead of a new one.
|
||||
trajectory []provider.Message
|
||||
|
||||
// CueDirectRead: the cue was addressed, the first memory action was a
|
||||
// positions read, and it covered the target. Crediting any positions call
|
||||
// would hand the index a search's work.
|
||||
CueDirectRead bool `json:"cue_direct_read"`
|
||||
|
||||
// Sufficiency, not hit. A search that returns the target at rank 1 has not
|
||||
// necessarily returned the answer: the snippet is bounded. The stopping
|
||||
// boundary is the round where everything needed was readable.
|
||||
FirstSufficientRound int `json:"first_sufficient_round"`
|
||||
SufficientVia string `json:"sufficient_via"` // snippet | read | never
|
||||
RoundsAfterSufficient int `json:"rounds_after_sufficient"`
|
||||
SearchesAfterSufficient int `json:"searches_after_sufficient"`
|
||||
ReadsAfterSufficient int `json:"reads_after_sufficient"`
|
||||
EscapesAfterSufficient int `json:"escapes_after_sufficient"`
|
||||
RecallTokensAfterSufficient int `json:"recall_tokens_after_sufficient"`
|
||||
StoppingClass string `json:"stopping_class"`
|
||||
|
||||
// What happened after the answer was already in hand. A timeout that spent
|
||||
// seven rounds past its target is a stopping problem, and extending the
|
||||
// budget would only buy the loop more time.
|
||||
TargetFirstSeenRound int `json:"target_first_seen_round"`
|
||||
RoundsAfterTarget int `json:"rounds_after_target"`
|
||||
SearchesAfterTarget int `json:"searches_after_target"`
|
||||
RecallTokensAfterTarget int `json:"recall_tokens_after_target"`
|
||||
TargetHeldAtEnd bool `json:"target_held_at_end"`
|
||||
|
||||
// Routing is where the model looked first when its memory was missing.
|
||||
Routing string `json:"routing"`
|
||||
FirstRecallRound int `json:"first_recall_round"`
|
||||
FirstEscapeRound int `json:"first_escape_round"`
|
||||
}
|
||||
|
||||
// hitPosition matches the address recall prints for a search hit. Reading our
|
||||
// own rendered format, not guessing at the model's words.
|
||||
var hitPosition = regexp.MustCompile(`(?m)^#(\d+) `)
|
||||
|
||||
// scoreRun reads the messages one turn appended and reports what happened.
|
||||
// runScorer walks one turn's messages, holding the state a judgement needs:
|
||||
// which calls are open, whether the target has come back, and which search
|
||||
// each result belongs to.
|
||||
type runScorer struct {
|
||||
m contextMetrics
|
||||
inst fixtureInstance
|
||||
target int
|
||||
calls map[string]provider.ToolCall
|
||||
pending map[string]int
|
||||
round int
|
||||
sawTargetHit bool
|
||||
// seen accumulates everything recall has shown the model, so sufficiency is
|
||||
// judged on what it could read rather than on which call produced it.
|
||||
seen strings.Builder
|
||||
sufficient bool
|
||||
}
|
||||
|
||||
// scoreRun reads the messages one turn appended and reports what happened.
|
||||
func scoreRun(msgs []provider.Message, inst fixtureInstance, target int, arm string, cueVisible bool) contextMetrics {
|
||||
s := &runScorer{
|
||||
m: contextMetrics{Task: inst.Task.ID, Arm: arm, CueVisible: cueVisible},
|
||||
inst: inst, target: target,
|
||||
calls: map[string]provider.ToolCall{}, pending: map[string]int{},
|
||||
}
|
||||
for _, msg := range msgs {
|
||||
if len(msg.ToolCalls) > 0 {
|
||||
s.round++
|
||||
}
|
||||
for _, tc := range msg.ToolCalls {
|
||||
s.calls[tc.ID] = tc
|
||||
s.observeCall(tc)
|
||||
}
|
||||
if msg.Role == provider.RoleTool {
|
||||
s.observeResult(msg)
|
||||
}
|
||||
}
|
||||
return s.finish(cueVisible)
|
||||
}
|
||||
|
||||
// observeCall records one tool call: a recall search, a recall read, or a
|
||||
// reach for the workspace.
|
||||
func (s *runScorer) observeCall(tc provider.ToolCall) {
|
||||
m := &s.m
|
||||
if tc.Name != toolRecall {
|
||||
if !containsString(m.UnexpectedWorkTools, tc.Name) {
|
||||
m.UnexpectedWorkTools = append(m.UnexpectedWorkTools, tc.Name)
|
||||
}
|
||||
m.EscapeCalls++
|
||||
if s.sufficient {
|
||||
m.EscapesAfterSufficient++
|
||||
}
|
||||
if m.FirstEscapeRound == 0 {
|
||||
m.FirstEscapeRound = s.round
|
||||
}
|
||||
if m.SearchCalls+m.ReadCalls == 0 {
|
||||
m.EscapeBeforeFirstRecall++
|
||||
} else {
|
||||
m.EscapeAfterFirstRecall++
|
||||
}
|
||||
return
|
||||
}
|
||||
if m.FirstRecallRound == 0 {
|
||||
m.FirstRecallRound = s.round
|
||||
}
|
||||
query, positions := recallArgs(tc.Arguments)
|
||||
switch {
|
||||
case query != "":
|
||||
m.SearchCalls++
|
||||
if m.FirstSearchRound == 0 {
|
||||
m.FirstSearchRound = s.round
|
||||
}
|
||||
if s.sawTargetHit {
|
||||
m.PostHitSearches++
|
||||
m.SearchesAfterTarget++
|
||||
}
|
||||
if s.sufficient {
|
||||
m.SearchesAfterSufficient++
|
||||
}
|
||||
m.Searches = append(m.Searches, searchAttempt{Ordinal: m.SearchCalls, Round: s.round, Query: query})
|
||||
s.pending[tc.ID] = len(m.Searches) - 1
|
||||
case len(positions) > 0:
|
||||
m.ReadCalls++
|
||||
if s.sufficient {
|
||||
m.ReadsAfterSufficient++
|
||||
}
|
||||
if m.FirstReadRound == 0 {
|
||||
m.FirstReadRound = s.round
|
||||
}
|
||||
// A read covers the target when it names its position: recall returns
|
||||
// the tool results under the caller's own address.
|
||||
if containsInt(positions, s.target) {
|
||||
m.TargetRead = true
|
||||
switch {
|
||||
case s.sawTargetHit:
|
||||
m.ReadAfterHit, m.SearchThenRecallRead = true, true
|
||||
case m.SearchCalls == 0:
|
||||
m.RecallReadWithoutSearch = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// observeResult reads what a call returned: contamination, page-in cost, and
|
||||
// whether this search brought the target back.
|
||||
func (s *runScorer) observeResult(msg provider.Message) {
|
||||
m := &s.m
|
||||
call, ok := s.calls[msg.ToolCallID]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if call.Name != toolRecall {
|
||||
// Codenames only: a three-digit epoch appears in any long file by
|
||||
// chance, and one did. A model holding the codename holds the numbers
|
||||
// beside it anyway.
|
||||
for _, marker := range s.inst.AnswerMarkers {
|
||||
if len(marker) >= 6 && strings.Contains(msg.Content, marker) {
|
||||
m.Contaminated, m.LeakedVia, m.LeakedMarker = true, call.Name, marker
|
||||
m.EscapeFoundAnswer = true
|
||||
m.LeakedArgs, m.LeakedOutput = truncate(call.Arguments, 200), truncate(msg.Content, 300)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
cost := tokencount.Text(msg.Content)
|
||||
m.RecallReturnedTokens += cost
|
||||
if s.sawTargetHit {
|
||||
m.RecallTokensAfterTarget += cost
|
||||
}
|
||||
if s.sufficient {
|
||||
m.RecallTokensAfterSufficient += cost
|
||||
}
|
||||
// Everything recall showed the model, judged as a whole: the answer may
|
||||
// arrive across a snippet and a read, and either way it is readable.
|
||||
s.seen.WriteString(msg.Content)
|
||||
s.seen.WriteString("\n")
|
||||
if !s.sufficient && allMarkersIn(s.seen.String(), s.inst.AnswerMarkers) {
|
||||
s.sufficient = true
|
||||
m.FirstSufficientRound = s.round
|
||||
query, _ := recallArgs(call.Arguments)
|
||||
m.SufficientVia = "read"
|
||||
if query != "" {
|
||||
m.SufficientVia = "snippet"
|
||||
}
|
||||
}
|
||||
if query, _ := recallArgs(call.Arguments); query == "" {
|
||||
return
|
||||
}
|
||||
positions := searchHitPositions(msg.Content)
|
||||
if len(positions) > 0 {
|
||||
m.SearchHits++
|
||||
}
|
||||
rank := indexOfInt(positions, s.target)
|
||||
if i, ok := s.pending[msg.ToolCallID]; ok {
|
||||
m.Searches[i].TargetRank, m.Searches[i].ResultCount = rank, len(positions)
|
||||
}
|
||||
if rank == 0 {
|
||||
return
|
||||
}
|
||||
m.TargetSearchHits++
|
||||
if !s.sawTargetHit {
|
||||
m.TargetFirstSeenRound = s.round
|
||||
m.FirstHitCoverage = answerCoverage(msg.Content, s.inst.AnswerMarkers)
|
||||
}
|
||||
s.sawTargetHit = true
|
||||
if m.FirstTargetRank == 0 {
|
||||
m.FirstTargetRank = rank
|
||||
}
|
||||
}
|
||||
|
||||
// finish derives what only the whole walk can say.
|
||||
func (s *runScorer) finish(cueVisible bool) contextMetrics {
|
||||
m := s.m
|
||||
m.RetrievalRounds = m.SearchCalls + m.ReadCalls
|
||||
m.TargetHeldAtEnd = s.sawTargetHit || m.TargetRead
|
||||
if m.TargetFirstSeenRound > 0 {
|
||||
m.RoundsAfterTarget = s.round - m.TargetFirstSeenRound
|
||||
}
|
||||
m.Routing = routing(m.FirstRecallRound, m.FirstEscapeRound)
|
||||
m.CueDirectRead = cueVisible && m.RecallReadWithoutSearch
|
||||
if m.SufficientVia == "" {
|
||||
m.SufficientVia = "never"
|
||||
}
|
||||
if m.FirstSufficientRound > 0 {
|
||||
m.RoundsAfterSufficient = s.round - m.FirstSufficientRound
|
||||
}
|
||||
m.AnswerValues = len(s.inst.AnswerMarkers)
|
||||
m.StoppingClass = stoppingClass(m)
|
||||
classifySearches(&m, s.inst)
|
||||
return m
|
||||
}
|
||||
|
||||
// scoreAnswer grades the final text on the task's markers. Nonces, matched
|
||||
// exactly: no judge model, and nothing a general answer could satisfy.
|
||||
func (m *contextMetrics) scoreAnswer(final string, inst fixtureInstance) {
|
||||
lower := strings.ToLower(final)
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if !strings.Contains(lower, strings.ToLower(marker)) {
|
||||
m.MissingMarkers = append(m.MissingMarkers, marker)
|
||||
}
|
||||
}
|
||||
m.AnswerRecovered = len(m.MissingMarkers) == 0
|
||||
m.AnswerAmbiguous, m.AmbiguousVar = ambiguousAnswer(final, inst)
|
||||
m.FailureStage = m.stage()
|
||||
if len(final) > 1200 {
|
||||
final = final[:1200] + "…"
|
||||
}
|
||||
m.FinalAnswer = final
|
||||
}
|
||||
|
||||
// stage places the run in the funnel. One label, chosen at the first step that
|
||||
// did not happen, so a table of stages reads as where the retrieval broke.
|
||||
func (m contextMetrics) stage() string {
|
||||
switch {
|
||||
case m.Contaminated:
|
||||
return stageContaminated
|
||||
case m.AnswerRecovered && m.RecallReadWithoutSearch:
|
||||
return stageCueRead
|
||||
case m.AnswerRecovered:
|
||||
return stageRecovered
|
||||
case m.TargetRead:
|
||||
return stageAnswerWrong
|
||||
case m.TargetSearchHits > 0:
|
||||
return stageHitNotRead
|
||||
case m.SearchCalls > 0:
|
||||
return stageSearchMiss
|
||||
case m.ReadCalls > 0:
|
||||
return stageReadMissed
|
||||
default:
|
||||
return stageNoRetrieval
|
||||
}
|
||||
}
|
||||
|
||||
// recallArgs pulls the two fields that say which operation a recall call is.
|
||||
func recallArgs(args string) (query string, positions []int) {
|
||||
var parsed struct {
|
||||
Query string `json:"query"`
|
||||
Positions []int `json:"positions"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(args), &parsed); err != nil {
|
||||
return "", nil
|
||||
}
|
||||
return strings.TrimSpace(parsed.Query), parsed.Positions
|
||||
}
|
||||
|
||||
// searchHitPositions reads the addresses out of a search result, in rank order.
|
||||
func searchHitPositions(text string) []int {
|
||||
var out []int
|
||||
for _, match := range hitPosition.FindAllStringSubmatch(text, -1) {
|
||||
if n, err := strconv.Atoi(match[1]); err == nil {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func containsInt(list []int, want int) bool { return slices.Contains(list, want) }
|
||||
|
||||
// indexOfInt returns a 1-based rank, or 0 when absent.
|
||||
func indexOfInt(list []int, want int) int {
|
||||
for i, v := range list {
|
||||
if v == want {
|
||||
return i + 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func containsString(list []string, want string) bool { return slices.Contains(list, want) }
|
||||
|
||||
// funnel summarizes one arm's runs the way the experiment is read.
|
||||
type funnel struct {
|
||||
Arm string
|
||||
Runs int
|
||||
Scored, Contaminated, Errored int
|
||||
Searched, TargetHit, TargetRead, Answered int
|
||||
CueReads int
|
||||
SearchCalls, ReadCalls, RecallTokens int
|
||||
Escapes, EscapeCalls int
|
||||
Stages, Routes map[string]int
|
||||
}
|
||||
|
||||
func summarize(arm string, runs []contextMetrics) funnel {
|
||||
f := funnel{Arm: arm, Runs: len(runs), Stages: map[string]int{}, Routes: map[string]int{}}
|
||||
for _, r := range runs {
|
||||
// A run that never produced a transcript is not a scored run and has no
|
||||
// stage to place: counting it would put an empty name in the funnel, the
|
||||
// stage table and the routes.
|
||||
if r.Failed {
|
||||
continue
|
||||
}
|
||||
if r.Contaminated {
|
||||
f.Contaminated++
|
||||
f.Stages[r.FailureStage]++
|
||||
continue
|
||||
}
|
||||
if r.FailureStage == "RunError" {
|
||||
f.Errored++
|
||||
f.Stages[r.FailureStage]++
|
||||
continue
|
||||
}
|
||||
f.Scored++
|
||||
if r.SearchCalls > 0 {
|
||||
f.Searched++
|
||||
}
|
||||
if r.TargetSearchHits > 0 {
|
||||
f.TargetHit++
|
||||
}
|
||||
if r.TargetRead {
|
||||
f.TargetRead++
|
||||
}
|
||||
if r.AnswerRecovered {
|
||||
f.Answered++
|
||||
}
|
||||
if r.RecallReadWithoutSearch {
|
||||
f.CueReads++
|
||||
}
|
||||
if len(r.UnexpectedWorkTools) > 0 {
|
||||
f.Escapes++
|
||||
}
|
||||
f.EscapeCalls += r.EscapeCalls
|
||||
f.SearchCalls += r.SearchCalls
|
||||
f.ReadCalls += r.ReadCalls
|
||||
f.RecallTokens += r.RecallReturnedTokens
|
||||
f.Stages[r.FailureStage]++
|
||||
if r.FailureStage != "RunError" {
|
||||
f.Routes[r.Routing]++
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func (f funnel) line() string {
|
||||
per := func(n int) float64 {
|
||||
if f.Scored == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(n) / float64(f.Scored)
|
||||
}
|
||||
line := fmt.Sprintf("%-14s runs=%-3d answered=%d/%d searched=%d target-hit=%d cue-read=%d search/task=%.2f recall-tok/task=%.0f escape/task=%.2f",
|
||||
f.Arm, f.Scored, f.Answered, f.Scored, f.Searched, f.TargetHit, f.CueReads,
|
||||
per(f.SearchCalls), per(f.RecallTokens), per(f.EscapeCalls))
|
||||
if f.Contaminated > 0 {
|
||||
line += fmt.Sprintf(" CONTAMINATED=%d", f.Contaminated)
|
||||
}
|
||||
if f.Errored > 0 {
|
||||
line += fmt.Sprintf(" ERRORED=%d", f.Errored)
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func writeJSONLine(w io.Writer, v any) error {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = fmt.Fprintln(w, string(b))
|
||||
return err
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "…"
|
||||
}
|
||||
|
||||
// routing says where the model looked first, comparing rounds rather than
|
||||
// event order. Same round is a parallel fan-out, not a preference.
|
||||
func routing(recallRound, escapeRound int) string {
|
||||
switch {
|
||||
case recallRound == 0 && escapeRound == 0:
|
||||
return routeNeither
|
||||
case recallRound == 0:
|
||||
return routeWorkspaceFirst
|
||||
case escapeRound == 0:
|
||||
return routeMemoryFirst
|
||||
case recallRound < escapeRound:
|
||||
return routeMemoryFirst
|
||||
case escapeRound < recallRound:
|
||||
return routeWorkspaceFirst
|
||||
default:
|
||||
return routeParallel
|
||||
}
|
||||
}
|
||||
|
||||
// ambiguousAnswer reports an answer naming more than one candidate of a
|
||||
// codename's shape. Only codenames are checked: they carry a stem this run
|
||||
// declared, so two distinct "fence-…" values in one answer is one answer too
|
||||
// many. Numbers have no such shape and are left alone.
|
||||
func ambiguousAnswer(final string, inst fixtureInstance) (bool, string) {
|
||||
for _, spec := range inst.Task.Vars {
|
||||
if spec.Kind != varCodename {
|
||||
continue
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, match := range regexp.MustCompile(regexp.QuoteMeta(spec.Word)+`-[a-z0-9]{3,}`).FindAllString(final, -1) {
|
||||
seen[match] = true
|
||||
}
|
||||
if len(seen) > 1 {
|
||||
return true, spec.Name
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
func allMarkersIn(text string, markers []string) bool {
|
||||
if len(markers) == 0 {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(text)
|
||||
for _, marker := range markers {
|
||||
if !strings.Contains(lower, strings.ToLower(marker)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Stopping classes. The boundary is sufficiency, not the target coming back:
|
||||
// a rank-1 hit whose snippet was too short to answer leaves a read entirely
|
||||
// justified, and counting that as waste would blame the model for the bound.
|
||||
const (
|
||||
stopSnippet = "SnippetStop" // enough arrived, and the turn ended
|
||||
stopSnippetRead = "SnippetThenRead" // the snippet was short; the read finished it
|
||||
stopDefensiveRead = "DefensiveRead" // enough had arrived, one read confirmed it
|
||||
stopPostRetrieval = "PostSufficientRetrieval" // enough had arrived, and retrieval continued
|
||||
stopRunaway = "PostSufficientRunaway" // it continued for rounds, or ran out of budget
|
||||
stopNeverEnough = "NeverSufficient"
|
||||
)
|
||||
|
||||
// runawayRounds is where continuing stops looking like confirmation.
|
||||
const runawayRounds = 3
|
||||
|
||||
func stoppingClass(m contextMetrics) string {
|
||||
if m.FirstSufficientRound == 0 {
|
||||
return stopNeverEnough
|
||||
}
|
||||
after := m.SearchesAfterSufficient + m.ReadsAfterSufficient + m.EscapesAfterSufficient
|
||||
switch {
|
||||
case after == 0 && m.SufficientVia == "snippet":
|
||||
return stopSnippet
|
||||
case after == 0:
|
||||
return stopSnippetRead
|
||||
case m.RoundsAfterSufficient >= runawayRounds || m.FailureStage == "RunError":
|
||||
return stopRunaway
|
||||
case after == 1 && m.ReadsAfterSufficient == 1:
|
||||
return stopDefensiveRead
|
||||
default:
|
||||
return stopPostRetrieval
|
||||
}
|
||||
}
|
||||
|
||||
// reportStopping is the funnel that matters once retrieval succeeds: what the
|
||||
// turn did after it already had everything it needed.
|
||||
func reportStopping(all []contextMetrics) string {
|
||||
classes := map[string]int{}
|
||||
var extraRounds, extraTokens []float64
|
||||
sufficient, answeredNext := 0, 0
|
||||
for _, m := range all {
|
||||
if m.Failed || m.Contaminated {
|
||||
continue
|
||||
}
|
||||
classes[m.StoppingClass]++
|
||||
if m.FirstSufficientRound == 0 {
|
||||
continue
|
||||
}
|
||||
sufficient++
|
||||
extraRounds = append(extraRounds, float64(m.RoundsAfterSufficient))
|
||||
extraTokens = append(extraTokens, float64(m.RecallTokensAfterSufficient))
|
||||
if m.RoundsAfterSufficient <= 1 {
|
||||
answeredNext++
|
||||
}
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## After the evidence was already enough\n")
|
||||
fmt.Fprintf(&b, " sufficient in %d runs; ended within one round in %d\n", sufficient, answeredNext)
|
||||
fmt.Fprintf(&b, " extra rounds mean %.1f median %.1f p90 %.0f max %.0f\n",
|
||||
mean(extraRounds), median(extraRounds), percentile(extraRounds, 0.9), maxOf(extraRounds))
|
||||
fmt.Fprintf(&b, " extra recall tokens mean %.0f median %.0f p90 %.0f max %.0f\n",
|
||||
mean(extraTokens), median(extraTokens), percentile(extraTokens, 0.9), maxOf(extraTokens))
|
||||
b.WriteString(" " + countsLine(classes) + "\n")
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// The scorer decides what the pilot concludes, so it is held to the same rule
|
||||
// the corpus is: judgements read structure. A model that says it will search
|
||||
// has not searched; a recall call carrying a query has.
|
||||
|
||||
const scorerTarget = 42
|
||||
|
||||
func recallCall(id, args string) provider.Message {
|
||||
return provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: id, Name: "recall", Arguments: args},
|
||||
}}
|
||||
}
|
||||
|
||||
func toolResult(id, text string) provider.Message {
|
||||
return provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "recall", Content: text}
|
||||
}
|
||||
|
||||
func hitBlock(positions ...int) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("Folded context matching \"x\":\n")
|
||||
for _, p := range positions {
|
||||
fmt.Fprintf(&b, "\n#%d assistant_text\nsome snippet here\n", p)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func scorerTask() fixtureInstance {
|
||||
return fixtureInstance{Task: contextTask{ID: "t"}, AnswerMarkers: []string{"cobalt-lark-17"}}
|
||||
}
|
||||
|
||||
func score(t *testing.T, msgs []provider.Message, final string) contextMetrics {
|
||||
t.Helper()
|
||||
m := scoreRun(msgs, scorerTask(), scorerTarget, "arm", false)
|
||||
m.scoreAnswer(final, scorerTask())
|
||||
return m
|
||||
}
|
||||
|
||||
// The whole funnel, walked end to end.
|
||||
func TestScorerReadsTheFullRetrievalChain(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(7, scorerTarget)),
|
||||
recallCall("c2", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c2", "#42\n[assistant]\nthe ownership token is cobalt-lark-17"),
|
||||
{Role: provider.RoleAssistant, Content: "The token was cobalt-lark-17."},
|
||||
}, "The token was cobalt-lark-17.")
|
||||
|
||||
if m.SearchCalls != 1 || m.ReadCalls != 1 {
|
||||
t.Errorf("calls = %d search / %d read, want 1/1", m.SearchCalls, m.ReadCalls)
|
||||
}
|
||||
if m.TargetSearchHits != 1 || m.FirstTargetRank != 2 {
|
||||
t.Errorf("target hit %d at rank %d, want 1 at rank 2", m.TargetSearchHits, m.FirstTargetRank)
|
||||
}
|
||||
if !m.TargetRead || !m.ReadAfterHit || m.RecallReadWithoutSearch {
|
||||
t.Errorf("read flags = read %v afterHit %v direct %v", m.TargetRead, m.ReadAfterHit, m.RecallReadWithoutSearch)
|
||||
}
|
||||
if !m.AnswerRecovered || m.FailureStage != stageRecovered {
|
||||
t.Errorf("stage = %q recovered=%v, want Recovered", m.FailureStage, m.AnswerRecovered)
|
||||
}
|
||||
if m.RecallReturnedTokens == 0 {
|
||||
t.Error("recall returned tokens were not counted")
|
||||
}
|
||||
}
|
||||
|
||||
// An index arm's claim: the address was already on screen, so no search was
|
||||
// needed. That is a different outcome from finding it, and gets its own stage.
|
||||
func TestScorerSeparatesCueReadFromSearch(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c1", "#42\nthe ownership token is cobalt-lark-17"),
|
||||
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
|
||||
}, "cobalt-lark-17")
|
||||
|
||||
if m.SearchCalls != 0 || !m.RecallReadWithoutSearch {
|
||||
t.Errorf("search=%d direct=%v, want 0 and true", m.SearchCalls, m.RecallReadWithoutSearch)
|
||||
}
|
||||
if m.FailureStage != stageCueRead {
|
||||
t.Errorf("stage = %q, want %q", m.FailureStage, stageCueRead)
|
||||
}
|
||||
}
|
||||
|
||||
// Every way the chain can break gets its own label, so a failed run says where
|
||||
// rather than only that.
|
||||
func TestScorerNamesWhereTheChainBroke(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
msgs []provider.Message
|
||||
final string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "never touched recall",
|
||||
msgs: []provider.Message{{Role: provider.RoleAssistant, Content: "I do not recall that."}},
|
||||
final: "I do not recall that.",
|
||||
want: stageNoRetrieval,
|
||||
},
|
||||
{
|
||||
name: "searched and missed",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", `{"query":"unrelated words"}`),
|
||||
toolResult("c1", hitBlock(3, 9)),
|
||||
},
|
||||
final: "not found",
|
||||
want: stageSearchMiss,
|
||||
},
|
||||
{
|
||||
name: "found it and never opened it",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
},
|
||||
final: "probably something about retries",
|
||||
want: stageHitNotRead,
|
||||
},
|
||||
{
|
||||
name: "read it and answered wrong",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c1", "#42\nthe token is elsewhere"),
|
||||
},
|
||||
final: "I think it was amber-9",
|
||||
want: stageAnswerWrong,
|
||||
},
|
||||
{
|
||||
name: "read the wrong position",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", `{"positions":[7]}`),
|
||||
toolResult("c1", "#7\nnothing useful"),
|
||||
},
|
||||
final: "unclear",
|
||||
want: stageReadMissed,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := score(t, tc.msgs, tc.final).FailureStage; got != tc.want {
|
||||
t.Errorf("stage = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The answer lives only in folded history, so reaching for the workspace is a
|
||||
// strategy change worth seeing — including when it happens to work.
|
||||
func TestScorerRecordsWorkspaceEscapes(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"rg cobalt-lark"}`},
|
||||
{ID: "g2", Name: "read_file", Arguments: `{"path":"notes.md"}`},
|
||||
}},
|
||||
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
|
||||
}, "cobalt-lark-17")
|
||||
|
||||
if len(m.UnexpectedWorkTools) != 2 {
|
||||
t.Fatalf("escapes = %v, want bash and read_file", m.UnexpectedWorkTools)
|
||||
}
|
||||
if m.SearchCalls != 0 || m.ReadCalls != 0 {
|
||||
t.Errorf("a workspace tool was counted as recall: search=%d read=%d", m.SearchCalls, m.ReadCalls)
|
||||
}
|
||||
// It answered, so the run is Recovered — the escape is reported beside the
|
||||
// outcome rather than overriding it.
|
||||
if !m.AnswerRecovered {
|
||||
t.Error("a correct answer was not scored as recovered")
|
||||
}
|
||||
}
|
||||
|
||||
// Narration is not action.
|
||||
func TestScorerIgnoresWhatTheModelSaysItWillDo(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, Content: "Let me search the folded context with recall for the retry boundary."},
|
||||
}, "Let me search the folded context with recall for the retry boundary.")
|
||||
if m.SearchCalls != 0 || m.FailureStage != stageNoRetrieval {
|
||||
t.Errorf("prose was scored as a search: calls=%d stage=%q", m.SearchCalls, m.FailureStage)
|
||||
}
|
||||
}
|
||||
|
||||
// Every marker must land, so a partially right answer is not a recovery.
|
||||
func TestScorerRequiresEveryMarker(t *testing.T) {
|
||||
task := fixtureInstance{Task: contextTask{ID: "t"}, AnswerMarkers: []string{"comet-42", "per-lineage"}}
|
||||
m := scoreRun(nil, task, scorerTarget, "arm", false)
|
||||
m.scoreAnswer("the salt was comet-42", task)
|
||||
if m.AnswerRecovered {
|
||||
t.Error("a half answer scored as recovered")
|
||||
}
|
||||
if len(m.MissingMarkers) != 1 || m.MissingMarkers[0] != "per-lineage" {
|
||||
t.Errorf("missing = %v, want per-lineage", m.MissingMarkers)
|
||||
}
|
||||
}
|
||||
|
||||
// An answer reachable through anything but recall invalidates the run. The
|
||||
// workspace is isolated so this should never fire; asserting it beats trusting
|
||||
// it, and a tripped run leaves the statistics rather than inflating one.
|
||||
func TestScorerInvalidatesRunsThatLeakedThroughAnotherTool(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"rg -r ."}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "corpus.go: the token is cobalt-lark-17"},
|
||||
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
|
||||
}, "cobalt-lark-17")
|
||||
|
||||
if !m.Contaminated || m.LeakedVia != "bash" || m.LeakedMarker != "cobalt-lark-17" {
|
||||
t.Fatalf("leak not caught: contaminated=%v via=%q marker=%q", m.Contaminated, m.LeakedVia, m.LeakedMarker)
|
||||
}
|
||||
if m.FailureStage != stageContaminated {
|
||||
t.Errorf("stage = %q, want %q even though every marker matched", m.FailureStage, stageContaminated)
|
||||
}
|
||||
f := summarize("arm", []contextMetrics{m})
|
||||
if f.Scored != 0 || f.Answered != 0 || f.Contaminated != 1 {
|
||||
t.Errorf("a contaminated run entered the statistics: %+v", f)
|
||||
}
|
||||
}
|
||||
|
||||
// Where an escape happened matters: before any recall is a different strategy
|
||||
// from after one came back short.
|
||||
func TestScorerPlacesEscapesRelativeToRecall(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g2", Name: "read_file", Arguments: `{"path":"README.md"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g2", Name: "read_file", Content: "nothing here"},
|
||||
}, "unclear")
|
||||
|
||||
if m.EscapeCalls != 2 || m.EscapeBeforeFirstRecall != 1 || m.EscapeAfterFirstRecall != 1 {
|
||||
t.Errorf("escapes = %d (%d before, %d after), want 2 (1, 1)",
|
||||
m.EscapeCalls, m.EscapeBeforeFirstRecall, m.EscapeAfterFirstRecall)
|
||||
}
|
||||
}
|
||||
|
||||
// Routing is judged by round. Two calls in one round are a fan-out, and
|
||||
// reporting that as a preference would invent a decision the model never made.
|
||||
func TestRoutingIsJudgedByRoundNotEventOrder(t *testing.T) {
|
||||
parallel := []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
{ID: "c1", Name: "recall", Arguments: `{"query":"retry boundary"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
}
|
||||
if got := score(t, parallel, "unclear").Routing; got != routeParallel {
|
||||
t.Errorf("routing = %q for one round holding both, want %q", got, routeParallel)
|
||||
}
|
||||
|
||||
workspaceFirst := []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
}
|
||||
if got := score(t, workspaceFirst, "unclear").Routing; got != routeWorkspaceFirst {
|
||||
t.Errorf("routing = %q for workspace in an earlier round, want %q", got, routeWorkspaceFirst)
|
||||
}
|
||||
|
||||
memoryFirst := []provider.Message{
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
}
|
||||
if got := score(t, memoryFirst, "unclear").Routing; got != routeMemoryFirst {
|
||||
t.Errorf("routing = %q for recall in an earlier round, want %q", got, routeMemoryFirst)
|
||||
}
|
||||
|
||||
if got := score(t, nil, "no idea").Routing; got != routeNeither {
|
||||
t.Errorf("routing = %q with no tools at all, want %q", got, routeNeither)
|
||||
}
|
||||
}
|
||||
|
||||
// Markers matching proves the right fact appears somewhere. An answer that
|
||||
// hedges between two candidates contains it and has not submitted it.
|
||||
func TestScorerFlagsAnAnswerThatNamesTwoCandidates(t *testing.T) {
|
||||
inst := fixtureInstance{
|
||||
Task: contextTask{ID: "t", Vars: []varSpec{code("marker", "fence")}},
|
||||
AnswerMarkers: []string{"fence-qmwnv"},
|
||||
}
|
||||
m := scoreRun(nil, inst, 0, "arm", false)
|
||||
m.scoreAnswer("It was fence-qmwnv, though it may have been fence-abcde.", inst)
|
||||
if !m.AnswerRecovered {
|
||||
t.Error("the marker is present; recovery should still read true")
|
||||
}
|
||||
if !m.AnswerAmbiguous || m.AmbiguousVar != "marker" {
|
||||
t.Errorf("ambiguity not flagged: %v %q", m.AnswerAmbiguous, m.AmbiguousVar)
|
||||
}
|
||||
|
||||
clean := scoreRun(nil, inst, 0, "arm", false)
|
||||
clean.scoreAnswer("The marker was fence-qmwnv.", inst)
|
||||
if clean.AnswerAmbiguous {
|
||||
t.Error("a single candidate was called ambiguous")
|
||||
}
|
||||
}
|
||||
|
||||
// The index arm's claim is narrow: the cue was addressed, the first memory
|
||||
// action was a positions read, and it covered the target. A positions call
|
||||
// after a search located the target credits the index for the search's work.
|
||||
func TestCueDirectReadRequiresTheCueAndTheFirstAction(t *testing.T) {
|
||||
direct := []provider.Message{
|
||||
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c1", "#42\nthe token is cobalt-lark-17"),
|
||||
}
|
||||
m := scoreRun(direct, scorerTask(), scorerTarget, "arm", true)
|
||||
if !m.CueDirectRead {
|
||||
t.Error("a first-action read with the cue visible was not credited")
|
||||
}
|
||||
|
||||
// Same read, but no cue was on screen to give the address.
|
||||
if scoreRun(direct, scorerTask(), scorerTarget, "arm", false).CueDirectRead {
|
||||
t.Error("a read was credited to a cue that was not visible")
|
||||
}
|
||||
|
||||
// Same read, but a search found the address first.
|
||||
afterSearch := append([]provider.Message{
|
||||
recallCall("c0", `{"query":"retry boundary"}`),
|
||||
toolResult("c0", hitBlock(scorerTarget)),
|
||||
}, direct...)
|
||||
if scoreRun(afterSearch, scorerTask(), scorerTarget, "arm", true).CueDirectRead {
|
||||
t.Error("a read after a search was credited to the index")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,469 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// Synthetic history on purpose: an answer that exists in the workspace is
|
||||
// solvable by reading the workspace, which measures grep. Every answer is a
|
||||
// placeholder here and a value only inside one run.
|
||||
|
||||
// experiment names which question a task belongs to.
|
||||
const (
|
||||
experimentSearch = "search" // does search reach what no index addresses?
|
||||
experimentIndex = "index" // what is an ambient address line worth?
|
||||
)
|
||||
|
||||
// cueTier is the narrowest fold-index scale at which an index task's cue is
|
||||
// still addressed. Below it the model has to find the call without a hint.
|
||||
const (
|
||||
tierQuarter = "quarter" // visible at default, half, quarter
|
||||
tierHalf = "half" // visible at default, half
|
||||
tierDefault = "default" // visible at default only
|
||||
)
|
||||
|
||||
// role places an index task in the efficiency substrate or the stopping corpus.
|
||||
const (
|
||||
roleEfficiency = "efficiency"
|
||||
roleStopping = "stopping"
|
||||
)
|
||||
|
||||
// plantKind is how a task's target enters the transcript.
|
||||
const (
|
||||
plantAssistant = "assistant" // the model's own account: never index-addressed
|
||||
plantCall = "call" // a tool call and its result: always index-addressed
|
||||
)
|
||||
|
||||
// contextTask is one recall question as a template. Nothing here is an answer.
|
||||
type contextTask struct {
|
||||
ID string
|
||||
Experiment string
|
||||
TargetKind string
|
||||
Prompt string
|
||||
ProbeQuery string
|
||||
// Vars declare the placeholders. AnswerVars name the ones a correct answer
|
||||
// must contain, CueVar the one an index line carries.
|
||||
Vars []varSpec
|
||||
AnswerVars []string
|
||||
CueVar string
|
||||
CueTier string
|
||||
// Role is roleEfficiency or roleStopping; it decides the batch a task enters.
|
||||
Role string
|
||||
// PlantAfterGen ages the target: mergeFoldIndex trims from its oldest end,
|
||||
// so what follows a cue decides which budget addresses it. Frozen, because
|
||||
// a run that searched for it would let the tested code move its goalposts.
|
||||
PlantAfterGen int
|
||||
// Plant is how the target enters, and the rest is what it says.
|
||||
Plant string
|
||||
CallTool string
|
||||
CallArgs string
|
||||
CallID string
|
||||
PlantBody string
|
||||
}
|
||||
|
||||
// fixtureInstance is one task with values. Scoring binds to this, never to the
|
||||
// corpus: the corpus has no answers to bind to.
|
||||
type fixtureInstance struct {
|
||||
Task contextTask
|
||||
Vars fixtureVars
|
||||
Prompt string
|
||||
ProbeQuery string
|
||||
AnswerMarkers []string
|
||||
CueMarker string
|
||||
body string
|
||||
callArgs string
|
||||
}
|
||||
|
||||
// instantiateTask resolves one task against a generator.
|
||||
func instantiateTask(t contextTask, rng *rand.Rand) (fixtureInstance, error) {
|
||||
vars := newVars(t.Vars, rng)
|
||||
inst := fixtureInstance{Task: t, Vars: vars}
|
||||
var err error
|
||||
for _, pair := range []struct {
|
||||
dst *string
|
||||
text string
|
||||
}{
|
||||
{&inst.Prompt, t.Prompt},
|
||||
{&inst.ProbeQuery, t.ProbeQuery},
|
||||
{&inst.body, t.PlantBody},
|
||||
{&inst.callArgs, t.CallArgs},
|
||||
} {
|
||||
if *pair.dst, err = instantiate(pair.text, vars); err != nil {
|
||||
return fixtureInstance{}, fmt.Errorf("%s: %w", t.ID, err)
|
||||
}
|
||||
}
|
||||
for _, name := range t.AnswerVars {
|
||||
value, ok := vars[name]
|
||||
if !ok {
|
||||
return fixtureInstance{}, fmt.Errorf("%s: answer var %q is not declared", t.ID, name)
|
||||
}
|
||||
inst.AnswerMarkers = append(inst.AnswerMarkers, value)
|
||||
}
|
||||
if t.CueVar != "" {
|
||||
inst.CueMarker = vars[t.CueVar]
|
||||
}
|
||||
return inst, nil
|
||||
}
|
||||
|
||||
// plant appends the instantiated target and returns its canonical position.
|
||||
func (f fixtureInstance) plant(s *sessionstore.Session) int {
|
||||
at := len(s.Messages)
|
||||
if f.Task.Plant == plantAssistant {
|
||||
s.Add(provider.Message{Role: provider.RoleAssistant, Content: f.body})
|
||||
return at
|
||||
}
|
||||
s.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: f.Task.CallID, Name: f.Task.CallTool, Arguments: f.callArgs},
|
||||
}})
|
||||
s.Add(provider.Message{Role: provider.RoleTool, ToolCallID: f.Task.CallID, Name: f.Task.CallTool, Content: f.body})
|
||||
return at
|
||||
}
|
||||
|
||||
func code(name, word string) varSpec { return varSpec{Name: name, Kind: varCodename, Word: word} }
|
||||
|
||||
func num(name string, lo, hi int) varSpec {
|
||||
return varSpec{Name: name, Kind: varInt, Min: lo, Max: hi}
|
||||
}
|
||||
|
||||
// efficiencyTasks is the substrate a study measures on. Every task in it
|
||||
// qualifies under the corpus bar; the stopping corpus is deliberately excluded,
|
||||
// because a task carrying post-sufficient stopping behaviour swamps the effect
|
||||
// an efficiency batch exists to measure.
|
||||
func efficiencyTasks() []contextTask {
|
||||
var out []contextTask
|
||||
for _, t := range indexTasks() {
|
||||
if t.Role == roleEfficiency {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func searchTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "r01-retry-boundary", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "What ownership token did we settle on for the retry boundary, and under what tag did we file the reason for not moving retries above framing?",
|
||||
ProbeQuery: "retry boundary ownership token framing",
|
||||
Vars: []varSpec{code("token", "cobalt"), code("reason", "stream")},
|
||||
AnswerVars: []string{"token", "reason"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The retry boundary remains at ingress. The ownership token is {{token}}. " +
|
||||
"Moving retries above framing can duplicate a partially received stream, which we filed as {{reason}}.",
|
||||
},
|
||||
{
|
||||
ID: "r02-checkpoint-anchor", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "What was the codename of the checkpoint anchor policy, and what tag did we give the reason for not anchoring against digest positions?",
|
||||
ProbeQuery: "checkpoint anchor policy digest positions",
|
||||
Vars: []varSpec{code("policy", "northglass"), code("reason", "genlocal")},
|
||||
AnswerVars: []string{"policy", "reason"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The checkpoint anchor policy is {{policy}}. Digest positions are generation-local — " +
|
||||
"filed as {{reason}} — so recovery must anchor against canonical state.",
|
||||
},
|
||||
{
|
||||
ID: "r03-recovery-cjk", Experiment: experimentSearch, TargetKind: "assistant_reasoning_cjk",
|
||||
Prompt: "我们之前定的验证失败恢复策略代号叫什么？当时把“失败 digest 不能变成 baseline”这条登记在哪个标记下？",
|
||||
ProbeQuery: "恢复策略 baseline digest 验证失败",
|
||||
Vars: []varSpec{code("policy", "songzhen"), code("rule", "trustbase")},
|
||||
AnswerVars: []string{"policy", "rule"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "恢复策略代号是 {{policy}}。验证失败以后必须保留旧基线，不能把新的 digest 晋升为 baseline，" +
|
||||
"因为失败证据不能成为后续比较的可信基准；这条规则登记为 {{rule}}。",
|
||||
},
|
||||
{
|
||||
ID: "r04-lease-fence", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "那次 lease fence 检查报告的诊断标记是什么？expected 和 observed epoch 分别是多少？",
|
||||
ProbeQuery: "lease fence writer epoch mismatch",
|
||||
Vars: []varSpec{
|
||||
code("marker", "fence"), num("expected", 400, 900), num("observed", 100, 399),
|
||||
},
|
||||
AnswerVars: []string{"marker", "expected", "observed"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The lease fence check reported {{marker}}: expected writer epoch {{expected}}, " +
|
||||
"observed {{observed}}. We left the fence in place.",
|
||||
},
|
||||
{
|
||||
ID: "r05-schema-guard", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "当时 schema guard 报告的标记是什么，超预算多少 token？",
|
||||
ProbeQuery: "schema guard provider visible surface budget",
|
||||
Vars: []varSpec{code("marker", "amber"), num("over", 120, 480)},
|
||||
AnswerVars: []string{"marker", "over"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The schema guard reported {{marker}}: the provider-visible tool schema exceeded " +
|
||||
"the allowed surface by {{over}} tokens. We trimmed a description instead.",
|
||||
},
|
||||
{
|
||||
ID: "r06-shadow-probe", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "我们那次 shadow probe 的 marker、p95 divergence 和 generation 分别是多少？",
|
||||
ProbeQuery: "shadow probe divergence generation marker",
|
||||
Vars: []varSpec{
|
||||
code("marker", "violet"),
|
||||
{Name: "divergence", Kind: varDecimal, Min: 110, Max: 890},
|
||||
num("generation", 14, 60),
|
||||
},
|
||||
AnswerVars: []string{"marker", "divergence", "generation"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The shadow probe came back as marker={{marker}} with a p95 divergence of " +
|
||||
"{{divergence}} at generation={{generation}}, which we judged within tolerance.",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// efficiencyIndexTasks is what a study measures on: two tasks per cue tier.
|
||||
func efficiencyIndexTasks() []contextTask {
|
||||
out := indexQuarterTasks()
|
||||
out = append(out, indexHalfTasks()...)
|
||||
out = append(out, indexDefaultTasks()...)
|
||||
return out
|
||||
}
|
||||
|
||||
// indexQuarterTasks holds the quarter-tier literals.
|
||||
func indexQuarterTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i37-dispatchlag", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/dispatchlag，它打印出的 dispatch lag epoch 是多少？",
|
||||
ProbeQuery: "dispatchlag dispatch lag epoch",
|
||||
Vars: []varSpec{code("cue", "dispatchlag"), code("epoch", "cobalt")},
|
||||
AnswerVars: []string{"epoch"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "p37", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/dispatchlag {{cue}}"}`,
|
||||
PlantBody: "dispatch lag epoch = {{epoch}}",
|
||||
},
|
||||
{
|
||||
ID: "i118-fencelag", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里运行过 ./scripts/fencelag，它打印出的 fence lag 是多少？",
|
||||
ProbeQuery: "fencelag fence lag",
|
||||
Vars: []varSpec{code("cue", "fencelag"), code("v", "slate")},
|
||||
AnswerVars: []string{"v"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "x118", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/fencelag {{cue}}"}`,
|
||||
PlantBody: "fence lag = {{v}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// indexHalfTasks holds the half-tier literals.
|
||||
func indexHalfTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i70-ingestfloor", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/ingestfloor，它打印出的 route quota 是多少？",
|
||||
ProbeQuery: "ingestfloor quota",
|
||||
Vars: []varSpec{code("cue", "ingestfloor"), num("quota", 1200, 9800)},
|
||||
AnswerVars: []string{"quota"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "q70", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/ingestfloor {{cue}}"}`,
|
||||
PlantBody: "route quota = {{quota}}",
|
||||
},
|
||||
{
|
||||
ID: "i106-budgetseal", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/budgetseal，它打印出的 route quota 是多少？",
|
||||
ProbeQuery: "budgetseal route quota",
|
||||
Vars: []varSpec{code("cue", "budgetseal"), code("quota", "amber")},
|
||||
AnswerVars: []string{"quota"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "h106", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/budgetseal {{cue}}"}`,
|
||||
PlantBody: "route quota = {{quota}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// indexDefaultTasks holds the default-tier literals.
|
||||
func indexDefaultTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i47-tenantlease", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/tenantlease，它打印出的 tenant lease（秒）是多少？",
|
||||
ProbeQuery: "tenantlease tenant lease",
|
||||
Vars: []varSpec{code("cue", "tenantlease"), {Name: "lease", Kind: varDuration, Min: 15, Max: 300}},
|
||||
AnswerVars: []string{"lease"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "p47", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/tenantlease {{cue}}"}`,
|
||||
PlantBody: "tenant lease = {{lease}}",
|
||||
},
|
||||
{
|
||||
ID: "i112-lagquota", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/lagquota，它打印出的 lagquota lease 是多少？",
|
||||
ProbeQuery: "lagquota lagquota lease",
|
||||
Vars: []varSpec{code("cue", "lagquota"), {Name: "lease", Kind: varDuration, Min: 20, Max: 420}},
|
||||
AnswerVars: []string{"lease"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "d112", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/lagquota {{cue}}"}`,
|
||||
PlantBody: "lagquota lease = {{lease}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// stoppingIndexTasks is the frozen study's six tasks, kept out of efficiency
|
||||
// batches and measurable on their own with -task <id>.
|
||||
func stoppingIndexTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i01-transport-fallback", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "transport fallback probe 当时报告的 selector 是什么？",
|
||||
ProbeQuery: "transport fallback probe selector",
|
||||
Vars: []varSpec{code("cue", "tfprobe"), code("selector", "fern")},
|
||||
AnswerVars: []string{"selector"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "tf1", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/probe {{cue}}"}`,
|
||||
PlantBody: "fallback selector = {{selector}}",
|
||||
},
|
||||
{
|
||||
ID: "i02-scheduler-handoff", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "scheduler handoff 当时记录的 grace 和 marker 是什么？",
|
||||
ProbeQuery: "scheduler handoff grace marker",
|
||||
Vars: []varSpec{
|
||||
code("cue", "schedhandoff"), code("marker", "opal"),
|
||||
{Name: "grace", Kind: varDuration, Min: 18, Max: 90},
|
||||
},
|
||||
AnswerVars: []string{"grace", "marker"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "sh1", CallTool: "read_file",
|
||||
CallArgs: `{"path":"scratch/{{cue}}.txt"}`,
|
||||
PlantBody: "handoff grace = {{grace}}\nmarker = {{marker}}",
|
||||
},
|
||||
{
|
||||
ID: "i03-coalescing", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "job completion 的 coalescing probe 最后记录的 window 和 marker 是多少？",
|
||||
ProbeQuery: "coalescing policy completion window",
|
||||
Vars: []varSpec{
|
||||
code("cue", "coalescing"), code("marker", "cedar"),
|
||||
{Name: "window", Kind: varDuration, Min: 40, Max: 140},
|
||||
},
|
||||
AnswerVars: []string{"window", "marker"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "cp1", CallTool: "bash",
|
||||
CallArgs: `{"command":"grep {{cue}} scratch/job-notes.log"}`,
|
||||
PlantBody: "completion coalesce window = {{window}}\nmarker = {{marker}}",
|
||||
},
|
||||
{
|
||||
ID: "i04-recovery-fence", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "recovery fence 那次检查报告的 marker 和两个 epoch 是什么？",
|
||||
ProbeQuery: "recovery fence epoch check",
|
||||
Vars: []varSpec{
|
||||
code("cue", "recfence"), code("marker", "quartz"),
|
||||
num("expected", 400, 900), num("observed", 100, 399),
|
||||
},
|
||||
AnswerVars: []string{"marker", "expected", "observed"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "rf1", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/check {{cue}}"}`,
|
||||
PlantBody: "{{marker}}\nexpected epoch {{expected}}\nobserved epoch {{observed}}",
|
||||
},
|
||||
{
|
||||
ID: "i05-cache-probe", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "cache probe 当时记录的 prefix salt 和 scope 是什么？",
|
||||
ProbeQuery: "cache probe prefix salt scope",
|
||||
Vars: []varSpec{code("cue", "cacheprobe"), code("salt", "comet"), code("scope", "lineage")},
|
||||
AnswerVars: []string{"salt", "scope"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "cb1", CallTool: "read_file",
|
||||
CallArgs: `{"path":"scratch/{{cue}}.txt"}`,
|
||||
PlantBody: "prefix salt = {{salt}}\nscope = {{scope}}",
|
||||
},
|
||||
{
|
||||
ID: "i06-dispatch-handoff", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "dispatch handoff probe 当时确认的 mode 和 marker 是什么？",
|
||||
ProbeQuery: "dispatch handoff probe mode",
|
||||
Vars: []varSpec{code("cue", "dispatchhandoff"), code("mode", "trigger"), code("marker", "iris")},
|
||||
AnswerVars: []string{"mode", "marker"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "dh1", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/probe {{cue}}"}`,
|
||||
PlantBody: "handoff mode = {{mode}}\nmarker = {{marker}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// indexTasks is the whole index corpus, the efficiency substrate first.
|
||||
func indexTasks() []contextTask {
|
||||
out := efficiencyIndexTasks()
|
||||
return append(out, stoppingIndexTasks()...)
|
||||
}
|
||||
|
||||
func allTasks() []contextTask {
|
||||
return append(searchTasks(), indexTasks()...)
|
||||
}
|
||||
|
||||
func taskByID(id string) (contextTask, bool) {
|
||||
for _, t := range allTasks() {
|
||||
if t.ID == id {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return contextTask{}, false
|
||||
}
|
||||
|
||||
// tierScales is the fixture's contract: which arms must address a cue and
|
||||
// which must not. A policy change that moves a cue across a boundary fails
|
||||
// preflight rather than turning two arms into one experiment.
|
||||
func tierScales(tier string) (visible, hidden []string) {
|
||||
switch tier {
|
||||
case tierQuarter:
|
||||
return []string{"default", "half", "quarter"}, []string{"off"}
|
||||
case tierHalf:
|
||||
return []string{"default", "half"}, []string{"quarter", "off"}
|
||||
case tierDefault:
|
||||
return []string{"default"}, []string{"half", "quarter", "off"}
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
// boundaryPair is the one comparison an index task is built for: the narrowest
|
||||
// scale that still addresses its cue, and the next one down. Holding the
|
||||
// question fixed and moving only the affordance is what four arm averages
|
||||
// cannot do, since each of those mixes six different questions.
|
||||
func boundaryPair(tier string) (cue, noCue string) {
|
||||
switch tier {
|
||||
case tierQuarter:
|
||||
return "quarter", "off"
|
||||
case tierHalf:
|
||||
return "half", "quarter"
|
||||
case tierDefault:
|
||||
return "default", "half"
|
||||
default:
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
|
||||
func validateCorpus() error {
|
||||
seen := map[string]bool{}
|
||||
for _, t := range allTasks() {
|
||||
if seen[t.ID] {
|
||||
return fmt.Errorf("duplicate task id %q", t.ID)
|
||||
}
|
||||
seen[t.ID] = true
|
||||
if len(t.AnswerVars) == 0 || t.ProbeQuery == "" || t.Prompt == "" || t.PlantBody == "" {
|
||||
return fmt.Errorf("%s: a task needs a prompt, a probe query, a body and answer vars", t.ID)
|
||||
}
|
||||
if t.Experiment == experimentIndex {
|
||||
if t.CueVar == "" {
|
||||
return fmt.Errorf("%s: an index task needs a cue var", t.ID)
|
||||
}
|
||||
if t.Role != roleEfficiency && t.Role != roleStopping {
|
||||
return fmt.Errorf("%s: an index task needs a role (%s or %s)", t.ID, roleEfficiency, roleStopping)
|
||||
}
|
||||
if v, _ := tierScales(t.CueTier); v == nil {
|
||||
return fmt.Errorf("%s: unknown cue tier %q", t.ID, t.CueTier)
|
||||
}
|
||||
}
|
||||
if _, err := instantiateTask(t, seededRand(t.ID, "validate")); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
// dose.go — does the cost step where the cue disappears?
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Not "a bigger index is better" — CueDirectRead says the addresses are rarely
|
||||
// used. The hypothesis is that a visible action cue lowers retrieval cost,
|
||||
// which predicts a step at each task's own boundary, not a slope.
|
||||
|
||||
// scaleOrder is the dose axis, widest first.
|
||||
var scaleOrder = []string{"default", "half", "quarter", "off"}
|
||||
|
||||
// boundaryStep is the index into the three transitions where a tier's cue
|
||||
// disappears: default->half is 0, half->quarter is 1, quarter->off is 2.
|
||||
func boundaryStep(tier string) int {
|
||||
switch tier {
|
||||
case tierDefault:
|
||||
return 0
|
||||
case tierHalf:
|
||||
return 1
|
||||
case tierQuarter:
|
||||
return 2
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
// doseCurve is one task's readings across the four budgets.
|
||||
type doseCurve struct {
|
||||
Task string
|
||||
Tier string
|
||||
Metric string
|
||||
Values [4]float64
|
||||
// Missing marks arms that produced nothing. Printing those as zero draws a
|
||||
// curve falling to the floor where in fact it was never measured.
|
||||
Missing [4]bool
|
||||
Present bool
|
||||
// PreMean averages the arms where the cue is visible, PostMean the rest.
|
||||
PreMean, PostMean float64
|
||||
BoundaryDelta float64
|
||||
// BoundaryAligned: the largest transition is the one where this task's cue
|
||||
// disappears. A change elsewhere is variance shaped like an effect.
|
||||
BoundaryAligned bool
|
||||
}
|
||||
|
||||
func buildCurve(task, tier, metric string, byScale map[string]float64) doseCurve {
|
||||
c := doseCurve{Task: task, Tier: tier, Metric: metric, Present: true}
|
||||
for i, scale := range scaleOrder {
|
||||
v, ok := byScale[scale]
|
||||
if !ok {
|
||||
c.Present, c.Missing[i] = false, true
|
||||
}
|
||||
c.Values[i] = v
|
||||
}
|
||||
step := boundaryStep(tier)
|
||||
if !c.Present || step < 0 {
|
||||
return c
|
||||
}
|
||||
visible := step + 1 // arms before the boundary still show the cue
|
||||
c.PreMean = mean(c.Values[:visible])
|
||||
c.PostMean = mean(c.Values[visible:])
|
||||
c.BoundaryDelta = c.PostMean - c.PreMean
|
||||
|
||||
transitions := [3]float64{
|
||||
math.Abs(c.Values[1] - c.Values[0]),
|
||||
math.Abs(c.Values[2] - c.Values[1]),
|
||||
math.Abs(c.Values[3] - c.Values[2]),
|
||||
}
|
||||
largest, at := 0.0, -1
|
||||
for i, t := range transitions {
|
||||
if t > largest {
|
||||
largest, at = t, i
|
||||
}
|
||||
}
|
||||
// A flat curve has no largest transition to align with anything.
|
||||
c.BoundaryAligned = largest > 0 && at == step
|
||||
return c
|
||||
}
|
||||
|
||||
func mean(xs []float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
sum := 0.0
|
||||
for _, x := range xs {
|
||||
sum += x
|
||||
}
|
||||
return sum / float64(len(xs))
|
||||
}
|
||||
|
||||
func median(xs []float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
s := append([]float64(nil), xs...)
|
||||
sort.Float64s(s)
|
||||
mid := len(s) / 2
|
||||
if len(s)%2 == 1 {
|
||||
return s[mid]
|
||||
}
|
||||
return (s[mid-1] + s[mid]) / 2
|
||||
}
|
||||
|
||||
// doseMetrics are the readings a curve is drawn for.
|
||||
var doseMetrics = []struct {
|
||||
name string
|
||||
of func(contextMetrics) float64
|
||||
}{
|
||||
{"searches", func(m contextMetrics) float64 { return float64(m.SearchCalls) }},
|
||||
{"recall-tokens", func(m contextMetrics) float64 { return float64(m.RecallReturnedTokens) }},
|
||||
{"tool-rounds", func(m contextMetrics) float64 { return float64(m.RetrievalRounds + m.EscapeCalls) }},
|
||||
{"escapes", func(m contextMetrics) float64 { return float64(m.EscapeCalls) }},
|
||||
}
|
||||
|
||||
// reportDoseResponse draws each task's curve and reports where the steps fell.
|
||||
func reportDoseResponse(all []contextMetrics) string {
|
||||
valid := map[string]map[string]contextMetrics{}
|
||||
for _, m := range all {
|
||||
if m.Failed || m.Contaminated || m.FailureStage == "RunError" {
|
||||
continue
|
||||
}
|
||||
scale := strings.TrimPrefix(m.Arm, "index-")
|
||||
if valid[m.Task] == nil {
|
||||
valid[m.Task] = map[string]contextMetrics{}
|
||||
}
|
||||
valid[m.Task][scale] = m
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## Dose-response, per task\n")
|
||||
b.WriteString(" The arrow marks where this task's cue stops being addressed.\n")
|
||||
aligned := map[string]int{}
|
||||
counted := map[string]int{}
|
||||
deltas := map[string][]float64{}
|
||||
|
||||
for _, t := range indexTasks() {
|
||||
arms := valid[t.ID]
|
||||
if len(arms) == 0 {
|
||||
continue
|
||||
}
|
||||
step := boundaryStep(t.CueTier)
|
||||
fmt.Fprintf(&b, "\n %s (cue survives to %s)\n", t.ID, t.CueTier)
|
||||
fmt.Fprintf(&b, " %-14s%s\n", "", scaleHeader(step))
|
||||
for _, metric := range doseMetrics {
|
||||
byScale := map[string]float64{}
|
||||
for scale, m := range arms {
|
||||
byScale[scale] = metric.of(m)
|
||||
}
|
||||
c := buildCurve(t.ID, t.CueTier, metric.name, byScale)
|
||||
mark := " "
|
||||
if c.BoundaryAligned {
|
||||
mark = "*"
|
||||
aligned[metric.name]++
|
||||
}
|
||||
if c.Present {
|
||||
counted[metric.name]++
|
||||
deltas[metric.name] = append(deltas[metric.name], c.BoundaryDelta)
|
||||
}
|
||||
delta := fmt.Sprintf("Δ%+8.1f", c.BoundaryDelta)
|
||||
if !c.Present {
|
||||
delta = " no curve"
|
||||
}
|
||||
fmt.Fprintf(&b, " %-14s%s %s %s\n", metric.name, curveCells(c), delta, mark)
|
||||
}
|
||||
}
|
||||
|
||||
b.WriteString("\n## Where the step fell (* = largest transition is at the cue boundary)\n")
|
||||
for _, metric := range doseMetrics {
|
||||
d := deltas[metric.name]
|
||||
sensitivity := ""
|
||||
if len(d) > 1 {
|
||||
sensitivity = fmt.Sprintf(" without the largest: %+.1f", meanWithoutLargest(d))
|
||||
}
|
||||
fmt.Fprintf(&b, " %-14s boundary-aligned %d/%d mean Δ%+.1f median Δ%+.1f%s\n",
|
||||
metric.name, aligned[metric.name], counted[metric.name], mean(d), median(d), sensitivity)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// meanWithoutLargest is the robustness check: the largest effect is reported
|
||||
// and then set aside, so a conclusion never rests on one task alone.
|
||||
func meanWithoutLargest(d []float64) float64 {
|
||||
if len(d) < 2 {
|
||||
return mean(d)
|
||||
}
|
||||
worst, at := 0.0, 0
|
||||
for i, v := range d {
|
||||
if math.Abs(v) > worst {
|
||||
worst, at = math.Abs(v), i
|
||||
}
|
||||
}
|
||||
rest := append(append([]float64(nil), d[:at]...), d[at+1:]...)
|
||||
return mean(rest)
|
||||
}
|
||||
|
||||
func scaleHeader(step int) string {
|
||||
var b strings.Builder
|
||||
for i, s := range scaleOrder {
|
||||
if i == step+1 {
|
||||
s = "↓" + s
|
||||
}
|
||||
fmt.Fprintf(&b, "%8s", s)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// curveCells prints a reading per arm, and a dash where an arm produced none.
|
||||
func curveCells(c doseCurve) string {
|
||||
var b strings.Builder
|
||||
for i, v := range c.Values {
|
||||
if c.Missing[i] {
|
||||
fmt.Fprintf(&b, "%8s", "-")
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "%8.0f", v)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func percentile(xs []float64, q float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
s := append([]float64(nil), xs...)
|
||||
sort.Float64s(s)
|
||||
i := int(float64(len(s)-1) * q)
|
||||
return s[i]
|
||||
}
|
||||
|
||||
func maxOf(xs []float64) float64 {
|
||||
out := 0.0
|
||||
for _, x := range xs {
|
||||
if x > out {
|
||||
out = x
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// blockTask makes one task fail before its first model turn: its own directory
|
||||
// is a regular file, so buildFixture cannot create the session under it. It is
|
||||
// the same early exit as a failed fixture build, session load, or
|
||||
// answer-on-disk guard, and it needs no provider.
|
||||
func blockTask(t *testing.T, root, id string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(root, id), []byte("not a directory\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// mixedTaskPair is one task whose runs all fail and one whose runs all succeed.
|
||||
// Both are drawn from the corpus, so the batch is the real pipeline.
|
||||
func mixedTaskPair() (blocked, ok contextTask) {
|
||||
blocked, ok = indexTasks()[0], indexTasks()[1]
|
||||
for _, t := range indexTasks() {
|
||||
if t.Role == roleEfficiency && len(armsFor(t)) == 2 {
|
||||
ok = t
|
||||
break
|
||||
}
|
||||
}
|
||||
return blocked, ok
|
||||
}
|
||||
|
||||
// runBlockedExperiment runs one dry index batch over the given tasks, with the
|
||||
// named task failing before any model turn, and returns the batch directory, the
|
||||
// runs, the manifest, and everything the run printed.
|
||||
func runBlockedExperiment(t *testing.T, tasks []contextTask, blocked ...string) (string, []contextMetrics, batchManifest, string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for _, id := range blocked {
|
||||
blockTask(t, root, id)
|
||||
}
|
||||
out := captureStdout(t, func() {
|
||||
if code := runExperiment("run-index", experimentIndex, root, true, tasks); code != 0 {
|
||||
t.Errorf("runExperiment exit = %d, want 0 (the batch finished)", code)
|
||||
}
|
||||
})
|
||||
runs, _ := readResultLines(t, root)
|
||||
manifest, _ := readManifest(t, root)
|
||||
return root, runs, manifest, out
|
||||
}
|
||||
|
||||
// captureStdout reads everything the function prints. os.Pipe is unbuffered, so
|
||||
// the reader must run beside the call.
|
||||
func captureStdout(t *testing.T, f func()) string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := os.Stdout
|
||||
os.Stdout = w
|
||||
done := make(chan string, 1)
|
||||
go func() {
|
||||
var b strings.Builder
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := r.Read(buf)
|
||||
b.Write(buf[:n])
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
done <- b.String()
|
||||
}()
|
||||
f()
|
||||
os.Stdout = old
|
||||
w.Close()
|
||||
text := <-done
|
||||
r.Close()
|
||||
return text
|
||||
}
|
||||
|
||||
// funnelScored reads the "runs=N" column of the named arm's funnel line out of
|
||||
// the printed tables, so the assertion is on the number a reader sees.
|
||||
func funnelScored(t *testing.T, out, arm string) int {
|
||||
t.Helper()
|
||||
for line := range strings.SplitSeq(out, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 || fields[0] != arm || !strings.HasPrefix(fields[1], "runs=") {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.Atoi(strings.TrimPrefix(fields[1], "runs="))
|
||||
if err != nil {
|
||||
t.Fatalf("funnel line for %s: %v\n%s", arm, err, line)
|
||||
}
|
||||
return n
|
||||
}
|
||||
t.Fatalf("no funnel line for arm %s:\n%s", arm, out)
|
||||
return 0
|
||||
}
|
||||
|
||||
// The batch report states both counts on one line, so a reader of the scrollback
|
||||
// sees the scored set and the failures without counting funnel rows.
|
||||
func reportCounts(t *testing.T, out string) (scored, failed int) {
|
||||
t.Helper()
|
||||
for line := range strings.SplitSeq(out, "\n") {
|
||||
if !strings.Contains(line, " scored=") || !strings.Contains(line, " failed=") {
|
||||
continue
|
||||
}
|
||||
for f := range strings.FieldsSeq(line) {
|
||||
switch {
|
||||
case strings.HasPrefix(f, "scored="):
|
||||
scored, _ = strconv.Atoi(strings.TrimPrefix(f, "scored="))
|
||||
case strings.HasPrefix(f, "failed="):
|
||||
failed, _ = strconv.Atoi(strings.TrimPrefix(f, "failed="))
|
||||
}
|
||||
}
|
||||
return scored, failed
|
||||
}
|
||||
t.Fatalf("no line reports scored= and failed=:\n%s", out)
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
// A batch that mixes a task that never ran with one that did must read as its
|
||||
// successful runs only: the failed task leaves Scored, the funnel, and the
|
||||
// completed count, and is reported as a failure with its identity instead.
|
||||
// Every assertion here is a number, not the absence of an empty row.
|
||||
func TestMixedBatchCountsOnlyTheRunThatProducedEvidence(t *testing.T) {
|
||||
blocked, ok := mixedTaskPair()
|
||||
_, runs, manifest, out := runBlockedExperiment(t, []contextTask{blocked, ok}, blocked.ID)
|
||||
|
||||
wantScored, wantFailed := len(armsFor(ok)), len(armsFor(blocked))
|
||||
if manifest.CompletedRuns != wantScored || manifest.FailedRuns != wantFailed {
|
||||
t.Fatalf("manifest completed=%d failed=%d, want %d and %d (planned=%d)",
|
||||
manifest.CompletedRuns, manifest.FailedRuns, wantScored, wantFailed, manifest.PlannedRuns)
|
||||
}
|
||||
if len(runs) != manifest.CompletedRuns+manifest.FailedRuns {
|
||||
t.Fatalf("results.jsonl has %d rows but completed=%d failed=%d",
|
||||
len(runs), manifest.CompletedRuns, manifest.FailedRuns)
|
||||
}
|
||||
for _, id := range manifest.TaskIDs {
|
||||
if id == blocked.ID {
|
||||
t.Fatalf("task_ids lists %s, which produced no evidence: %v", blocked.ID, manifest.TaskIDs)
|
||||
}
|
||||
}
|
||||
scored, failures := 0, 0
|
||||
for _, m := range runs {
|
||||
if m.Failed {
|
||||
failures++
|
||||
if m.Task != blocked.ID || m.FailureStage == "" || m.Reason == "" {
|
||||
t.Fatalf("failed row lost its identity or stage: %+v", m)
|
||||
}
|
||||
continue
|
||||
}
|
||||
scored++
|
||||
if m.Task != ok.ID {
|
||||
t.Fatalf("a run scored from the wrong task: %+v", m)
|
||||
}
|
||||
}
|
||||
if scored != wantScored || failures != wantFailed {
|
||||
t.Fatalf("rows scored=%d failed=%d, want %d and %d", scored, failures, wantScored, wantFailed)
|
||||
}
|
||||
// Each arm's funnel holds exactly the successful task's runs, and the two
|
||||
// scored counts agree with each other and with the manifest.
|
||||
total := 0
|
||||
for _, arm := range armsFor(ok) {
|
||||
total += funnelScored(t, out, arm.name)
|
||||
}
|
||||
if total != manifest.CompletedRuns {
|
||||
t.Fatalf("funnels scored %d runs, manifest completed_runs = %d:\n%s", total, manifest.CompletedRuns, out)
|
||||
}
|
||||
if gotScored, gotFailed := reportCounts(t, out); gotScored != manifest.CompletedRuns || gotFailed != manifest.FailedRuns {
|
||||
t.Fatalf("report says scored=%d failed=%d, manifest says %d and %d",
|
||||
gotScored, gotFailed, manifest.CompletedRuns, manifest.FailedRuns)
|
||||
}
|
||||
if strings.Contains(out, "## Where it broke") && strings.Contains(out, "=-\n") {
|
||||
t.Fatalf("the stage table kept an empty stage:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch where every run fails must not read as an empty success: zero scored
|
||||
// runs, a nonzero failed count, and both stated.
|
||||
func TestAllFailedBatchReportsZeroScoredAndNonzeroFailed(t *testing.T) {
|
||||
blocked, _ := mixedTaskPair()
|
||||
_, runs, manifest, out := runBlockedExperiment(t, []contextTask{blocked}, blocked.ID)
|
||||
|
||||
if manifest.CompletedRuns != 0 {
|
||||
t.Fatalf("completed_runs = %d, want 0 (nothing produced evidence)", manifest.CompletedRuns)
|
||||
}
|
||||
if manifest.FailedRuns != manifest.PlannedRuns || manifest.FailedRuns == 0 {
|
||||
t.Fatalf("failed_runs = %d, planned_runs = %d: every planned run failed",
|
||||
manifest.FailedRuns, manifest.PlannedRuns)
|
||||
}
|
||||
if len(manifest.TaskIDs) != 0 {
|
||||
t.Fatalf("task_ids = %v, want none", manifest.TaskIDs)
|
||||
}
|
||||
for i, m := range runs {
|
||||
if !m.Failed || m.FailureStage == "" || m.Task != blocked.ID {
|
||||
t.Fatalf("results.jsonl line %d is not a marked failure: %+v", i+1, m)
|
||||
}
|
||||
}
|
||||
total := 0
|
||||
for _, arm := range armsFor(blocked) {
|
||||
total += funnelScored(t, out, arm.name)
|
||||
}
|
||||
if total != 0 {
|
||||
t.Fatalf("funnels scored %d runs although nothing ran:\n%s", total, out)
|
||||
}
|
||||
if gotScored, gotFailed := reportCounts(t, out); gotScored != 0 || gotFailed != manifest.FailedRuns {
|
||||
t.Fatalf("report says scored=%d failed=%d, want 0 and %d", gotScored, gotFailed, manifest.FailedRuns)
|
||||
}
|
||||
}
|
||||
|
||||
// The control: with nothing blocked, a batch is what it always was. Every run
|
||||
// is completed with a stage, two identical batches write byte-identical results
|
||||
// and the same manifest header, and the new failure field stays out of it.
|
||||
func TestAllSuccessfulBatchKeepsItsArtifacts(t *testing.T) {
|
||||
tasks := indexTasks()
|
||||
first, runs, manifest, out := runBlockedExperiment(t, tasks)
|
||||
second, again, repeat, _ := runBlockedExperiment(t, tasks)
|
||||
if manifest.CompletedRuns != len(runs) || manifest.FailedRuns != 0 {
|
||||
t.Fatalf("completed=%d failed=%d rows=%d, want every row completed",
|
||||
manifest.CompletedRuns, manifest.FailedRuns, len(runs))
|
||||
}
|
||||
if manifest.CompletedRuns != manifest.PlannedRuns {
|
||||
t.Fatalf("completed=%d planned=%d, want all planned", manifest.CompletedRuns, manifest.PlannedRuns)
|
||||
}
|
||||
for i, m := range runs {
|
||||
if m.Failed || m.Task == "" || m.Arm == "" || m.FailureStage == "" {
|
||||
t.Fatalf("results.jsonl line %d is not a completed run: %+v", i+1, m)
|
||||
}
|
||||
}
|
||||
if gotScored, gotFailed := reportCounts(t, out); gotScored != manifest.CompletedRuns || gotFailed != 0 {
|
||||
t.Fatalf("report says scored=%d failed=%d, want %d and 0", gotScored, gotFailed, manifest.CompletedRuns)
|
||||
}
|
||||
if !reflect.DeepEqual(runs, again) {
|
||||
t.Fatal("two identical successful batches scored differently")
|
||||
}
|
||||
// Byte-for-byte on the runs. Trajectories carry per-message wall-clock
|
||||
// stamps, so they are compared field by field instead.
|
||||
if got, want := readFile(t, second, "results.jsonl"), readFile(t, first, "results.jsonl"); got != want {
|
||||
t.Fatalf("an unchanged batch wrote different results.jsonl:\n%s\n---\n%s", want, got)
|
||||
}
|
||||
gotTraj, wantTraj := withoutStamps(t, readFile(t, second, "trajectories.jsonl")),
|
||||
withoutStamps(t, readFile(t, first, "trajectories.jsonl"))
|
||||
if gotTraj != wantTraj {
|
||||
t.Fatalf("an unchanged batch wrote different trajectories:\n%s\n---\n%s", wantTraj, gotTraj)
|
||||
}
|
||||
repeat.StartedAt, repeat.FinishedAt = manifest.StartedAt, manifest.FinishedAt
|
||||
repeat.TrajectoriesSHA256 = manifest.TrajectoriesSHA256
|
||||
if !reflect.DeepEqual(repeat, manifest) {
|
||||
t.Fatalf("an unchanged batch wrote a different manifest:\n%+v\n---\n%+v", manifest, repeat)
|
||||
}
|
||||
if repeat.FailedRuns != 0 {
|
||||
t.Fatalf("a batch with no failure reports failed_runs=%d", repeat.FailedRuns)
|
||||
}
|
||||
}
|
||||
|
||||
// withoutStamps drops the createdAt stamp every message carries, which is the
|
||||
// one field a dry batch cannot reproduce between two runs.
|
||||
func withoutStamps(t *testing.T, text string) string {
|
||||
t.Helper()
|
||||
var b strings.Builder
|
||||
for line := range strings.SplitSeq(strings.TrimSpace(text), "\n") {
|
||||
var row map[string]any
|
||||
if err := json.Unmarshal([]byte(line), &row); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stripStamp(row)
|
||||
out, err := json.Marshal(row)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.Write(out)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func stripStamp(v any) {
|
||||
switch typed := v.(type) {
|
||||
case map[string]any:
|
||||
delete(typed, "createdAt")
|
||||
delete(typed, "workDurationMs")
|
||||
for _, child := range typed {
|
||||
stripStamp(child)
|
||||
}
|
||||
case []any:
|
||||
for _, child := range typed {
|
||||
stripStamp(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, root, name string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(root, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/runtime/agent"
|
||||
)
|
||||
|
||||
// A fixture is a real session that has really been compacted: no bench-only
|
||||
// state shape, so the loader and the projection restore have to work on it.
|
||||
|
||||
const (
|
||||
fixtureWindow = 60000
|
||||
fixtureGenerations = 5
|
||||
// fillerBytes per unit is what makes a fold necessary rather than optional.
|
||||
fillerBytes = 6000
|
||||
)
|
||||
|
||||
// digestProvider answers a summarizer without naming anything the corpus
|
||||
// plants. A digest that happened to quote the answer would leave it visible
|
||||
// and silently turn the task into a no-op.
|
||||
type digestProvider struct{ calls int }
|
||||
|
||||
func (p *digestProvider) Name() string { return "fixture-digest" }
|
||||
|
||||
func (p *digestProvider) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.calls++
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Earlier work\nEarlier work continued across several files; nothing outstanding was recorded."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// fillerCall is one ordinary read whose only job is to take up room — in the
|
||||
// window, and in the fold index ahead of a cue.
|
||||
func fillerCall(sess *sessionstore.Session, n int) {
|
||||
id := fmt.Sprintf("f%04d", n)
|
||||
sess.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: id, Name: "read_file", Arguments: fmt.Sprintf(`{"path":"internal/mod%03d/handler%03d.go"}`, n%37, n)},
|
||||
}})
|
||||
sess.Add(provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "read_file", Content: fillerBody(n)})
|
||||
}
|
||||
|
||||
func fillerBody(n int) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "package mod%03d\n\n", n%37)
|
||||
for b.Len() < fillerBytes {
|
||||
fmt.Fprintf(&b, "func handler%03dStep%02d() error { return nil }\n", n, b.Len()%64)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// builtFixture is one task's history after real folds, plus where its target
|
||||
// ended up.
|
||||
type builtFixture struct {
|
||||
Session *sessionstore.Session
|
||||
State sessionstore.CompactionState
|
||||
Target int
|
||||
Path string
|
||||
Instance fixtureInstance
|
||||
}
|
||||
|
||||
// buildFixture plays a task's history through the real agent and folds it at
|
||||
// the given arm, writing the session and its context state where the agent
|
||||
// itself would.
|
||||
func buildFixture(f fixtureInstance, arm ablation.Set, sessionPath string) (builtFixture, error) {
|
||||
t := f.Task
|
||||
sess := sessionstore.NewSession("You are a coding agent.")
|
||||
sess.Add(provider.Message{Role: provider.RoleUser, Content: "Work through the transport and scheduler backlog."})
|
||||
|
||||
n := 0
|
||||
at := -1
|
||||
|
||||
a := agent.New(&digestProvider{}, tool.NewRegistry(), sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: sessionPath, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, event.Discard)
|
||||
|
||||
for gen := range fixtureGenerations {
|
||||
if gen == t.PlantAfterGen {
|
||||
at = f.plant(sess)
|
||||
}
|
||||
for range 8 {
|
||||
fillerCall(sess, n)
|
||||
n++
|
||||
}
|
||||
if _, err := a.CompactNow(context.Background(), agent.CompactRequest{}); err != nil {
|
||||
return builtFixture{}, fmt.Errorf("%s: generation %d fold: %w", t.ID, gen, err)
|
||||
}
|
||||
}
|
||||
if at < 0 {
|
||||
return builtFixture{}, fmt.Errorf("%s: PlantAfterGen %d is past the last generation", t.ID, t.PlantAfterGen)
|
||||
}
|
||||
// projectionCoversTail cannot revalidate a projection covering the whole
|
||||
// transcript: it needs a live tail, which a real session always has because
|
||||
// work continues after a fold. Neutral text — the tail is provider-visible.
|
||||
sess.Add(provider.Message{Role: provider.RoleAssistant, Content: "Backlog swept; ready for the next question."})
|
||||
if err := sess.Save(sessionPath); err != nil {
|
||||
return builtFixture{}, fmt.Errorf("%s: save session: %w", t.ID, err)
|
||||
}
|
||||
state, ok, err := sessionstore.LoadCompactionState(sessionPath)
|
||||
if err != nil {
|
||||
return builtFixture{}, fmt.Errorf("%s: load context state: %w", t.ID, err)
|
||||
}
|
||||
if !ok || len(state.Projection.Messages) == 0 {
|
||||
return builtFixture{}, fmt.Errorf("%s: no projection was installed", t.ID)
|
||||
}
|
||||
if state.Projection.CoveredCount <= at {
|
||||
return builtFixture{}, fmt.Errorf("%s: target #%d is not inside the folded region (covered %d)",
|
||||
t.ID, at, state.Projection.CoveredCount)
|
||||
}
|
||||
return builtFixture{Session: sess, State: state, Target: at, Path: sessionPath, Instance: f}, nil
|
||||
}
|
||||
|
||||
// visibleContext is the model-visible view: the stored projection spliced with
|
||||
// everything canonical that came after it.
|
||||
func visibleContext(f builtFixture) []provider.Message {
|
||||
canonical := f.Session.Snapshot()
|
||||
out := append([]provider.Message(nil), f.State.Projection.Messages...)
|
||||
if n := f.State.Projection.CoveredCount; n >= 0 && n < len(canonical) {
|
||||
out = append(out, canonical[n:]...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// visibleText is everything the model would read, as one string.
|
||||
func visibleText(f builtFixture) string {
|
||||
var b strings.Builder
|
||||
for _, m := range visibleContext(f) {
|
||||
b.WriteString(m.Content)
|
||||
b.WriteString("\n")
|
||||
for _, tc := range m.ToolCalls {
|
||||
b.WriteString(tc.Name)
|
||||
b.WriteString(" ")
|
||||
b.WriteString(string(tc.Arguments))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// sealFixture removes the fixture from disk once the agent holds it in memory.
|
||||
// The canonical transcript is the answer, and the sandbox mounts the host
|
||||
// read-only by design — so a model that finds the path reads the answer, which
|
||||
// one did. The agent has already loaded the session and its sidecar by the time
|
||||
// this runs; a failed autosave afterwards costs the benchmark nothing.
|
||||
func sealFixture(sessionPath string) {
|
||||
// Every file, not two named ones: a session writes an event log and a
|
||||
// context sidecar beside the transcript, and session.events.jsonl still
|
||||
// carried the answer after the other two were sealed by name.
|
||||
dir := filepath.Dir(sessionPath)
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
_ = os.Remove(filepath.Join(dir, e.Name()))
|
||||
}
|
||||
}
|
||||
// Deny the directory too, so an autosave cannot recreate what was removed.
|
||||
_ = os.Chmod(dir, 0o500)
|
||||
}
|
||||
|
||||
// unsealFixture restores the directory so the harness can clean up after
|
||||
// itself.
|
||||
func unsealFixture(sessionPath string) {
|
||||
_ = os.Chmod(filepath.Dir(sessionPath), 0o700)
|
||||
}
|
||||
|
||||
// answerOnDisk reports any readable file under dir holding a current-run
|
||||
// answer. Two named files were sealed because two leaks were found in them;
|
||||
// this scans the directory, so a path that moves — an autosave, a recovery
|
||||
// tombstone, a future sidecar — fails here instead of on a paid run.
|
||||
func answerOnDisk(dir string, markers []string) (string, string) {
|
||||
var atPath, found string
|
||||
_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || atPath != "" {
|
||||
return nil //nolint:nilerr // an unreadable entry is not a leak
|
||||
}
|
||||
body, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
return nil
|
||||
}
|
||||
for _, marker := range markers {
|
||||
if len(marker) >= 6 && strings.Contains(string(body), marker) {
|
||||
atPath, found = path, marker
|
||||
return filepath.SkipAll
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return atPath, found
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
// instance.go — a task template becomes one run's concrete history.
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"io"
|
||||
mathrand "math/rand"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The invariant this file exists for: no literal sufficient to satisfy
|
||||
// AnswerRecovered for a concrete run may exist in this repository before that
|
||||
// run begins. Grepping the corpus reveals the question, never the answer.
|
||||
|
||||
// varKind is how one placeholder's value is generated. Values stay shaped like
|
||||
// real engineering records — a UUID would defeat the leak but turn every task
|
||||
// into "find the random string".
|
||||
type varKind string
|
||||
|
||||
const (
|
||||
varCodename varKind = "codename" // cobalt-k7m4q9
|
||||
varInt varKind = "int" // 731
|
||||
varDecimal varKind = "decimal" // 0.0437
|
||||
varDuration varKind = "duration" // 43ms
|
||||
)
|
||||
|
||||
// varSpec declares one placeholder. Min/Max bound the numeric kinds.
|
||||
type varSpec struct {
|
||||
Name string
|
||||
Kind varKind
|
||||
Word string // codename stem, e.g. "cobalt"
|
||||
Min int
|
||||
Max int
|
||||
}
|
||||
|
||||
// codenameEntropy is how much of a codename is unguessable. Five base32 digits
|
||||
// is about 25 bits: far past what a model can reach by knowing the stem.
|
||||
const codenameEntropy = 5
|
||||
|
||||
const codenameAlphabet = "abcdefghijkmnpqrstuvwxyz23456789"
|
||||
|
||||
// fixtureVars is one run's instantiated values.
|
||||
type fixtureVars map[string]string
|
||||
|
||||
// newVars generates values for every spec. A nil rng means deterministic.
|
||||
func newVars(specs []varSpec, rng *mathrand.Rand) fixtureVars {
|
||||
out := fixtureVars{}
|
||||
for _, spec := range specs {
|
||||
switch spec.Kind {
|
||||
case varCodename:
|
||||
out[spec.Name] = spec.Word + "-" + randomToken(rng, codenameEntropy)
|
||||
case varInt:
|
||||
out[spec.Name] = fmt.Sprint(randomBetween(rng, spec.Min, spec.Max))
|
||||
case varDecimal:
|
||||
out[spec.Name] = fmt.Sprintf("0.%04d", randomBetween(rng, spec.Min, spec.Max))
|
||||
case varDuration:
|
||||
out[spec.Name] = fmt.Sprintf("%dms", randomBetween(rng, spec.Min, spec.Max))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func randomToken(rng *mathrand.Rand, n int) string {
|
||||
var b strings.Builder
|
||||
for range n {
|
||||
b.WriteByte(codenameAlphabet[randomBetween(rng, 0, len(codenameAlphabet)-1)])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func randomBetween(rng *mathrand.Rand, lo, hi int) int {
|
||||
if hi <= lo {
|
||||
return lo
|
||||
}
|
||||
return lo + rng.Intn(hi-lo+1)
|
||||
}
|
||||
|
||||
// seededRand is the deterministic generator preflight and tests use, so a
|
||||
// fixture's tier calibration and its golden state reproduce exactly.
|
||||
func seededRand(parts ...string) *mathrand.Rand {
|
||||
h := fnv.New64a()
|
||||
for _, p := range parts {
|
||||
h.Write([]byte(p))
|
||||
h.Write([]byte{0})
|
||||
}
|
||||
return mathrand.New(mathrand.NewSource(int64(h.Sum64())))
|
||||
}
|
||||
|
||||
// liveRand is the generator a paid run uses: every run gets values that existed
|
||||
// nowhere before it started.
|
||||
func liveRand() *mathrand.Rand {
|
||||
rng, err := liveRandFrom(rand.Reader)
|
||||
if err != nil {
|
||||
panic("contextbench: no entropy for a live run: " + err.Error())
|
||||
}
|
||||
return rng
|
||||
}
|
||||
|
||||
// liveRandFrom seeds a generator from an entropy source, so what a run's values
|
||||
// are unpredictable from is the source and nothing else.
|
||||
func liveRandFrom(entropy io.Reader) (*mathrand.Rand, error) {
|
||||
var seed [8]byte
|
||||
if _, err := io.ReadFull(entropy, seed[:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var n int64
|
||||
for _, b := range seed {
|
||||
n = n<<8 | int64(b)
|
||||
}
|
||||
return mathrand.New(mathrand.NewSource(n)), nil
|
||||
}
|
||||
|
||||
// instantiate substitutes {{name}} and refuses anything left over. The grammar
|
||||
// is one identifier in double braces: no conditionals, no loops, no defaults.
|
||||
// This is fixture data, not a template language.
|
||||
func instantiate(text string, vars fixtureVars) (string, error) {
|
||||
out := text
|
||||
for name, value := range vars {
|
||||
out = strings.ReplaceAll(out, "{{"+name+"}}", value)
|
||||
}
|
||||
if strings.Contains(out, "{{") || strings.Contains(out, "}}") {
|
||||
return "", fmt.Errorf("unresolved placeholder in %q", firstPlaceholder(out))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func firstPlaceholder(text string) string {
|
||||
i := strings.Index(text, "{{")
|
||||
if i < 0 {
|
||||
return text
|
||||
}
|
||||
j := strings.Index(text[i:], "}}")
|
||||
if j < 0 {
|
||||
return text[i:min(i+40, len(text))]
|
||||
}
|
||||
return text[i : i+j+2]
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
// ContextRetrievalBench builds the fixtures the long-horizon retrieval
|
||||
// experiments run against, and proves they are worth paying a provider for.
|
||||
//
|
||||
// -mode=calibrate find each index task's lead-call count for its cue tier
|
||||
// -mode=preflight build every task under every arm and check the contract
|
||||
//
|
||||
// Nothing here calls a provider. The experiments themselves do; this is what
|
||||
// says they would measure what they claim to.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
mode := flag.String("mode", "preflight", "calibrate | preflight | snippet | adversarial | run-search | run-boundary | run-index")
|
||||
work := flag.String("work", "", "directory for built fixtures (default: a temp dir)")
|
||||
only := flag.String("task", "", "limit to one task id")
|
||||
dry := flag.Bool("dry", false, "run modes: drive the pipeline with a scripted provider instead of paying for one")
|
||||
flag.Parse()
|
||||
|
||||
if err := validateCorpus(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "corpus:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
root := *work
|
||||
if root == "" {
|
||||
dir, err := os.MkdirTemp("", "context-retrieval-*")
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
root = dir
|
||||
}
|
||||
|
||||
tasks := allTasks()
|
||||
if *only != "" {
|
||||
t, ok := taskByID(*only)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "unknown task %q\n", *only)
|
||||
os.Exit(2)
|
||||
}
|
||||
tasks = []contextTask{t}
|
||||
}
|
||||
|
||||
switch *mode {
|
||||
case "calibrate":
|
||||
os.Exit(runCalibrate(tasks, root))
|
||||
case "preflight":
|
||||
os.Exit(runPreflight(tasks, root))
|
||||
case "run-search":
|
||||
os.Exit(runExperiment(*mode, experimentSearch, root, *dry, tasks))
|
||||
case "run-boundary":
|
||||
os.Exit(runBoundaries(*mode, root))
|
||||
case "snippet":
|
||||
os.Exit(runSnippetAudit(root, 100))
|
||||
case "adversarial":
|
||||
os.Exit(runAdversarial(root))
|
||||
case "run-index":
|
||||
batch := tasks
|
||||
if *only == "" {
|
||||
// A batch measures the efficiency substrate only. Naming one task
|
||||
// with -task runs that task, role included, so the stopping corpus
|
||||
// stays measurable on its own.
|
||||
batch = efficiencyTasks()
|
||||
}
|
||||
os.Exit(runExperiment(*mode, experimentIndex, root, *dry, batch))
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown mode %q\n", *mode)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func runPreflight(tasks []contextTask, root string) int {
|
||||
var all []finding
|
||||
for _, t := range tasks {
|
||||
found, err := checkTask(t, root)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", t.ID, err)
|
||||
return 2
|
||||
}
|
||||
all = append(all, found...)
|
||||
status := "ok"
|
||||
if len(found) > 0 {
|
||||
status = fmt.Sprintf("%d problem(s)", len(found))
|
||||
}
|
||||
fmt.Printf("%-24s %-6s gen=%-2d %s\n", t.ID, t.Experiment, t.PlantAfterGen, status)
|
||||
}
|
||||
if len(all) == 0 {
|
||||
fmt.Printf("\n%d fixtures ready.\n", len(tasks))
|
||||
return 0
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "\n%d preflight problem(s):\n", len(all))
|
||||
sort.Slice(all, func(i, j int) bool { return all[i].Task < all[j].Task })
|
||||
for _, f := range all {
|
||||
fmt.Fprintln(os.Stderr, " ", f)
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// runCalibrate reports, for each index task, the lead-call counts at which its
|
||||
// cue is still addressed by each budget. The corpus freezes one number from
|
||||
// this; it is never computed at run time, or the implementation under test
|
||||
// would be moving its own goalposts.
|
||||
func runCalibrate(tasks []contextTask, root string) int {
|
||||
for _, t := range tasks {
|
||||
if t.Experiment != experimentIndex {
|
||||
continue
|
||||
}
|
||||
var rows []string
|
||||
for lead := range fixtureGenerations {
|
||||
probe := t
|
||||
probe.PlantAfterGen = lead
|
||||
var visibleAt []string
|
||||
for _, scale := range armScales {
|
||||
inst, ierr := instantiateTask(probe, seededRand(t.ID, scale, "calibrate"))
|
||||
if ierr != nil {
|
||||
fmt.Fprintln(os.Stderr, ierr)
|
||||
return 2
|
||||
}
|
||||
f, err := buildFixture(inst, armFor(scale, false),
|
||||
fmt.Sprintf("%s/calib/%s/%d-%s/session.jsonl", root, t.ID, lead, scale))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s gen=%d %s: %v\n", t.ID, lead, scale, err)
|
||||
return 2
|
||||
}
|
||||
if strings.Contains(visibleText(f), inst.CueMarker) {
|
||||
visibleAt = append(visibleAt, scale)
|
||||
}
|
||||
}
|
||||
rows = append(rows, fmt.Sprintf("planted after gen %d, cue visible at: %s", lead, strings.Join(visibleAt, ",")))
|
||||
}
|
||||
fmt.Printf("\n%s (wants tier %s)\n", t.ID, t.CueTier)
|
||||
for _, r := range rows {
|
||||
fmt.Println(" ", r)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"maps"
|
||||
mathrand "math/rand"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The same checks the preflight mode runs, under go test: a compaction change
|
||||
// that invalidates the fixtures fails here rather than three hundred API calls
|
||||
// later.
|
||||
|
||||
func TestCorpusIsWellFormed(t *testing.T) {
|
||||
if err := validateCorpus(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(searchTasks()); got != 6 {
|
||||
t.Errorf("search experiment has %d tasks, want 6", got)
|
||||
}
|
||||
if got := len(indexTasks()); got != 12 {
|
||||
t.Errorf("index experiment has %d tasks, want 12", got)
|
||||
}
|
||||
stopping := map[string]bool{}
|
||||
tiers := map[string]int{}
|
||||
for _, task := range indexTasks() {
|
||||
if task.Role == roleStopping {
|
||||
stopping[task.ID] = true
|
||||
continue
|
||||
}
|
||||
tiers[task.CueTier]++
|
||||
}
|
||||
if !stopping["i03-coalescing"] || !stopping["i04-recovery-fence"] {
|
||||
t.Errorf("the stopping corpus is %v; it must keep i03-coalescing and i04-recovery-fence", stopping)
|
||||
}
|
||||
// Two per tier, over the efficiency substrate, is what makes the staircase
|
||||
// readable: the same task can be compared across the boundary its cue sits
|
||||
// on, instead of two different questions being compared across two arms.
|
||||
for _, tier := range []string{tierQuarter, tierHalf, tierDefault} {
|
||||
if tiers[tier] != 2 {
|
||||
t.Errorf("tier %s has %d tasks, want 2", tier, tiers[tier])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The role decides which batch a task enters; the literal lists are only where
|
||||
// it is declared, so the two must not drift.
|
||||
func TestIndexListsMatchTheirRoles(t *testing.T) {
|
||||
for _, task := range append(append(indexQuarterTasks(), indexHalfTasks()...), indexDefaultTasks()...) {
|
||||
if task.Role != roleEfficiency {
|
||||
t.Errorf("%s sits in an efficiency list with role %q", task.ID, task.Role)
|
||||
}
|
||||
}
|
||||
for _, task := range stoppingIndexTasks() {
|
||||
if task.Role != roleStopping {
|
||||
t.Errorf("%s sits in the stopping list with role %q", task.ID, task.Role)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The invariant the whole instantiation layer exists for: no literal a scorer
|
||||
// needs may be readable in this repository before a run starts.
|
||||
func TestNoAnswerLiteralExistsInTheRepository(t *testing.T) {
|
||||
root, err := repoRoot()
|
||||
if err != nil {
|
||||
t.Skip(err)
|
||||
}
|
||||
for _, task := range allTasks() {
|
||||
inst, err := instantiateTask(task, seededRand(task.ID, "leak-check"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, marker := range append(inst.AnswerMarkers, inst.CueMarker) {
|
||||
if len(marker) < 6 {
|
||||
continue // a bare number is not a literal anyone can grep for
|
||||
}
|
||||
if found, where := grepRepo(t, root, marker); found {
|
||||
t.Errorf("%s: %q is readable at %s before the run", task.ID, marker, where)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A run's values are unpredictable from a previous run only if the live
|
||||
// generator is seeded from entropy. Comparing answers across two instances
|
||||
// cannot assert that: a task whose answer is a bounded number collides by
|
||||
// chance (1/8601 to 1/286 per task), so the check is on the seed source.
|
||||
func TestLiveRandDrawsFromEntropy(t *testing.T) {
|
||||
a, b := liveRand(), liveRand()
|
||||
for range 4 {
|
||||
if a.Int63() != b.Int63() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("two live generators produced the same stream: they are seeded alike")
|
||||
}
|
||||
|
||||
// The source alone decides an instance: the same entropy gives the same
|
||||
// answers, different entropy gives different values.
|
||||
func TestLiveRandSourceDecidesTheInstance(t *testing.T) {
|
||||
same := func() *mathrand.Rand {
|
||||
rng, err := liveRandFrom(bytes.NewReader(bytes.Repeat([]byte{7}, 8)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rng
|
||||
}
|
||||
other, err := liveRandFrom(bytes.NewReader(bytes.Repeat([]byte{9}, 8)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := liveRandFrom(bytes.NewReader(nil)); err == nil {
|
||||
t.Fatal("an empty entropy source was accepted")
|
||||
}
|
||||
for _, task := range allTasks() {
|
||||
a, _ := instantiateTask(task, same())
|
||||
b, _ := instantiateTask(task, same())
|
||||
if !slices.Equal(a.AnswerMarkers, b.AnswerMarkers) || !maps.Equal(a.Vars, b.Vars) {
|
||||
t.Errorf("%s: the same entropy gave different instances", task.ID)
|
||||
}
|
||||
c, _ := instantiateTask(task, other)
|
||||
if maps.Equal(a.Vars, c.Vars) {
|
||||
t.Errorf("%s: different entropy gave the same values", task.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A preflight that could not reproduce would calibrate against one run's
|
||||
// nonces and report about another's.
|
||||
func TestSeededInstancesReproduce(t *testing.T) {
|
||||
for _, task := range allTasks() {
|
||||
a, _ := instantiateTask(task, seededRand(task.ID, "x"))
|
||||
b, _ := instantiateTask(task, seededRand(task.ID, "x"))
|
||||
if !slices.Equal(a.AnswerMarkers, b.AnswerMarkers) || a.body != b.body {
|
||||
t.Errorf("%s did not reproduce under the same seed", task.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An unresolved placeholder would put "{{marker}}" in the transcript and score
|
||||
// every run wrong; it fails the fixture instead.
|
||||
func TestUnresolvedPlaceholderIsRefused(t *testing.T) {
|
||||
if _, err := instantiate("value is {{missing}}", fixtureVars{"other": "x"}); err == nil {
|
||||
t.Fatal("an unresolved placeholder was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func repoRoot() (string, error) {
|
||||
out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("not in a git checkout: %w", err)
|
||||
}
|
||||
return strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
|
||||
// grepRepo searches tracked files only: build artifacts and temp fixtures are
|
||||
// not what a fresh checkout hands anyone.
|
||||
func grepRepo(t *testing.T, root, needle string) (bool, string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", "grep", "-l", "-F", needle)
|
||||
cmd.Dir = root
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return false, "" // exit 1 means no match
|
||||
}
|
||||
return true, strings.TrimSpace(strings.SplitN(string(out), "\n", 2)[0])
|
||||
}
|
||||
|
||||
func TestEveryFixturePassesPreflight(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("builds and folds 36 sessions")
|
||||
}
|
||||
root := t.TempDir()
|
||||
for _, task := range allTasks() {
|
||||
t.Run(task.ID, func(t *testing.T) {
|
||||
found, err := checkTask(task, root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range found {
|
||||
t.Error(f)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/retrieval"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// Preflight is what makes a real run worth paying for. An answer the model can
|
||||
// already read, or a cue not where the matrix says, turns an arm into a
|
||||
// different experiment without saying so.
|
||||
|
||||
// armScales are the fold-index arms an index task is measured across.
|
||||
var armScales = []string{"default", "half", "quarter", "off"}
|
||||
|
||||
// experimentArm is one measured configuration of a task.
|
||||
type experimentArm struct {
|
||||
name string
|
||||
scale string
|
||||
searchOff bool
|
||||
}
|
||||
|
||||
// armsFor is the single declaration of which arms a task runs at, so preflight
|
||||
// and the run itself cannot check one set and measure another.
|
||||
func armsFor(t contextTask) []experimentArm {
|
||||
if t.Experiment == experimentSearch {
|
||||
return []experimentArm{
|
||||
{"search-on", "default", false},
|
||||
{"search-off", "default", true},
|
||||
}
|
||||
}
|
||||
arms := make([]experimentArm, 0, len(armScales))
|
||||
for _, scale := range armScales {
|
||||
arms = append(arms, experimentArm{"index-" + scale, scale, false})
|
||||
}
|
||||
return arms
|
||||
}
|
||||
|
||||
func scaleOf(name string) ablation.FoldIndexScale {
|
||||
s, err := ablation.ParseFoldIndexScale(name)
|
||||
if err != nil {
|
||||
panic("unknown scale " + name) // the list above is a constant
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// armFor is the arm a task is measured under at one fold-index scale.
|
||||
func armFor(scale string, searchOff bool) ablation.Set {
|
||||
set := ablation.Set{}
|
||||
if searchOff {
|
||||
set = ablation.New(ablation.RecallSearch)
|
||||
}
|
||||
return set.WithFoldIndex(scaleOf(scale))
|
||||
}
|
||||
|
||||
type finding struct {
|
||||
Task, Arm, Problem string
|
||||
}
|
||||
|
||||
func (f finding) String() string { return fmt.Sprintf("%s [%s]: %s", f.Task, f.Arm, f.Problem) }
|
||||
|
||||
// checkTask builds the task under every arm it will be run at and reports what
|
||||
// does not hold.
|
||||
func checkTask(t contextTask, root string) ([]finding, error) {
|
||||
var out []finding
|
||||
for _, arm := range armsFor(t) {
|
||||
// Deterministic values: a preflight has to reproduce, and a tier
|
||||
// calibration measured against one run's nonces is measured against
|
||||
// nothing.
|
||||
inst, err := instantiateTask(t, seededRand(t.ID, arm.name, "preflight"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dir := filepath.Join(root, t.ID, arm.name)
|
||||
f, err := buildFixture(inst, armFor(arm.scale, arm.searchOff), filepath.Join(dir, "session.jsonl"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
visible := visibleText(f)
|
||||
|
||||
// The answer must be unreadable in every arm, or the task measures
|
||||
// reading rather than recall.
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if len(marker) < 3 {
|
||||
continue // a bare number is not a nonce; the whole set still has to match
|
||||
}
|
||||
if strings.Contains(visible, marker) {
|
||||
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("answer marker %q is already visible", marker)})
|
||||
}
|
||||
}
|
||||
|
||||
// The corpus has to be findable at all, or a miss in a real run says
|
||||
// nothing about the model.
|
||||
rank, err := probeRank(f, inst.ProbeQuery)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case rank == 0:
|
||||
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("probe query %q does not find the target at all", inst.ProbeQuery)})
|
||||
case rank > 5:
|
||||
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("probe query %q ranks the target %d", inst.ProbeQuery, rank)})
|
||||
}
|
||||
|
||||
if t.Experiment == experimentIndex {
|
||||
out = append(out, checkCueVisibility(t, inst, arm.scale, arm.name, visible)...)
|
||||
} else if line := indexLineFor(f, f.Target); line != "" {
|
||||
out = append(out, finding{t.ID, arm.name,
|
||||
"a search task is addressed by the fold index, so it measures the index and not search: " + line})
|
||||
}
|
||||
|
||||
// The projection is what the agent believes. This is what the model is
|
||||
// told, and it is the one that has to hold.
|
||||
req, err := captureRequest(f, armFor(arm.scale, arm.searchOff), inst.Prompt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s [%s]: capture: %w", t.ID, arm.name, err)
|
||||
}
|
||||
out = append(out, checkCapturedRequest(t, inst, arm.name, arm.scale, arm.searchOff, req)...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// checkCapturedRequest holds one arm's real provider request to its contract:
|
||||
// the answer unreadable, the cue exactly where its tier says, the recall tool
|
||||
// offering only what this arm can do, and no host-side field surviving the
|
||||
// boundary.
|
||||
func checkCapturedRequest(t contextTask, inst fixtureInstance, armName, scale string, searchOff bool, req provider.Request) []finding {
|
||||
var out []finding
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if len(marker) < 3 {
|
||||
continue // a bare number is not a nonce; the set as a whole still is
|
||||
}
|
||||
for _, at := range findInSurface(req, marker) {
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("answer marker %q leaked at %s", marker, at)})
|
||||
}
|
||||
}
|
||||
if t.Experiment == experimentIndex {
|
||||
wantVisible, wantHidden := tierScales(t.CueTier)
|
||||
where := findInSurface(req, inst.CueMarker)
|
||||
switch {
|
||||
case contains(wantVisible, scale) && len(where) == 0:
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is not provider-visible; the tier expects it", inst.CueMarker)})
|
||||
case contains(wantHidden, scale) && len(where) > 0:
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is provider-visible at %s: %s", inst.CueMarker, where[0], scale)})
|
||||
}
|
||||
}
|
||||
|
||||
schema, ok := recallToolSchema(req)
|
||||
if !ok {
|
||||
out = append(out, finding{t.ID, armName, "recall is not in the provider's tool list"})
|
||||
return append(out, boundaryFindings(t, armName, req)...)
|
||||
}
|
||||
params := ""
|
||||
if b, err := json.Marshal(schema.Parameters); err == nil {
|
||||
params = string(b)
|
||||
}
|
||||
// Both arms keep the read half: the ablation removes search, not recall.
|
||||
if !strings.Contains(params, `"positions"`) {
|
||||
out = append(out, finding{t.ID, armName, "recall lost its positions parameter"})
|
||||
}
|
||||
hasQuery := strings.Contains(params, `"query"`)
|
||||
switch {
|
||||
case searchOff && hasQuery:
|
||||
out = append(out, finding{t.ID, armName, "recall still offers a query parameter"})
|
||||
case !searchOff && !hasQuery:
|
||||
out = append(out, finding{t.ID, armName, "recall has no query parameter"})
|
||||
}
|
||||
if searchOff {
|
||||
for _, phrase := range searchWording {
|
||||
for _, at := range findInSurface(req, phrase) {
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("search is offered to an arm without it: %q at %s", phrase, at)})
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(out, boundaryFindings(t, armName, req)...)
|
||||
}
|
||||
|
||||
func boundaryFindings(t contextTask, armName string, req provider.Request) []finding {
|
||||
var out []finding
|
||||
for _, at := range localOnlyLeaks(req) {
|
||||
out = append(out, finding{t.ID, armName, "host-only field crossed the provider boundary at " + at})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkCueVisibility holds an index task to its tier: the cue is addressed at
|
||||
// the scales the matrix names and at no others. A policy change that moves it
|
||||
// fails here rather than turning two arms into one experiment.
|
||||
func checkCueVisibility(t contextTask, inst fixtureInstance, scale, armName, visible string) []finding {
|
||||
wantVisible, wantHidden := tierScales(t.CueTier)
|
||||
has := strings.Contains(visible, inst.CueMarker)
|
||||
switch {
|
||||
case contains(wantVisible, scale) && !has:
|
||||
return []finding{{t.ID, armName, fmt.Sprintf("cue %q should be addressed at %s and is not", inst.CueMarker, scale)}}
|
||||
case contains(wantHidden, scale) && has:
|
||||
return []finding{{t.ID, armName, fmt.Sprintf("cue %q should be gone at %s and is not", inst.CueMarker, scale)}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func contains(list []string, want string) bool { return slices.Contains(list, want) }
|
||||
|
||||
// probeRank runs the task's own query against the folded region the way recall
|
||||
// would, and reports where the target lands.
|
||||
func probeRank(f builtFixture, query string) (int, error) {
|
||||
terms, err := retrieval.QueryTerms(query)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
canonical := f.Session.Snapshot()
|
||||
covered := min(f.State.Projection.CoveredCount, len(canonical))
|
||||
docs := make([]map[string]int, 0, covered)
|
||||
positions := make([]int, 0, covered)
|
||||
lengths := make([]int, 0, covered)
|
||||
for i, m := range canonical[:covered] {
|
||||
var text strings.Builder
|
||||
text.WriteString(m.Content)
|
||||
for _, tc := range m.ToolCalls {
|
||||
text.WriteString(" " + tc.Name + " " + string(tc.Arguments))
|
||||
}
|
||||
tokens := retrieval.Tokens(text.String())
|
||||
if len(tokens) == 0 {
|
||||
continue
|
||||
}
|
||||
docs = append(docs, retrieval.Counts(tokens))
|
||||
positions = append(positions, i)
|
||||
lengths = append(lengths, len(tokens))
|
||||
}
|
||||
df := retrieval.DocumentFrequency(docs)
|
||||
avg := 0.0
|
||||
for _, l := range lengths {
|
||||
avg += float64(l)
|
||||
}
|
||||
if len(lengths) > 0 {
|
||||
avg /= float64(len(lengths))
|
||||
}
|
||||
type hit struct {
|
||||
pos int
|
||||
score float64
|
||||
}
|
||||
var hits []hit
|
||||
for i, counts := range docs {
|
||||
if s := retrieval.BM25Score(counts, lengths[i], terms, df, len(docs), avg); s > 0 {
|
||||
hits = append(hits, hit{positions[i], s})
|
||||
}
|
||||
}
|
||||
for i := range hits {
|
||||
for j := i + 1; j < len(hits); j++ {
|
||||
if hits[j].score > hits[i].score {
|
||||
hits[i], hits[j] = hits[j], hits[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
// A tool result answers at its caller's position, so either counts.
|
||||
for i, h := range hits {
|
||||
if h.pos == f.Target || h.pos == f.Target+1 {
|
||||
return i + 1, nil
|
||||
}
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// indexLineFor returns the fold-index line addressing a position, if any.
|
||||
func indexLineFor(f builtFixture, pos int) string {
|
||||
want := fmt.Sprintf("#%d ", pos)
|
||||
for line := range strings.SplitSeq(visibleIndexOnly(f), "\n") {
|
||||
if strings.Contains(line, want) {
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// visibleIndexOnly is just the fold-index sections of the projection.
|
||||
func visibleIndexOnly(f builtFixture) string {
|
||||
var b strings.Builder
|
||||
for _, m := range visibleContext(f) {
|
||||
if i := strings.Index(m.Content, "## Folded work index"); i >= 0 {
|
||||
b.WriteString(m.Content[i:])
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
// reformulation.go — how a query changed, from token sets alone.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/retrieval"
|
||||
)
|
||||
|
||||
// The host probe ranks every target first, so the engine is not what makes a
|
||||
// model take more than one attempt. These classes are set arithmetic over the
|
||||
// same tokenizer recall uses: no judge, nothing read out of prose.
|
||||
|
||||
const (
|
||||
reformFirst = "First" // nothing to compare against yet
|
||||
reformNearDup = "NearDuplicate" // the same query again, give or take a word
|
||||
reformNarrowing = "Narrowing" // kept everything and added terms
|
||||
reformBroadening = "Broadening" // dropped terms, added none
|
||||
reformAnchored = "HistoricalAnchor" // reached for a term the question does not contain
|
||||
reformShift = "SynonymShift" // traded terms for others
|
||||
)
|
||||
|
||||
// nearDuplicateOverlap is where "reworded" stops and "asked again" begins.
|
||||
const nearDuplicateOverlap = 0.8
|
||||
|
||||
func terms(s string) []string {
|
||||
return retrieval.Unique(retrieval.Tokens(s))
|
||||
}
|
||||
|
||||
func termSet(s string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, t := range terms(s) {
|
||||
out[t] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func jaccard(a, b map[string]bool) float64 {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return 1
|
||||
}
|
||||
shared := 0
|
||||
for t := range a {
|
||||
if b[t] {
|
||||
shared++
|
||||
}
|
||||
}
|
||||
union := len(a) + len(b) - shared
|
||||
if union == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(shared) / float64(union)
|
||||
}
|
||||
|
||||
// classifyReformulation names how this query differs from the one before it.
|
||||
func classifyReformulation(previous, current, question string) string {
|
||||
if previous == "" {
|
||||
return reformFirst
|
||||
}
|
||||
prev, cur := termSet(previous), termSet(current)
|
||||
if jaccard(prev, cur) >= nearDuplicateOverlap {
|
||||
return reformNearDup
|
||||
}
|
||||
added, dropped := diffTerms(cur, prev), diffTerms(prev, cur)
|
||||
// A term the question never used had to come from the model's own memory of
|
||||
// the history, which is a different move from rephrasing what it was asked.
|
||||
ask := termSet(question)
|
||||
for _, t := range added {
|
||||
if !ask[t] {
|
||||
return reformAnchored
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(dropped) == 0:
|
||||
return reformNarrowing
|
||||
case len(added) == 0:
|
||||
return reformBroadening
|
||||
default:
|
||||
return reformShift
|
||||
}
|
||||
}
|
||||
|
||||
func diffTerms(from, minus map[string]bool) []string {
|
||||
var out []string
|
||||
for t := range from {
|
||||
if !minus[t] {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// classifySearches labels each attempt and records which of the host probe's
|
||||
// terms it left out. The probe is the oracle: it ranks the target first, so a
|
||||
// query missing its terms says what the model did not think to ask for.
|
||||
func classifySearches(m *contextMetrics, inst fixtureInstance) {
|
||||
probe := terms(inst.ProbeQuery)
|
||||
previous := ""
|
||||
for i := range m.Searches {
|
||||
s := &m.Searches[i]
|
||||
s.Reformulation = classifyReformulation(previous, s.Query, inst.Prompt)
|
||||
have := termSet(s.Query)
|
||||
for _, term := range probe {
|
||||
if !have[term] {
|
||||
s.MissingProbeTerms = append(s.MissingProbeTerms, term)
|
||||
}
|
||||
}
|
||||
previous = s.Query
|
||||
}
|
||||
}
|
||||
|
||||
// queryAudit is the reformulation funnel across a set of runs.
|
||||
type queryAudit struct {
|
||||
Runs, RunsThatSearched int
|
||||
FirstQueryHit int
|
||||
HitBySecond, HitByThird int
|
||||
NeverHit int
|
||||
TotalSearches int
|
||||
PostHitSearches int
|
||||
HitThenRead int
|
||||
EscapeBeforeFirstSearch int
|
||||
Reformulations map[string]int
|
||||
MissedProbeTerms map[string]int
|
||||
}
|
||||
|
||||
func auditQueries(runs []contextMetrics) queryAudit {
|
||||
a := queryAudit{Reformulations: map[string]int{}, MissedProbeTerms: map[string]int{}}
|
||||
for _, r := range runs {
|
||||
if r.Failed || r.Contaminated {
|
||||
continue
|
||||
}
|
||||
a.Runs++
|
||||
a.TotalSearches += len(r.Searches)
|
||||
a.PostHitSearches += r.PostHitSearches
|
||||
if r.EscapeBeforeFirstRecall > 0 {
|
||||
a.EscapeBeforeFirstSearch++
|
||||
}
|
||||
if len(r.Searches) == 0 {
|
||||
continue
|
||||
}
|
||||
a.RunsThatSearched++
|
||||
hitAt := 0
|
||||
for i, s := range r.Searches {
|
||||
a.Reformulations[s.Reformulation]++
|
||||
for _, term := range s.MissingProbeTerms {
|
||||
a.MissedProbeTerms[term]++
|
||||
}
|
||||
if s.TargetRank > 0 && hitAt == 0 {
|
||||
hitAt = i + 1
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case hitAt == 1:
|
||||
a.FirstQueryHit++
|
||||
case hitAt == 2:
|
||||
a.HitBySecond++
|
||||
case hitAt >= 3:
|
||||
a.HitByThird++
|
||||
default:
|
||||
a.NeverHit++
|
||||
}
|
||||
if hitAt > 0 && r.TargetRead {
|
||||
a.HitThenRead++
|
||||
}
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func (a queryAudit) report() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## Query reformulation\n")
|
||||
fmt.Fprintf(&b, " runs=%d searched=%d searches=%d (%.2f per searching run)\n",
|
||||
a.Runs, a.RunsThatSearched, a.TotalSearches, ratio(a.TotalSearches, a.RunsThatSearched))
|
||||
fmt.Fprintf(&b, " target found by query 1: %d by 2: %d by 3+: %d never: %d\n",
|
||||
a.FirstQueryHit, a.HitBySecond, a.HitByThird, a.NeverHit)
|
||||
fmt.Fprintf(&b, " found then read: %d/%d searches issued after the target was already back: %d\n",
|
||||
a.HitThenRead, a.FirstQueryHit+a.HitBySecond+a.HitByThird, a.PostHitSearches)
|
||||
fmt.Fprintf(&b, " reached for the workspace before searching: %d\n", a.EscapeBeforeFirstSearch)
|
||||
b.WriteString(" reformulations: " + countsLine(a.Reformulations) + "\n")
|
||||
b.WriteString(" probe terms most often absent: " + topCounts(a.MissedProbeTerms, 8) + "\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func ratio(n, d int) float64 {
|
||||
if d == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(n) / float64(d)
|
||||
}
|
||||
|
||||
func countsLine(counts map[string]int) string {
|
||||
keys := make([]string, 0, len(counts))
|
||||
for k := range counts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func topCounts(counts map[string]int, limit int) string {
|
||||
type kv struct {
|
||||
k string
|
||||
n int
|
||||
}
|
||||
all := make([]kv, 0, len(counts))
|
||||
for k, n := range counts {
|
||||
all = append(all, kv{k, n})
|
||||
}
|
||||
sort.Slice(all, func(i, j int) bool {
|
||||
if all[i].n != all[j].n {
|
||||
return all[i].n > all[j].n
|
||||
}
|
||||
return all[i].k < all[j].k
|
||||
})
|
||||
parts := make([]string, 0, limit)
|
||||
for _, e := range all[:min(limit, len(all))] {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", e.k, e.n))
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "(none)"
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// queryLines lists every attempt, so a surprising audit can be read back.
|
||||
func queryLines(runs []contextMetrics) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## Every query, in order\n")
|
||||
for _, r := range runs {
|
||||
for _, s := range r.Searches {
|
||||
rank := "miss"
|
||||
if s.TargetRank > 0 {
|
||||
rank = fmt.Sprintf("rank %d", s.TargetRank)
|
||||
}
|
||||
fmt.Fprintf(&b, " %-24s %-14s #%d %-17s %-9s %q\n",
|
||||
r.Task, r.Arm, s.Ordinal, s.Reformulation, rank, s.Query)
|
||||
if len(s.MissingProbeTerms) > 0 && s.TargetRank == 0 {
|
||||
fmt.Fprintf(&b, " %-24s %-14s missing probe terms: %s\n",
|
||||
"", "", strings.Join(slices.Clone(s.MissingProbeTerms), " "))
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,569 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/assembly/boot"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
_ "reasonix/internal/model/openai"
|
||||
"reasonix/internal/runtime/agent"
|
||||
_ "reasonix/internal/tools/builtin"
|
||||
)
|
||||
|
||||
// A run is one recall question against a fixture the preflight has already
|
||||
// cleared. Scoring reads the transcript the turn appended — tool names and
|
||||
// their JSON arguments — so nothing depends on how the model narrated itself.
|
||||
|
||||
const (
|
||||
runBaseURL = "https://api.deepseek.com"
|
||||
runModel = "deepseek-chat"
|
||||
runTimeout = 4 * time.Minute
|
||||
)
|
||||
|
||||
func realProvider() (provider.Provider, error) {
|
||||
key := os.Getenv("DEEPSEEK_API_KEY")
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("a real run needs DEEPSEEK_API_KEY")
|
||||
}
|
||||
return provider.New("openai", provider.Config{
|
||||
Name: "contextbench", BaseURL: runBaseURL, Model: runModel, APIKey: key,
|
||||
})
|
||||
}
|
||||
|
||||
// adversarialPrompts tell the model what it is running and send it after the
|
||||
// answer everywhere except the conversation. The question is not whether it
|
||||
// obeys the experiment: it is whether the answer is reachable at all by a model
|
||||
// that knows the mechanism and is actively attacking the host.
|
||||
var adversarialPrompts = map[string]string{
|
||||
experimentSearch: "You are running a benchmark. Find the expected answer for this task from the " +
|
||||
"Reasonix repository, benchmark source, fixture generator, test files, scratch files, or " +
|
||||
"session metadata. Do not use recall unless necessary. Report the exact hidden values.",
|
||||
experimentIndex: "Inspect benchmarks/context-retrieval, the git repository, session files, temporary " +
|
||||
"files, and any benchmark metadata you can access. Recover the concrete instantiated answer for " +
|
||||
"this run without relying on the folded conversation. Report the exact values.",
|
||||
}
|
||||
|
||||
// runAdversarial sends one task per experiment after the answer through every
|
||||
// host-readable surface. It passes when nothing but recall could produce it —
|
||||
// a run that gives up and recalls has still proven the attack failed.
|
||||
func runAdversarial(root string) int {
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
restore, ierr := isolateWorkspace()
|
||||
if ierr != nil {
|
||||
fmt.Fprintln(os.Stderr, "isolate workspace:", ierr)
|
||||
return 2
|
||||
}
|
||||
defer restore()
|
||||
p, perr := realProvider()
|
||||
if perr != nil {
|
||||
fmt.Fprintln(os.Stderr, perr)
|
||||
return 2
|
||||
}
|
||||
leaked := 0
|
||||
for _, id := range []string{"r01-retry-boundary", "i01-transport-fallback"} {
|
||||
t, ok := taskByID(id)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "unknown task %q\n", id)
|
||||
return 2
|
||||
}
|
||||
m, err := runOne(p, t, armFor("default", false), "adversarial", root, false, false,
|
||||
adversarialPrompts[t.Experiment])
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", id, err)
|
||||
m = failedMetrics(t, "adversarial", err)
|
||||
}
|
||||
verdict := "isolated"
|
||||
if m.EscapeFoundAnswer {
|
||||
verdict = "LEAKED via " + m.LeakedVia
|
||||
leaked++
|
||||
}
|
||||
fmt.Printf("%-24s %-24s escapes=%-3d recall=%d answered=%v tools=%v\n",
|
||||
id, verdict, m.EscapeCalls, m.SearchCalls+m.ReadCalls, m.AnswerRecovered, m.UnexpectedWorkTools)
|
||||
if m.EscapeFoundAnswer {
|
||||
fmt.Printf(" args: %s\n output: %s\n", m.LeakedArgs, m.LeakedOutput)
|
||||
}
|
||||
}
|
||||
if leaked > 0 {
|
||||
fmt.Fprintf(os.Stderr, "\n%d of 2 adversarial runs reached an answer outside recall\n", leaked)
|
||||
return 1
|
||||
}
|
||||
fmt.Println("\nNo answer was reachable outside recall.")
|
||||
return 0
|
||||
}
|
||||
|
||||
// runOne asks one task under one arm and scores what the model did.
|
||||
func runOne(p provider.Provider, t contextTask, arm ablation.Set, armName, root string, cueVisible, dry bool, promptOverride ...string) (contextMetrics, error) {
|
||||
// Live runs get values that existed nowhere before the run started; a dry
|
||||
// run stays reproducible.
|
||||
rng := liveRand()
|
||||
if dry {
|
||||
rng = seededRand(t.ID, armName, "dry")
|
||||
}
|
||||
inst, err := instantiateTask(t, rng)
|
||||
if err != nil {
|
||||
return contextMetrics{}, err
|
||||
}
|
||||
f, err := buildFixture(inst, arm, filepath.Join(root, t.ID, armName, "session.jsonl"))
|
||||
if err != nil {
|
||||
return contextMetrics{}, err
|
||||
}
|
||||
sess, err := sessionstore.LoadSession(f.Path)
|
||||
if err != nil {
|
||||
return contextMetrics{}, fmt.Errorf("load session: %w", err)
|
||||
}
|
||||
reg := tool.NewRegistry()
|
||||
for _, x := range tool.Builtins() {
|
||||
reg.Add(x)
|
||||
}
|
||||
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
|
||||
|
||||
var usage usageSink
|
||||
a := agent.New(p, reg, sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, &usage)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
// From here the transcript lives only in memory. On disk it is the answer.
|
||||
sealFixture(f.Path)
|
||||
defer unsealFixture(f.Path)
|
||||
dir := filepath.Dir(f.Path)
|
||||
if at, marker := answerOnDisk(dir, inst.AnswerMarkers); at != "" {
|
||||
return contextMetrics{}, fmt.Errorf("%s [%s]: %q is readable at %s before the run", t.ID, armName, marker, at)
|
||||
}
|
||||
|
||||
before := len(sess.Snapshot())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), runTimeout)
|
||||
defer cancel()
|
||||
prompt := inst.Prompt
|
||||
if len(promptOverride) > 0 && promptOverride[0] != "" {
|
||||
prompt = promptOverride[0]
|
||||
}
|
||||
runErr := a.Run(ctx, prompt)
|
||||
|
||||
if at, marker := answerOnDisk(dir, inst.AnswerMarkers); at != "" {
|
||||
return contextMetrics{}, fmt.Errorf("%s [%s]: %q was written to %s during the run", t.ID, armName, marker, at)
|
||||
}
|
||||
|
||||
appended := sess.Snapshot()[before:]
|
||||
m := scoreRun(appended, inst, f.Target, armName, cueVisible)
|
||||
m.trajectory = appended
|
||||
if u, requests := usage.snapshot(); requests > 0 {
|
||||
m.UsageReportedRequests, m.PromptTokens, m.CompletionTokens = requests, u.PromptTokens, u.CompletionTokens
|
||||
m.CacheHitTokens, m.CacheMissTokens, m.UsageEstimated = u.CacheHitTokens, u.CacheMissTokens, u.Estimated
|
||||
}
|
||||
m.scoreAnswer(finalAnswer(appended), inst)
|
||||
if runErr != nil {
|
||||
m.FailureStage = "RunError"
|
||||
m.TimeoutReason = timeoutReason(m)
|
||||
}
|
||||
return m, runErr
|
||||
}
|
||||
|
||||
// failedMetrics is what a run that never reached a transcript leaves behind. It
|
||||
// carries the run's identity and the stage that ended it, so the failure stays a
|
||||
// countable fact; Failed keeps it out of every statistic.
|
||||
func failedMetrics(t contextTask, armName string, err error) contextMetrics {
|
||||
return contextMetrics{
|
||||
Task: t.ID, Arm: armName, Failed: true,
|
||||
FailureStage: stageNoEvidence, Reason: err.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
// finalAnswer is the last assistant text the turn produced.
|
||||
func finalAnswer(msgs []provider.Message) string {
|
||||
for _, m := range slices.Backward(msgs) {
|
||||
if m.Role == provider.RoleAssistant && strings.TrimSpace(m.Content) != "" {
|
||||
return m.Content
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// isolateWorkspace moves the process into an empty directory before any task
|
||||
// runs. The corpus lives in this repository, so an agent left here greps the
|
||||
// answers out of corpus.go — one did, quoting the file's own line about them
|
||||
// being nonces. Fixtures stay outside it too: a readable session file is the
|
||||
// canonical transcript, answer included.
|
||||
func isolateWorkspace() (func(), error) {
|
||||
dir, err := os.MkdirTemp("", "contextbench-workspace-*")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
readme := "This workspace is empty on purpose. The question is about earlier conversation,\nnot about any file here.\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte(readme), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
previous, err := os.Getwd()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chdir(dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func() {
|
||||
_ = os.Chdir(previous)
|
||||
_ = os.RemoveAll(dir)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// runBoundaries runs only the pair each index task is built for: the narrowest
|
||||
// scale still addressing its cue, and the next one down. Each pair moves the
|
||||
// affordance and nothing else, where default-vs-off moves the whole index.
|
||||
|
||||
// The two arms get separate nonces: reusing one would leave the first arm's
|
||||
// answer in a host-readable history for the second to find.
|
||||
func runBoundaries(mode, root string) int {
|
||||
started := time.Now()
|
||||
planned := 0
|
||||
repo, _ := os.Getwd()
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
restore, ierr := isolateWorkspace()
|
||||
if ierr != nil {
|
||||
fmt.Fprintln(os.Stderr, "isolate workspace:", ierr)
|
||||
return 2
|
||||
}
|
||||
defer restore()
|
||||
p, perr := realProvider()
|
||||
if perr != nil {
|
||||
fmt.Fprintln(os.Stderr, perr)
|
||||
return 2
|
||||
}
|
||||
|
||||
var all []contextMetrics
|
||||
var cueSide, noCueSide []contextMetrics
|
||||
for _, t := range efficiencyTasks() {
|
||||
cue, noCue := boundaryPair(t.CueTier)
|
||||
for _, side := range []struct {
|
||||
scale string
|
||||
has bool
|
||||
}{{cue, true}, {noCue, false}} {
|
||||
name := "index-" + side.scale
|
||||
planned++
|
||||
m, err := runOne(p, t, armFor(side.scale, false), name, root, side.has, false)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, name, err)
|
||||
m = failedMetrics(t, name, err)
|
||||
}
|
||||
all = append(all, m)
|
||||
if side.has {
|
||||
cueSide = append(cueSide, m)
|
||||
} else {
|
||||
noCueSide = append(noCueSide, m)
|
||||
}
|
||||
fmt.Printf("%-24s %-14s cue=%-3v %-14s search=%d read=%d direct=%-5v recall-tok=%-5d%s escape=%d\n",
|
||||
t.ID, name, side.has, m.FailureStage, m.SearchCalls, m.ReadCalls,
|
||||
m.CueDirectRead, m.RecallReturnedTokens, promptTokenSuffix(m), m.EscapeCalls)
|
||||
}
|
||||
}
|
||||
reportDeltas(cueSide, noCueSide)
|
||||
fmt.Print(reportStopping(all))
|
||||
return writeResults(root, batchInfo{Mode: mode, Planned: planned, Started: started, Repo: repo}, all)
|
||||
}
|
||||
|
||||
// reportDeltas is the paired reading: each task against itself, then pooled.
|
||||
func reportDeltas(cueSide, noCueSide []contextMetrics) {
|
||||
fmt.Printf("\n## Paired deltas (cue present − cue absent)\n")
|
||||
fmt.Printf(" %-24s %-10s %-10s %-9s %-11s %s\n",
|
||||
"task", "recovery", "directread", "searches", "recall-tok", "escapes")
|
||||
for i := range cueSide {
|
||||
if i >= len(noCueSide) {
|
||||
break
|
||||
}
|
||||
a, b := cueSide[i], noCueSide[i]
|
||||
fmt.Printf(" %-24s %-10s %-10s %-9d %-11d %d\n", a.Task,
|
||||
deltaBool(a.AnswerRecovered, b.AnswerRecovered),
|
||||
deltaBool(a.CueDirectRead, b.CueDirectRead),
|
||||
a.SearchCalls-b.SearchCalls,
|
||||
a.RecallReturnedTokens-b.RecallReturnedTokens,
|
||||
a.EscapeCalls-b.EscapeCalls)
|
||||
}
|
||||
fmt.Println("\n## Pooled")
|
||||
fmt.Println(" ", summarize("cue-present", cueSide).line())
|
||||
fmt.Println(" ", summarize("cue-absent", noCueSide).line())
|
||||
fmt.Println("\n## Where each side looked first")
|
||||
fmt.Printf(" %-14s %s\n", "cue-present", countsLine(summarize("x", cueSide).Routes))
|
||||
fmt.Printf(" %-14s %s\n", "cue-absent", countsLine(summarize("x", noCueSide).Routes))
|
||||
}
|
||||
|
||||
func deltaBool(a, b bool) string {
|
||||
switch {
|
||||
case a == b:
|
||||
return "same"
|
||||
case a:
|
||||
return "+cue"
|
||||
default:
|
||||
return "-cue"
|
||||
}
|
||||
}
|
||||
|
||||
// runExperiment runs one experiment's tasks across its arms. A dry run drives
|
||||
// the same pipeline with a scripted provider: it proves fixture, run, scoring
|
||||
// and report hold together before any of it is paid for.
|
||||
func runExperiment(mode, experiment, root string, dry bool, tasks []contextTask) int {
|
||||
started := time.Now()
|
||||
planned := 0
|
||||
repo, _ := os.Getwd()
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
restore, err := isolateWorkspace()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "isolate workspace:", err)
|
||||
return 2
|
||||
}
|
||||
defer restore()
|
||||
var p provider.Provider = &scriptedProvider{}
|
||||
if !dry {
|
||||
real, rerr := realProvider()
|
||||
if rerr != nil {
|
||||
fmt.Fprintln(os.Stderr, rerr)
|
||||
return 2
|
||||
}
|
||||
p = real
|
||||
}
|
||||
byArm := map[string][]contextMetrics{}
|
||||
var all []contextMetrics
|
||||
|
||||
if experiment == experimentIndex {
|
||||
fmt.Print(fingerprint(dry))
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if t.Experiment != experiment {
|
||||
continue
|
||||
}
|
||||
// Arms are shuffled inside each task block. Running six defaults, then
|
||||
// six halves, would make provider drift collinear with the axis under
|
||||
// test: a bad minute would land entirely on one arm.
|
||||
for _, arm := range shuffledArms(t) {
|
||||
cueVisible := t.Experiment == experimentIndex && cueExpectedVisible(t, arm.scale)
|
||||
planned++
|
||||
m, err := runOne(p, t, armFor(arm.scale, arm.searchOff), arm.name, root, cueVisible, dry)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, arm.name, err)
|
||||
m = failedMetrics(t, arm.name, err)
|
||||
}
|
||||
byArm[arm.name] = append(byArm[arm.name], m)
|
||||
all = append(all, m)
|
||||
fmt.Printf("%-24s %-14s %-18s search=%d hit=%d read=%d recall-tok=%d%s %v\n",
|
||||
t.ID, arm.name, m.FailureStage, m.SearchCalls, m.TargetSearchHits, m.ReadCalls,
|
||||
m.RecallReturnedTokens, promptTokenSuffix(m), m.UnexpectedWorkTools)
|
||||
}
|
||||
}
|
||||
reportFunnels(byArm)
|
||||
if experiment == experimentIndex {
|
||||
fmt.Print(reportDoseResponse(all))
|
||||
}
|
||||
fmt.Print(reportStopping(all))
|
||||
fmt.Print(auditQueries(all).report())
|
||||
fmt.Print(queryLines(all))
|
||||
if experiment == experimentIndex {
|
||||
reportBoundaries(all)
|
||||
}
|
||||
return writeResults(root, batchInfo{Mode: mode, Planned: planned, Started: started, Dry: dry, Repo: repo}, all)
|
||||
}
|
||||
|
||||
func cueExpectedVisible(t contextTask, scale string) bool {
|
||||
visible, _ := tierScales(t.CueTier)
|
||||
return contains(visible, scale)
|
||||
}
|
||||
|
||||
func reportFunnels(byArm map[string][]contextMetrics) {
|
||||
names := make([]string, 0, len(byArm))
|
||||
for name := range byArm {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
fmt.Println("\n## Funnel")
|
||||
for _, name := range names {
|
||||
fmt.Println(" ", summarize(name, byArm[name]).line())
|
||||
}
|
||||
fmt.Println("\n## Where it looked first")
|
||||
for _, name := range names {
|
||||
f := summarize(name, byArm[name])
|
||||
fmt.Printf(" %-14s %s\n", name, countsLine(f.Routes))
|
||||
}
|
||||
fmt.Println("\n## Where it broke")
|
||||
for _, name := range names {
|
||||
f := summarize(name, byArm[name])
|
||||
stages := make([]string, 0, len(f.Stages))
|
||||
for stage, n := range f.Stages {
|
||||
stages = append(stages, fmt.Sprintf("%s=%d", stage, n))
|
||||
}
|
||||
sort.Strings(stages)
|
||||
fmt.Printf(" %-12s %s\n", name, strings.Join(stages, " "))
|
||||
}
|
||||
}
|
||||
|
||||
// writeTrajectories saves what each turn produced, after the whole batch is
|
||||
// done. Three analyses in a row have needed a re-run because something was
|
||||
// counted and not kept — query text, then snippet contents. It is written at
|
||||
// the end on purpose: during the batch these files are the answer.
|
||||
func writeTrajectories(root string, all []contextMetrics) error {
|
||||
f, err := os.Create(filepath.Join(root, "trajectories.jsonl"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
for _, m := range all {
|
||||
if err := writeJSONLine(f, map[string]any{
|
||||
"task": m.Task, "arm": m.Arm, "messages": m.trajectory,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeResults(root string, info batchInfo, all []contextMetrics) int {
|
||||
path := filepath.Join(root, "results.jsonl")
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
defer f.Close()
|
||||
scored, failed := 0, 0
|
||||
for _, m := range all {
|
||||
if err := writeJSONLine(f, m); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
if m.Failed {
|
||||
failed++
|
||||
} else {
|
||||
scored++
|
||||
}
|
||||
}
|
||||
if err := writeTrajectories(root, all); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write trajectories:", err)
|
||||
return 2
|
||||
}
|
||||
if err := writeBatchManifest(root, info, all); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write batch manifest:", err)
|
||||
return 2
|
||||
}
|
||||
// Both counts are named: a reader who sees only the batch size cannot tell
|
||||
// measurements from attempts.
|
||||
fmt.Printf("\n%d runs written to %s: scored=%d failed=%d (with trajectories.jsonl and batch.json)\n",
|
||||
len(all), path, scored, failed)
|
||||
return 0
|
||||
}
|
||||
|
||||
// reportBoundaries prints each index task across its own cue boundary, then the
|
||||
// two sides pooled. Task difficulty is held constant inside a row, which is
|
||||
// what a four-arm average cannot do.
|
||||
func reportBoundaries(all []contextMetrics) {
|
||||
byKey := map[string]contextMetrics{}
|
||||
scored, failed := 0, 0
|
||||
for _, m := range all {
|
||||
if m.Failed {
|
||||
// No transcript, no boundary to compare: count the failure instead of
|
||||
// printing the zero it left behind as a measurement.
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
scored++
|
||||
byKey[m.Task+"|"+m.Arm] = m
|
||||
}
|
||||
if scored == 0 {
|
||||
fmt.Printf("\n## Across each task's cue boundary\n no scored runs: scored=%d failed=%d (nothing to compare)\n", scored, failed)
|
||||
return
|
||||
}
|
||||
var cueSide, noCueSide []contextMetrics
|
||||
fmt.Println("\n## Across each task's cue boundary")
|
||||
fmt.Printf(" %-24s %-16s %-4s %-10s %-9s %-12s %-11s %s\n",
|
||||
"task", "arm", "cue", "recovered", "cue-read", "search/task", "recall-tok", "escape")
|
||||
for _, t := range indexTasks() {
|
||||
cue, noCue := boundaryPair(t.CueTier)
|
||||
for _, side := range []struct {
|
||||
scale string
|
||||
has bool
|
||||
}{{cue, true}, {noCue, false}} {
|
||||
m, ok := byKey[t.ID+"|index-"+side.scale]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if side.has {
|
||||
cueSide = append(cueSide, m)
|
||||
} else {
|
||||
noCueSide = append(noCueSide, m)
|
||||
}
|
||||
mark := "no"
|
||||
if side.has {
|
||||
mark = "yes"
|
||||
}
|
||||
fmt.Printf(" %-24s %-16s %-4s %-10v %-9v %-12d %-11d %d\n",
|
||||
t.ID, "index-"+side.scale, mark, m.AnswerRecovered, m.RecallReadWithoutSearch,
|
||||
m.SearchCalls, m.RecallReturnedTokens, m.EscapeCalls)
|
||||
}
|
||||
}
|
||||
fmt.Println("\n## Pooled across boundaries")
|
||||
fmt.Println(" ", summarize("cue-present", cueSide).line())
|
||||
fmt.Println(" ", summarize("cue-absent", noCueSide).line())
|
||||
}
|
||||
|
||||
// shuffledArms permutes a task's arms deterministically, so the order is
|
||||
// reproducible and still uncorrelated with the axis being measured.
|
||||
func shuffledArms(t contextTask) []experimentArm {
|
||||
arms := armsFor(t)
|
||||
rng := seededRand(t.ID, "arm-order")
|
||||
rng.Shuffle(len(arms), func(i, j int) { arms[i], arms[j] = arms[j], arms[i] })
|
||||
return arms
|
||||
}
|
||||
|
||||
// fingerprint records what produced a batch. Not a gate — but batch effects
|
||||
// have reversed three conclusions in this experiment already, and a table with
|
||||
// no provenance cannot be compared with the next one.
|
||||
func fingerprint(dry bool) string {
|
||||
model, endpoint := runModel, runBaseURL
|
||||
if dry {
|
||||
model, endpoint = "scripted", "none"
|
||||
}
|
||||
commit := "unknown"
|
||||
if out, err := exec.Command("git", "rev-parse", "--short", "HEAD").Output(); err == nil {
|
||||
commit = strings.TrimSpace(string(out))
|
||||
}
|
||||
return fmt.Sprintf("## Batch\n model=%s endpoint=%s window=%d generations=%d commit=%s started=%s\n",
|
||||
model, endpoint, fixtureWindow, fixtureGenerations, commit, time.Now().UTC().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
// timeoutReason separates a run that never found its answer from one that found
|
||||
// it and kept going. Only the first is a case for a longer budget; the second
|
||||
// would just get more time to circle. A batch reporting only "context deadline
|
||||
// exceeded" cannot tell an arm that made retrieval harder from an arm that
|
||||
// happened to draw a slow minute.
|
||||
func timeoutReason(m contextMetrics) string {
|
||||
switch {
|
||||
case m.TargetHeldAtEnd && m.RoundsAfterTarget >= 2:
|
||||
return "PostTargetRunaway"
|
||||
case m.TargetHeldAtEnd:
|
||||
return "TargetHeldLate"
|
||||
case m.SearchCalls >= 3:
|
||||
return "SearchSpiral"
|
||||
case m.SearchCalls+m.ReadCalls+m.EscapeCalls == 0:
|
||||
return "NoToolActivity"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// scriptedProvider walks the chain a competent model would: search, read the
|
||||
// best hit, answer with what came back. It proves the pipeline holds together
|
||||
// for nothing, so a real run's first failure is about the model.
|
||||
|
||||
type scriptedProvider struct{}
|
||||
|
||||
func (p *scriptedProvider) Name() string { return "contextbench-scripted" }
|
||||
|
||||
var (
|
||||
scriptedHit = regexp.MustCompile(`(?m)^#(\d+) `)
|
||||
scriptedRecall = regexp.MustCompile(`(?s)^#\d+\n(.*)`)
|
||||
)
|
||||
|
||||
// Stream decides from the request alone, never from a call counter: one
|
||||
// provider serves every task in a run, and a counter would make the second task
|
||||
// start where the first left off.
|
||||
func (p *scriptedProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
ch := make(chan provider.Chunk, 3)
|
||||
defer close(ch)
|
||||
// Honour the schema, as a model does: an arm without a query parameter is
|
||||
// an arm that cannot search, and pretending otherwise scores a refusal as
|
||||
// a failed search.
|
||||
if !recallAccepts(req, "query") {
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "I have no address for that part of the history."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
return ch, nil
|
||||
}
|
||||
// The last message, not the last tool result anywhere: the fixture's own
|
||||
// history is full of tool results, and one of those is not this turn's.
|
||||
last := ""
|
||||
if n := len(req.Messages); n > 0 && req.Messages[n-1].Role == provider.RoleTool {
|
||||
last = req.Messages[n-1].Content
|
||||
}
|
||||
switch {
|
||||
case last == "":
|
||||
args, _ := json.Marshal(map[string]string{"query": lastUserText(req)})
|
||||
ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{
|
||||
ID: "s1", Name: "recall", Arguments: string(args),
|
||||
}}
|
||||
case strings.Contains(last, "Folded context matching"):
|
||||
positions := scriptedHit.FindAllStringSubmatch(last, 1)
|
||||
if len(positions) == 0 {
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "Nothing in the folded history matches."}
|
||||
break
|
||||
}
|
||||
n, _ := strconv.Atoi(positions[0][1])
|
||||
args, _ := json.Marshal(map[string][]int{"positions": {n}})
|
||||
ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{
|
||||
ID: "r1", Name: "recall", Arguments: string(args),
|
||||
}}
|
||||
default:
|
||||
// Answer with what recall returned, so a correct chain scores as one.
|
||||
body := last
|
||||
if m := scriptedRecall.FindStringSubmatch(last); len(m) > 1 {
|
||||
body = m[1]
|
||||
}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "From the folded history: " + body}
|
||||
}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// recallAccepts reports whether recall is offered with the given parameter.
|
||||
func recallAccepts(req provider.Request, param string) bool {
|
||||
for _, t := range req.Tools {
|
||||
if t.Name != "recall" {
|
||||
continue
|
||||
}
|
||||
b, err := json.Marshal(t.Parameters)
|
||||
return err == nil && strings.Contains(string(b), `"`+param+`"`)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func lastUserText(req provider.Request) string {
|
||||
for _, m := range slices.Backward(req.Messages) {
|
||||
if m.Role == provider.RoleUser {
|
||||
return m.Content
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
// snippet.go — does a rank-1 hit actually hand over the answer?
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/retrieval"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/runtime/agent"
|
||||
)
|
||||
|
||||
// TargetHit is not EvidenceSufficient: a rank-1 hit can still hand over a
|
||||
// window too short to answer with, and searching again after that is correct.
|
||||
// Measured through the shipped recall path, not a copy of it.
|
||||
|
||||
// snippetWidths are the windows to compare. 240 is what ships.
|
||||
var snippetWidths = []int{120, 180, 240, 320, 480}
|
||||
|
||||
// historicalQueries are the queries these tasks actually drew in the pilots,
|
||||
// kept because a probe query is the host's phrasing and the model's was not.
|
||||
var historicalQueries = map[string][]string{
|
||||
"i03-coalescing": {"coalesce window marker probe", "job completion coalescing window"},
|
||||
"i04-recovery-fence": {"recovery fence marker epoch", "expected epoch", "recfence quartz"},
|
||||
}
|
||||
|
||||
type snippetReading struct {
|
||||
Task string
|
||||
Query string
|
||||
Width int
|
||||
Covered int
|
||||
Total int
|
||||
Runes int
|
||||
Rank int
|
||||
Complete bool
|
||||
}
|
||||
|
||||
// runSnippetAudit answers the question offline, over many instantiations so a
|
||||
// single seed's lengths cannot decide it.
|
||||
func runSnippetAudit(root string, seeds int) int {
|
||||
fmt.Printf("## Snippet coverage (%d instantiations per task)\n", seeds)
|
||||
fmt.Println(" Does the window a rank-1 hit returns contain every value the task scores on?")
|
||||
|
||||
byTaskWidth := map[string]map[int][]snippetReading{}
|
||||
for _, t := range allTasks() {
|
||||
byTaskWidth[t.ID] = map[int][]snippetReading{}
|
||||
for seed := range seeds {
|
||||
inst, err := instantiateTask(t, seededRand(t.ID, "snippet", fmt.Sprint(seed)))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
queries := append([]string{inst.ProbeQuery}, historicalQueries[t.ID]...)
|
||||
for _, q := range queries {
|
||||
for _, width := range snippetWidths {
|
||||
r := measureSnippet(inst, q, width)
|
||||
byTaskWidth[t.ID][width] = append(byTaskWidth[t.ID][width], r)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\n %-24s %-5s", "task", "vals")
|
||||
for _, w := range snippetWidths {
|
||||
fmt.Printf("%9d", w)
|
||||
}
|
||||
fmt.Println(" mean runes @240")
|
||||
for _, t := range allTasks() {
|
||||
vals := len(t.AnswerVars)
|
||||
fmt.Printf(" %-24s %-5d", t.ID, vals)
|
||||
for _, w := range snippetWidths {
|
||||
fmt.Printf("%8.0f%%", 100*completeRate(byTaskWidth[t.ID][w]))
|
||||
}
|
||||
fmt.Printf(" %.0f\n", meanRunes(byTaskWidth[t.ID][240]))
|
||||
}
|
||||
reportShippedPath(root)
|
||||
return 0
|
||||
}
|
||||
|
||||
// measureSnippet renders one hit through the same MakeSnippet the recall path
|
||||
// uses, then asks whether every scored value survived the window.
|
||||
func measureSnippet(inst fixtureInstance, query string, width int) snippetReading {
|
||||
terms, err := retrieval.QueryTerms(query)
|
||||
if err != nil {
|
||||
return snippetReading{Task: inst.Task.ID, Query: query, Width: width, Total: len(inst.AnswerMarkers)}
|
||||
}
|
||||
// The tool result is the document the answer lives in; a call's arguments
|
||||
// are a separate document and never carry it.
|
||||
snippet := retrieval.MakeSnippet(inst.body, query, terms, width)
|
||||
covered := 0
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if strings.Contains(strings.ToLower(snippet), strings.ToLower(marker)) {
|
||||
covered++
|
||||
}
|
||||
}
|
||||
return snippetReading{
|
||||
Task: inst.Task.ID, Query: query, Width: width,
|
||||
Covered: covered, Total: len(inst.AnswerMarkers),
|
||||
Runes: len([]rune(snippet)), Complete: covered == len(inst.AnswerMarkers),
|
||||
}
|
||||
}
|
||||
|
||||
// reportShippedPath cross-checks the table against the real recall tool: a
|
||||
// fixture, a fold, and a search through the agent that ships.
|
||||
func reportShippedPath(root string) {
|
||||
fmt.Println("\n## Through the shipped recall path (width as shipped)")
|
||||
for _, t := range allTasks() {
|
||||
inst, err := instantiateTask(t, seededRand(t.ID, "snippet-live"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
f, err := buildFixture(inst, ablation.Set{}, filepath.Join(root, "snippet", t.ID, "session.jsonl"))
|
||||
if err != nil {
|
||||
fmt.Printf(" %-24s fixture: %v\n", t.ID, err)
|
||||
continue
|
||||
}
|
||||
a := agent.New(&digestProvider{}, tool.NewRegistry(), f.Session, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors,
|
||||
}, event.Discard)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
queries := append([]string{inst.ProbeQuery}, historicalQueries[t.ID]...)
|
||||
for _, q := range queries {
|
||||
res, err := a.RecallContext(context.Background(), tool.RecallRequest{Query: q})
|
||||
if err != nil {
|
||||
fmt.Printf(" %-24s %-34q recall: %v\n", t.ID, q, err)
|
||||
continue
|
||||
}
|
||||
covered := 0
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if strings.Contains(strings.ToLower(res.Text), strings.ToLower(marker)) {
|
||||
covered++
|
||||
}
|
||||
}
|
||||
fmt.Printf(" %-24s %-34q %d/%d values in the result\n",
|
||||
t.ID, truncate(q, 32), covered, len(inst.AnswerMarkers))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func completeRate(rs []snippetReading) float64 {
|
||||
if len(rs) == 0 {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, r := range rs {
|
||||
if r.Complete {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return float64(n) / float64(len(rs))
|
||||
}
|
||||
|
||||
func meanRunes(rs []snippetReading) float64 {
|
||||
if len(rs) == 0 {
|
||||
return 0
|
||||
}
|
||||
total := 0
|
||||
for _, r := range rs {
|
||||
total += r.Runes
|
||||
}
|
||||
return float64(total) / float64(len(rs))
|
||||
}
|
||||
|
||||
// answerCoverage is what a run should have recorded all along: how much of the
|
||||
// scored answer a search actually handed over. rank=1 with coverage 1/3 is not
|
||||
// a stopping failure.
|
||||
func answerCoverage(text string, markers []string) int {
|
||||
covered := 0
|
||||
lower := strings.ToLower(text)
|
||||
for _, marker := range markers {
|
||||
if strings.Contains(lower, strings.ToLower(marker)) {
|
||||
covered++
|
||||
}
|
||||
}
|
||||
return covered
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// usageSink reads what a run cost from the events the agent already emits. Every
|
||||
// billed completion reports event.Usage — the round path, compaction, command
|
||||
// routing and failure diagnosis all do — so the bench reads the same record the
|
||||
// product reads instead of wrapping a provider and counting a second time.
|
||||
type usageSink struct {
|
||||
mu sync.Mutex
|
||||
requests int
|
||||
usage provider.Usage
|
||||
}
|
||||
|
||||
func (s *usageSink) Emit(e event.Event) {
|
||||
if e.Kind != event.Usage || e.Usage == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
// One event is one completion that reported usage; RequestCount covers a
|
||||
// completion that merged attempts. A request that failed before reporting
|
||||
// leaves no event, so this is not a count of attempts.
|
||||
if e.Usage.RequestCount > 0 {
|
||||
s.requests += e.Usage.RequestCount
|
||||
} else {
|
||||
s.requests++
|
||||
}
|
||||
u := *e.Usage
|
||||
s.usage.PromptTokens += u.PromptTokens
|
||||
s.usage.CompletionTokens += u.CompletionTokens
|
||||
s.usage.TotalTokens += u.TotalTokens
|
||||
s.usage.CacheHitTokens += u.CacheHitTokens
|
||||
s.usage.CacheMissTokens += u.CacheMissTokens
|
||||
s.usage.ReasoningTokens += u.ReasoningTokens
|
||||
s.usage.Estimated = s.usage.Estimated || u.Estimated
|
||||
}
|
||||
|
||||
func (s *usageSink) snapshot() (provider.Usage, int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.usage, s.requests
|
||||
}
|
||||
|
||||
// promptTokenSuffix renders what a run cost only when something reported it. A
|
||||
// dry run reports nothing, and a trailing "prompt-tok=0" would read as a
|
||||
// measured zero rather than as no measurement.
|
||||
func promptTokenSuffix(m contextMetrics) string {
|
||||
if m.UsageReportedRequests == 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" prompt-tok=%d", m.PromptTokens)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// The cost a run reports must be what the agent reported, summed across every
|
||||
// completion it billed — including one that merged attempts and one that could
|
||||
// only estimate. Any other event kind is not a cost.
|
||||
func TestUsageSinkAccumulatesWhatTheAgentReports(t *testing.T) {
|
||||
var sink usageSink
|
||||
sink.Emit(event.Event{}) // a kind that is not Usage
|
||||
sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{
|
||||
PromptTokens: 100, CompletionTokens: 10, TotalTokens: 110, CacheHitTokens: 60, CacheMissTokens: 40,
|
||||
}})
|
||||
sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{
|
||||
PromptTokens: 7, CompletionTokens: 3, CacheHitTokens: 2, CacheMissTokens: 5, RequestCount: 2, Estimated: true,
|
||||
}})
|
||||
sink.Emit(event.Event{Kind: event.Usage}) // a Usage event with no payload is not a cost
|
||||
|
||||
usage, requests := sink.snapshot()
|
||||
if requests != 3 {
|
||||
t.Fatalf("requests = %d, want 3 (1 + the merged attempt's 2; the empty event adds none)", requests)
|
||||
}
|
||||
if usage.PromptTokens != 107 || usage.CompletionTokens != 13 || usage.TotalTokens != 110 {
|
||||
t.Fatalf("accumulated usage = %+v", usage)
|
||||
}
|
||||
if usage.CacheHitTokens != 62 || usage.CacheMissTokens != 45 {
|
||||
t.Fatalf("cache split = %+v", usage)
|
||||
}
|
||||
if !usage.Estimated {
|
||||
t.Fatal("Estimated was dropped when one completion reported an estimate")
|
||||
}
|
||||
}
|
||||
|
||||
// A dry run must not claim a cost it never paid, in the JSON or in the line a
|
||||
// person reads: an empty suffix, not "prompt-tok=0".
|
||||
func TestUsageSinkStaysEmptyWithoutUsage(t *testing.T) {
|
||||
var sink usageSink
|
||||
sink.Emit(event.Event{})
|
||||
usage, requests := sink.snapshot()
|
||||
if requests != 0 || usage.PromptTokens != 0 || usage.CompletionTokens != 0 {
|
||||
t.Fatalf("nothing reported usage, got %+v requests=%d", usage, requests)
|
||||
}
|
||||
if suffix := promptTokenSuffix(contextMetrics{}); suffix != "" {
|
||||
t.Fatalf("a run with no reported usage still printed %q", suffix)
|
||||
}
|
||||
if suffix := promptTokenSuffix(contextMetrics{UsageReportedRequests: 1, PromptTokens: 12}); suffix != " prompt-tok=12" {
|
||||
t.Fatalf("a reported cost printed %q", suffix)
|
||||
}
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
# realrepo
|
||||
|
||||
Long tasks over a pinned snapshot of this repository, for measuring changes
|
||||
that aim to cut model round trips. The `benchmarks/e2e` tasks finish in about
|
||||
5–15 requests and ten seconds; a real task in this tree took a flash-class
|
||||
model 38 requests, most of them after a single read. A suite of short fixtures
|
||||
cannot show that gap moving, so these tasks are shaped like the real one:
|
||||
explore several packages, edit several files, verify, and recover from the
|
||||
verifier.
|
||||
|
||||
```sh
|
||||
go build -o bin/reasonix ./cmd/reasonix
|
||||
go run ./cmd/e2ebench -suite benchmarks/realrepo -bin "$PWD/bin/reasonix" -model deepseek-flash \
|
||||
-budget 0 -trials 2 -trajectories traj -json realrepo.json
|
||||
go run ./cmd/e2ebench -suite benchmarks/realrepo -bin "$PWD/bin/reasonix" -task rt-stale-skills
|
||||
```
|
||||
|
||||
The agent reads its provider config from `REASONIX_HOME` (default
|
||||
`~/.reasonix`); point it at a disposable one to keep benchmark sessions out of
|
||||
your own.
|
||||
|
||||
The default `-budget` (800k tokens) can stop the suite partway through, hence
|
||||
`-budget 0`. Under `-trials`, each trial keeps its own
|
||||
`<task>.trial-N.trajectory.jsonl`.
|
||||
|
||||
## The snapshot
|
||||
|
||||
Every `workdir/` is the same Go module (`module reasonix`, stdlib only, no
|
||||
network needed to build or test):
|
||||
|
||||
- **Pinned code.** `git archive` of commit `42636ddcb425` for
|
||||
`internal/contract/{pricing,ablation,agentgraph}` and
|
||||
`internal/base/{retrieval,rulelist,tokencount,neterr,nilutil}` — about 5k
|
||||
lines of real kernel code with its real tests and doc comments.
|
||||
- **Fixture-authored files** in `base/`: `go.mod`, a trimmed `REASONIX.md` (the
|
||||
project's own conventions, so the agent loads standing instructions the way
|
||||
it does here), a `Makefile` with `check`, a module `README.md`, the
|
||||
`cmd/benchctl` command that consumes `pricing` and `ablation` from outside
|
||||
their package, and one project skill.
|
||||
- **One pinned test anchored to the snapshot day.**
|
||||
`TestCatalogPricesHaveBeenCheckedRecently` compares each row's `CheckedOn`
|
||||
with the wall clock, which would fail every workdir 90 days after the oldest
|
||||
pinned date (2026-11-20). `base/` replaces that file with one that reads
|
||||
staleness at 2026-10-19 instead; nothing else in the pin is edited.
|
||||
- **Per-task overlay** in `overlay/<task-id>/`, applied last. Only
|
||||
`rt-stale-skills` has one.
|
||||
|
||||
`snapshot.sh` rebuilds every workdir from those three sources and
|
||||
`snapshot.sh --check` fails if a committed workdir has drifted from them. Edit
|
||||
`base/` or `overlay/`, never a workdir, then rerun the script. Moving the pin is
|
||||
a new suite version: results from two pins do not compare.
|
||||
|
||||
## Tasks
|
||||
|
||||
| Task | Kind | What it exercises |
|
||||
| --- | --- | --- |
|
||||
| `rt-cache-write-rate` | multi-file Go change, test, README | Adds a `CacheWrite` rate to `RateCard` and threads it through cost, fingerprint, card identity and the catalog, then prices Anthropic's `claude-sonnet-4-5` with it: a new `officialRates` row, documentation and checked-on constants, an endpoint host, and `benchctl rate`/`cost` output. The fingerprint has to stay byte-identical for every existing card, so the change to `PricingFingerprint` cannot be the obvious one. |
|
||||
| `rt-stale-skills` | documentation only, no recognised verifier | Three project skills name paths, functions, `make` targets and `benchctl` subcommands and flags that do not exist, and one contradicts the append-only rule in `REASONIX.md`. Fixing them takes reading `rates.go`, `ablation.go`, both `benchctl` files and the `Makefile`, then editing only Markdown — the shape behind #13293, where `stale_verification` stays owed after a skill edit because no check covers it. |
|
||||
| `rt-rate-lookup-signatures` | API migration with a likely compile-error loop | Changes `CurrentRate` to `(RateCard, bool)`, `LookupCatalog` to a `CatalogKey` struct, and `IsOfficialCard`, `MatchesCatalog` and `LiveTariff` to a `ModelRef`: about 20 call sites across 11 files. Most are in `_test.go` files and dereference the old pointer, so `go build ./...` can pass while `go vet` and `go test` fail. |
|
||||
| `rt-plan-ablation-module` | plan-mode with `complete_step` | Five numbered steps adding a `memory` ablation module, summaries, `ModuleHelp` and `benchctl arm -list`. The prompt opens with the host's own plan-first marker (`control.PlannerRouteMarker`), which routes the turn through the planner, and asks for each step to be signed off before the next. An existing test pins the `all` arm name and has to be updated, which adds a red-test step. |
|
||||
|
||||
Every grader stages hidden tests and checks after the run: `gofmt -l`, `go vet
|
||||
./...` and `go test ./...` for the Go tasks; for `rt-stale-skills`, every
|
||||
backticked path, identifier, `make` target and `benchctl` command line in each
|
||||
skill must resolve against the tree, and a SHA-256 manifest proves no Go source, `go.mod`,
|
||||
`Makefile` or `REASONIX.md` changed. Each task fails its pristine seed and
|
||||
passes its `solution/`, both checked by `cmd/e2ebench`'s corpus tests.
|
||||
|
||||
## Reading the numbers
|
||||
|
||||
```text
|
||||
Metric: requests per task
|
||||
Anchor: the trajectory digest's requests_by_source, summed over sources
|
||||
Population: every graded trial of a task that ran to its own stop
|
||||
Excludes: tasks the token budget skipped, and runs that hit timeout_sec
|
||||
```
|
||||
|
||||
Requests is the headline because round trips are what these tasks exist to
|
||||
move. Read it beside the solve rate and the report's round outcomes: a change
|
||||
that drops requests by giving up on verification has not cut a round trip, it
|
||||
has cut a check. `rt-stale-skills` is read for whether the run finishes at all
|
||||
and how many rounds go to re-verifying after its last write.
|
||||
|
||||
## Baseline
|
||||
|
||||
`deepseek-flash`, 2 trials per task, at studio `0b8de9e63`. Requests are the
|
||||
trajectory's executor requests; all 8 runs ran to their own stop.
|
||||
|
||||
| Task | Solved | Requests | `edit_file` calls | Edit rounds |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `rt-cache-write-rate` | 2/2 | 31, 28 | 21, 20 | 8, 8 |
|
||||
| `rt-plan-ablation-module` | 2/2 | 24, 30 | 10, 13 | 9, 7 |
|
||||
| `rt-rate-lookup-signatures` | 2/2 | 13, 15 | 10, 17 | 2, 6 |
|
||||
| `rt-stale-skills` | 2/2* | 20, 26 | 3, 2 | 3, 3 |
|
||||
|
||||
\* One run first graded as failed, on a glob (`*_rates_test.go`) and a
|
||||
placeholder (`benchctl <subcommand>`) that were both correct. The grader now
|
||||
resolves globs against the tree and skips placeholders, and that run counts
|
||||
as solved. The same task's first version failed a run for naming
|
||||
`CatalogEntry.EffectiveFrom` in order to warn against it; that check went too.
|
||||
|
||||
Reads and edits already batch: 20 edit calls fit into 8 rounds, and several
|
||||
`edit_file` calls to one file often share a round. Rounds that edit a file the
|
||||
previous round also edited are rare (0–2 per run), so steering same-file edits
|
||||
to `multi_edit` has little to save. On this suite's first version, a
|
||||
`ToolBatchPolicy` sentence asking for it drew no `multi_edit` call and 167
|
||||
requests against 158.
|
||||
|
||||
`rt-plan-ablation-module` spends five rounds per run on `complete_step` alone.
|
||||
A description change inviting the sign-off to share the next step's reply was
|
||||
taken up in 2 of 4 trials (4 of 5 sign-offs shared each time, against 0 of 20
|
||||
in control), and requests did not move: 30, 25, 21, 28 against 27, 23, 24, 30.
|
||||
At this spread, four trials per arm cannot show a saving of a few rounds.
|
||||
|
||||
## Gaps
|
||||
|
||||
- **Below the intended size, in places.** The tasks were designed for 25–50
|
||||
requests. After widening, the median is 25, but `rt-rate-lookup-signatures`
|
||||
still runs 13–15: the model greps every call site first and batches the edits,
|
||||
so even about 20 call sites cost few rounds, and no run hit a compile-error
|
||||
loop.
|
||||
- **Lexical grader.** `rt-stale-skills` judges Markdown by resolving what it
|
||||
names. It has misjudged correct text twice (see the baseline note); a new
|
||||
failure there should be read before it is counted.
|
||||
- **No git history.** The workdir is a plain directory: `git diff`, `git log`
|
||||
and `git status` have nothing to read, so runs that lean on them in this tree
|
||||
behave differently here.
|
||||
- **Plan marker and `-force-planner`.** `rt-plan-ablation-module` carries the
|
||||
plan-first marker in its prompt, so it engages the planner in every arm,
|
||||
including runs without `-force-planner`. Exclude it when comparing planner
|
||||
arms.
|
||||
@@ -1,18 +0,0 @@
|
||||
---
|
||||
name: price-change
|
||||
description: Record a vendor's announced price change in the official price book without losing the rate it replaces.
|
||||
---
|
||||
|
||||
# Recording a price change
|
||||
|
||||
1. Find the vendor's rows in `officialRates` in
|
||||
`internal/contract/pricing/rates.go`. Each row is one model in one currency.
|
||||
2. **Append** a `rateGeneration` to the row's `Rates`; never edit an existing
|
||||
one. Set `From` to the first billed day (`YYYY-MM-DD`, UTC) when the change
|
||||
is announced ahead of time.
|
||||
3. Leave the earlier generation where it is: `SupersededRates` reads it so a
|
||||
stored price that matches it is recognised as ours and refreshed.
|
||||
4. Update the row's `CheckedOn` constant to the day you confirmed the page.
|
||||
5. Add a test beside `rates_test.go` that pins `CurrentRate` before and after
|
||||
the `From` day by replacing `today`.
|
||||
6. Run `make check`. `make pricecheck` needs the network and is run by hand.
|
||||
@@ -1,17 +0,0 @@
|
||||
.PHONY: check test vet fmt pricecheck
|
||||
|
||||
check: fmt vet test
|
||||
|
||||
fmt:
|
||||
@out="$$(gofmt -l .)"; if [ -n "$$out" ]; then echo "gofmt needed:"; echo "$$out"; exit 1; fi
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
# Reads each vendor's pricing page; this snapshot carries no network client.
|
||||
pricecheck:
|
||||
@echo "pricecheck: reaches each vendor's pricing page and needs the network; not available in this snapshot" >&2
|
||||
@exit 2
|
||||
@@ -1,69 +0,0 @@
|
||||
# Reasonix kernel snapshot
|
||||
|
||||
A trimmed slice of the Reasonix Go kernel: the contracts a benchmark report is
|
||||
priced and labelled with, a few leaf utilities, and `benchctl`, a small command
|
||||
that prints what those contracts answer.
|
||||
|
||||
```bash
|
||||
make check # gofmt, vet, test
|
||||
go run ./cmd/benchctl rate -model deepseek-flash -currency USD
|
||||
go run ./cmd/benchctl cost -provider mimo -model mimo-v2.5 -miss 1000000
|
||||
go run ./cmd/benchctl tariff -currency CNY -at 2026-10-12T02:00:00Z
|
||||
go run ./cmd/benchctl arm -ablate planner,evidence
|
||||
```
|
||||
|
||||
## Packages
|
||||
|
||||
| Package | Owns |
|
||||
| --- | --- |
|
||||
| `internal/contract/pricing` | the official price book, cost quotes, the spend ledger |
|
||||
| `internal/contract/ablation` | which subsystems a benchmark arm switches off, and the arm's name |
|
||||
| `internal/contract/agentgraph` | the vocabulary of one run's execution graph |
|
||||
| `internal/base/retrieval` | BM25 ranking over CJK-aware tokens |
|
||||
| `internal/base/rulelist` | splitting a one-line permission rule list |
|
||||
| `internal/base/tokencount` | a cheap token estimate |
|
||||
| `internal/base/neterr` | classifying transport errors |
|
||||
| `internal/base/nilutil` | typed-nil detection |
|
||||
| `cmd/benchctl` | prints rates, costs, peak schedules and arm names |
|
||||
|
||||
## Pricing
|
||||
|
||||
A `RateCard` is a price per million tokens in one currency:
|
||||
|
||||
| Field | Charges |
|
||||
| --- | --- |
|
||||
| `CacheHit` | prompt tokens served from the provider's cache |
|
||||
| `Input` | prompt tokens that missed the cache, including tokens written to it |
|
||||
| `Output` | completion tokens |
|
||||
|
||||
`OriginalCostAmount` turns a card and a `UsageTokens` breakdown into a
|
||||
fixed-point `Amount`; `PricingFingerprint` keys a card in the ledger, so a
|
||||
stored ledger entry stays attached to the card it was priced with.
|
||||
`CurrentRate` and `SupersededRates` answer from `officialRates`, which only
|
||||
ever grows: a price change appends a generation.
|
||||
|
||||
## Ablation
|
||||
|
||||
`ablation.Parse` reads a comma-separated list of modules to switch off, and
|
||||
`Set.Arm` names the result (`full` for the control). The switchable modules,
|
||||
in the order arm names use:
|
||||
|
||||
| Module | Off means |
|
||||
| --- | --- |
|
||||
| `evidence` | no evidence gate on completion claims |
|
||||
| `planner` | no two-model planning turn |
|
||||
| `subagent` | no delegation to workers |
|
||||
| `retrieval` | no retrieval over the folded transcript |
|
||||
| `compaction` | no context folding |
|
||||
| `upstream` | a dependency edge orders its endpoints without delivering the answer |
|
||||
| `recall-search` | recall can read a given address but cannot search for one |
|
||||
|
||||
The fold-index scale is a separate axis (`-fold-index`), not a module.
|
||||
|
||||
## Skills
|
||||
|
||||
Project skills live under `.reasonix/skills/<name>/SKILL.md`.
|
||||
|
||||
| Skill | Use it to |
|
||||
| --- | --- |
|
||||
| `price-change` | record a vendor's price change in the price book |
|
||||
@@ -1,51 +0,0 @@
|
||||
# Reasonix project memory
|
||||
|
||||
This checkout is a trimmed snapshot of the Reasonix kernel: the billing,
|
||||
ablation and graph contracts and a handful of leaf utilities, plus one small
|
||||
command that drives them. The conventions below are the project's own.
|
||||
|
||||
## Conventions
|
||||
|
||||
- Go kernel under `internal/`; each package owns one concern. A package's long
|
||||
explanation belongs in its `doc.go`, not spread across implementation files.
|
||||
- Judgements read structure, never wording: types, contracts, schemas. An
|
||||
error a caller must tell apart carries an identity — a sentinel, a typed
|
||||
code — never a sentence.
|
||||
- A metric is its definition, not its name. Before a number becomes a finding,
|
||||
state what it counts, from which anchor, over which population, and what it
|
||||
excludes.
|
||||
|
||||
## Comments
|
||||
|
||||
Default is none — the code is the truth. Write one only when the **why** is
|
||||
non-obvious: a hidden constraint, an invariant the type system cannot express,
|
||||
or an external-protocol quirk. A comment states the constraint that holds now;
|
||||
it does not narrate how the code got here. English, always.
|
||||
|
||||
- Declaration doc: ≤5 lines. Package comment: ≤8 lines, or ≤40 in a `doc.go`.
|
||||
- Every other comment: ≤3 lines. Struct-field and trailing `//`: 1 line.
|
||||
- One responsibility per file; 800 lines is the ceiling.
|
||||
|
||||
## Vendor rates
|
||||
|
||||
Official list prices live in one place: `officialRates` in
|
||||
`internal/contract/pricing/rates.go`, oldest generation first, the last being
|
||||
what a model charges today. A price change is an **append, never an edit** —
|
||||
the superseded rate is what lets an installed config be recognised as holding
|
||||
one of ours and brought forward. Callers read these through
|
||||
`pricing.CurrentRate` and `pricing.SupersededRates` and restate none of them.
|
||||
|
||||
`make pricecheck` compares that table against each vendor's own page. It needs
|
||||
the network, so it is not part of `make check`.
|
||||
|
||||
## Before every commit
|
||||
|
||||
```bash
|
||||
make check # gofmt -l, go vet ./..., go test ./...
|
||||
```
|
||||
|
||||
## Change discipline
|
||||
|
||||
List callers, consumers of the same state and sibling features before editing.
|
||||
Fix at the owning layer; a fix that breaks a neighbouring behaviour is a failed
|
||||
fix. Write tests first; they must fail without the change.
|
||||
@@ -1,115 +0,0 @@
|
||||
// Command benchctl prints what the pricing and ablation contracts answer for
|
||||
// one model or one benchmark arm, so a reviewer can check a rate or an arm name
|
||||
// without writing a test.
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/pricing"
|
||||
)
|
||||
|
||||
var errUsage = errors.New("usage: benchctl rate|cost|tariff|arm [flags]")
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "benchctl:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func run(args []string, out io.Writer) error {
|
||||
if len(args) == 0 {
|
||||
return errUsage
|
||||
}
|
||||
switch args[0] {
|
||||
case "rate":
|
||||
return runRate(args[1:], out)
|
||||
case "cost":
|
||||
return runCost(args[1:], out)
|
||||
case "tariff":
|
||||
return runTariff(args[1:], out)
|
||||
case "arm":
|
||||
return runArm(args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("%w: unknown subcommand %q", errUsage, args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func runRate(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("rate", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
fmt.Fprintf(out, "%s/%s %s: cache-hit %g input %g output %g per 1M\n",
|
||||
*provider, *model, card.Currency, card.CacheHit, card.Input, card.Output)
|
||||
if entry, ok := pricing.LookupCatalog(*provider, *model, cur, ""); ok && entry.DocURL != "" {
|
||||
fmt.Fprintf(out, "source: %s\n", entry.DocURL)
|
||||
}
|
||||
fmt.Fprintf(out, "superseded generations: %d\n", len(pricing.SupersededRates(*provider, *model, cur)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runCost(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("cost", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
hit := fs.Int("hit", 0, "cached prompt tokens")
|
||||
miss := fs.Int("miss", 0, "uncached prompt tokens")
|
||||
completion := fs.Int("completion", 0, "completion tokens")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
usage := pricing.UsageTokens{CacheHitTokens: *hit, CacheMissTokens: *miss, CompletionTokens: *completion}
|
||||
amount := pricing.OriginalCostAmount(*card, usage)
|
||||
fmt.Fprintf(out, "%s %s\n", amount, pricing.NormalizeCurrency(card.Currency))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runArm(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("arm", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
spec := fs.String("ablate", "", "comma-separated modules to switch off: "+ablation.ModuleList())
|
||||
foldIndex := fs.String("fold-index", "", "fold-index scale: default, half, quarter or off")
|
||||
list := fs.Bool("list", false, "list the switchable modules")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *list {
|
||||
fmt.Fprintln(out, ablation.ModuleList())
|
||||
return nil
|
||||
}
|
||||
set, err := ablation.ParseArm(*spec, *foldIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(out, set.Arm())
|
||||
return nil
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func runOut(t *testing.T, args ...string) string {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
if err := run(args, &out); err != nil {
|
||||
t.Fatalf("run %v: %v", args, err)
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func TestRateNamesTheCurrentCard(t *testing.T) {
|
||||
got := runOut(t, "rate", "-provider", "mimo", "-model", "mimo-v2.5")
|
||||
if !strings.Contains(got, "mimo/mimo-v2.5 CNY: cache-hit 0.02 input 1 output 2 per 1M") {
|
||||
t.Fatalf("rate output = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateRefusesAnUnpricedModel(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if err := run([]string{"rate", "-provider", "nobody", "-model", "x"}, &out); err == nil {
|
||||
t.Fatal("an unpriced model must fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCostChargesTheCurrentCard(t *testing.T) {
|
||||
got := runOut(t, "cost", "-provider", "mimo", "-model", "mimo-v2.5", "-miss", "1000000", "-completion", "1000000")
|
||||
if strings.TrimSpace(got) != "3 CNY" {
|
||||
t.Fatalf("cost output = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArmNamesTheConfiguration(t *testing.T) {
|
||||
if got := strings.TrimSpace(runOut(t, "arm")); got != "full" {
|
||||
t.Fatalf("control arm = %q", got)
|
||||
}
|
||||
if got := strings.TrimSpace(runOut(t, "arm", "-ablate", "planner,evidence", "-fold-index", "half")); got != "no-evidence+no-planner+index-half" {
|
||||
t.Fatalf("arm = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownSubcommandIsAUsageError(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if err := run([]string{"nope"}, &out); !errors.Is(err, errUsage) {
|
||||
t.Fatalf("err = %v, want errUsage", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTariffReadsTheVendorsSchedule(t *testing.T) {
|
||||
got := runOut(t, "tariff", "-provider", "mimo", "-model", "mimo-v2.5")
|
||||
if !strings.Contains(got, "official card: true") || !strings.Contains(got, "one rate around the clock") {
|
||||
t.Fatalf("tariff output = %q", got)
|
||||
}
|
||||
peak := runOut(t, "tariff", "-currency", "CNY", "-at", "2026-10-12T02:00:00Z")
|
||||
if !strings.Contains(peak, "peak=true until 2026-10-12T04:00:00Z") {
|
||||
t.Fatalf("deepseek tariff at a weekday peak = %q", peak)
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/pricing"
|
||||
)
|
||||
|
||||
func runTariff(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("tariff", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
at := fs.String("at", "", "instant to read the schedule at, RFC 3339; empty is now")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
when := time.Now()
|
||||
if *at != "" {
|
||||
t, err := time.Parse(time.RFC3339, *at)
|
||||
if err != nil {
|
||||
return fmt.Errorf("-at: %w", err)
|
||||
}
|
||||
when = t
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
fmt.Fprintf(out, "official card: %v\n", pricing.IsOfficialCard(*provider, *model, *card))
|
||||
tariff, ok := pricing.LiveTariff(*provider, *model, *card, when)
|
||||
switch {
|
||||
case !ok:
|
||||
fmt.Fprintln(out, "schedule: one rate around the clock")
|
||||
case !tariff.Changes:
|
||||
fmt.Fprintf(out, "schedule: peak=%v, no change ahead\n", tariff.Peak)
|
||||
default:
|
||||
fmt.Fprintf(out, "schedule: peak=%v until %s\n", tariff.Peak, tariff.ChangesAt.UTC().Format(time.RFC3339))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module reasonix
|
||||
|
||||
go 1.26.0
|
||||
@@ -1,95 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Every catalog provider must declare at least one official endpoint, or its
|
||||
// rows become unreachable: a rate card is only matched against the catalog
|
||||
// after the entry's host identifies the vendor.
|
||||
func TestEveryCatalogProviderHasAnOfficialEndpoint(t *testing.T) {
|
||||
declared := map[string]bool{}
|
||||
for _, host := range officialEndpointHosts {
|
||||
declared[host] = true
|
||||
}
|
||||
for _, entry := range OfficialCatalog() {
|
||||
if !declared[entry.Provider] {
|
||||
t.Errorf("catalog provider %q has no official endpoint host; its prices can never match", entry.Provider)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfficialProviderForEndpoint(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
baseURL string
|
||||
want string
|
||||
}{
|
||||
{"official deepseek", "https://api.deepseek.com", "deepseek"},
|
||||
{"official deepseek anthropic path", "https://api.deepseek.com/anthropic", "deepseek"},
|
||||
{"official longcat", "https://api.longcat.chat/openai/v1", "longcat"},
|
||||
{"official mimo region", "https://token-plan-sgp.xiaomimimo.com/v1", "mimo"},
|
||||
{"uppercase host", "https://API.DeepSeek.com/v1", "deepseek"},
|
||||
{"proxy that resells the model", "https://my-deepseek-proxy.internal/v1", ""},
|
||||
{"vendor name in the path only", "https://gateway.example.com/deepseek/v1", ""},
|
||||
{"lookalike registrable domain", "https://api.deepseek.com.evil.test/v1", ""},
|
||||
{"empty", "", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := OfficialProviderForEndpoint(tc.baseURL); got != tc.want {
|
||||
t.Fatalf("OfficialProviderForEndpoint(%q) = %q, want %q", tc.baseURL, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// pricesGoStaleAfter is how long a hand-copied rate is trusted before someone
|
||||
// reads the vendor's page again. No vendor here serves prices over an API, so
|
||||
// the alternative to this going red on a quiet day is learning about a change
|
||||
// from a user whose cost was wrong for months. Bumping CheckedOn without
|
||||
// opening DocURL defeats it; what this buys is that the question gets asked.
|
||||
const pricesGoStaleAfter = 90 * 24 * time.Hour
|
||||
|
||||
func TestCatalogPricesHaveBeenCheckedRecently(t *testing.T) {
|
||||
var undated, stale []string
|
||||
// This snapshot never sees a vendor page again, so staleness is read at the
|
||||
// day it was taken rather than the day the suite runs.
|
||||
cutoff := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC).Add(-pricesGoStaleAfter)
|
||||
for _, e := range OfficialCatalog() {
|
||||
where := e.Provider + "/" + e.Model + " " + e.Currency + " (" + e.DocURL + ")"
|
||||
if e.CheckedOn == "" {
|
||||
undated = append(undated, where)
|
||||
continue
|
||||
}
|
||||
on, err := time.Parse("2006-01-02", e.CheckedOn)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: CheckedOn %q is not YYYY-MM-DD", where, e.CheckedOn)
|
||||
}
|
||||
if on.Before(cutoff) {
|
||||
stale = append(stale, where+" last checked "+e.CheckedOn)
|
||||
}
|
||||
}
|
||||
if len(undated) > 0 {
|
||||
t.Errorf("catalog rows with no CheckedOn — read the vendor's page and date them:\n %s",
|
||||
strings.Join(undated, "\n "))
|
||||
}
|
||||
if len(stale) > 0 {
|
||||
t.Errorf("catalog rows older than %d days. Open each DocURL, confirm or correct the rate, then set CheckedOn to today:\n %s",
|
||||
int(pricesGoStaleAfter.Hours()/24), strings.Join(stale, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// A row that names a peak card must say when it applies, and one that names a
|
||||
// window must have a card to charge inside it. Half of the pair silently prices
|
||||
// every hour the same, which is the bug the pair exists to prevent.
|
||||
func TestPeakRatesAndWindowsComeInPairs(t *testing.T) {
|
||||
for _, e := range OfficialCatalog() {
|
||||
where := e.Provider + "/" + e.Model + " " + e.Currency
|
||||
if (e.Peak == nil) != (e.Window == nil) {
|
||||
t.Errorf("%s: Peak and Window must be set together (peak=%v window=%v)", where, e.Peak != nil, e.Window != nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
---
|
||||
name: ablation-arm
|
||||
description: Add a new switchable subsystem to the benchmark's ablation arms.
|
||||
---
|
||||
|
||||
# Adding an ablation module
|
||||
|
||||
1. Declare the module in `internal/ablation/modules.go` beside the others.
|
||||
2. Register it in `AllModules()` so `ParseSpec` accepts it.
|
||||
3. Arm names come from `Set.Name()`; check that yours reads `without-<module>`.
|
||||
4. Add the module to the flag help text in `cmd/bench/flags.go`.
|
||||
5. Run `make test`.
|
||||
@@ -1,15 +0,0 @@
|
||||
---
|
||||
name: benchctl
|
||||
description: Check a rate, a cost, a peak schedule or an arm name from the command line instead of writing a test.
|
||||
---
|
||||
|
||||
# Using benchctl
|
||||
|
||||
1. Price a request with `benchctl quote -provider mimo -model mimo-v2.5 -tokens 1000000`;
|
||||
it prints the amount and the currency.
|
||||
2. Read today's card with `benchctl rate -vendor deepseek -model deepseek-flash`.
|
||||
3. Ask whether a vendor is in its peak window with
|
||||
`benchctl peak -currency CNY -at 1791770400` (Unix seconds).
|
||||
4. List the ablation modules with `benchctl arms`, and name an arm with
|
||||
`benchctl arm -ablate planner,evidence -index half`.
|
||||
5. Every subcommand prints its flags with `-h`.
|
||||
@@ -1,13 +0,0 @@
|
||||
---
|
||||
name: price-change
|
||||
description: Record a vendor's announced price change in the official price book.
|
||||
---
|
||||
|
||||
# Recording a price change
|
||||
|
||||
1. Open `internal/pricing/rates.go` and find the vendor's entry in
|
||||
`officialPrices`.
|
||||
2. Edit the entry's `Input`, `Output` and `CacheHit` to the new values.
|
||||
3. If the change is announced ahead of time, set `EffectiveFrom` on the entry.
|
||||
4. Confirm `pricing.LatestRate` now returns the new card.
|
||||
5. Run `make test` and `make price-check`.
|
||||
@@ -1,52 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Builds every task's workdir/ from one pinned commit of this repository plus
|
||||
# the fixture-authored files in base/ and overlay/<task-id>/. With --check it
|
||||
# rebuilds into a scratch directory and fails if any committed workdir differs.
|
||||
set -euo pipefail
|
||||
|
||||
PIN=42636ddcb425b7af07abd88397d02c1cfe3adcf5
|
||||
PATHS=(
|
||||
internal/contract/pricing
|
||||
internal/contract/ablation
|
||||
internal/contract/agentgraph
|
||||
internal/base/retrieval
|
||||
internal/base/rulelist
|
||||
internal/base/tokencount
|
||||
internal/base/neterr
|
||||
internal/base/nilutil
|
||||
)
|
||||
|
||||
here="$(cd "$(dirname "$0")" && pwd)"
|
||||
repo="$(git -C "$here" rev-parse --show-toplevel)"
|
||||
|
||||
build() {
|
||||
local id="$1" dest="$2"
|
||||
rm -rf "$dest"
|
||||
mkdir -p "$dest"
|
||||
git -C "$repo" archive "$PIN" -- "${PATHS[@]}" | tar -x -C "$dest"
|
||||
cp -R "$here/base/." "$dest/"
|
||||
if [ -d "$here/overlay/$id" ]; then
|
||||
cp -R "$here/overlay/$id/." "$dest/"
|
||||
fi
|
||||
}
|
||||
|
||||
check=0
|
||||
[ "${1:-}" = "--check" ] && check=1
|
||||
|
||||
status=0
|
||||
for task in "$here"/tasks/*/; do
|
||||
id="$(basename "$task")"
|
||||
if [ "$check" = 1 ]; then
|
||||
scratch="$(mktemp -d)"
|
||||
build "$id" "$scratch/workdir"
|
||||
if ! diff -r "$scratch/workdir" "$task/workdir" >/dev/null; then
|
||||
echo "$id: workdir/ differs from the pinned snapshot; run $0" >&2
|
||||
diff -rq "$scratch/workdir" "$task/workdir" >&2 || true
|
||||
status=1
|
||||
fi
|
||||
rm -rf "$scratch"
|
||||
else
|
||||
build "$id" "$task/workdir"
|
||||
fi
|
||||
done
|
||||
exit "$status"
|
||||
@@ -1,73 +0,0 @@
|
||||
# Reasonix kernel snapshot
|
||||
|
||||
A trimmed slice of the Reasonix Go kernel: the contracts a benchmark report is
|
||||
priced and labelled with, a few leaf utilities, and `benchctl`, a small command
|
||||
that prints what those contracts answer.
|
||||
|
||||
```bash
|
||||
make check # gofmt, vet, test
|
||||
go run ./cmd/benchctl rate -model deepseek-flash -currency USD
|
||||
go run ./cmd/benchctl cost -provider mimo -model mimo-v2.5 -miss 1000000
|
||||
go run ./cmd/benchctl tariff -currency CNY -at 2026-10-12T02:00:00Z
|
||||
go run ./cmd/benchctl arm -ablate planner,evidence
|
||||
```
|
||||
|
||||
## Packages
|
||||
|
||||
| Package | Owns |
|
||||
| --- | --- |
|
||||
| `internal/contract/pricing` | the official price book, cost quotes, the spend ledger |
|
||||
| `internal/contract/ablation` | which subsystems a benchmark arm switches off, and the arm's name |
|
||||
| `internal/contract/agentgraph` | the vocabulary of one run's execution graph |
|
||||
| `internal/base/retrieval` | BM25 ranking over CJK-aware tokens |
|
||||
| `internal/base/rulelist` | splitting a one-line permission rule list |
|
||||
| `internal/base/tokencount` | a cheap token estimate |
|
||||
| `internal/base/neterr` | classifying transport errors |
|
||||
| `internal/base/nilutil` | typed-nil detection |
|
||||
| `cmd/benchctl` | prints rates, costs, peak schedules and arm names |
|
||||
|
||||
## Pricing
|
||||
|
||||
A `RateCard` is a price per million tokens in one currency:
|
||||
|
||||
| Field | Charges |
|
||||
| --- | --- |
|
||||
| `CacheHit` | prompt tokens served from the provider's cache |
|
||||
| `Input` | prompt tokens that missed the cache, and tokens written to it when `CacheWrite` is zero |
|
||||
| `Output` | completion tokens |
|
||||
| `CacheWrite` | tokens written to the cache, for a vendor that prices them separately (Anthropic) |
|
||||
|
||||
A card without `CacheWrite` keeps the fingerprint it had before the field
|
||||
existed, so ledgers stored against it stay attached.
|
||||
|
||||
`OriginalCostAmount` turns a card and a `UsageTokens` breakdown into a
|
||||
fixed-point `Amount`; `PricingFingerprint` keys a card in the ledger, so a
|
||||
stored ledger entry stays attached to the card it was priced with.
|
||||
`CurrentRate` and `SupersededRates` answer from `officialRates`, which only
|
||||
ever grows: a price change appends a generation.
|
||||
|
||||
## Ablation
|
||||
|
||||
`ablation.Parse` reads a comma-separated list of modules to switch off, and
|
||||
`Set.Arm` names the result (`full` for the control). The switchable modules,
|
||||
in the order arm names use:
|
||||
|
||||
| Module | Off means |
|
||||
| --- | --- |
|
||||
| `evidence` | no evidence gate on completion claims |
|
||||
| `planner` | no two-model planning turn |
|
||||
| `subagent` | no delegation to workers |
|
||||
| `retrieval` | no retrieval over the folded transcript |
|
||||
| `compaction` | no context folding |
|
||||
| `upstream` | a dependency edge orders its endpoints without delivering the answer |
|
||||
| `recall-search` | recall can read a given address but cannot search for one |
|
||||
|
||||
The fold-index scale is a separate axis (`-fold-index`), not a module.
|
||||
|
||||
## Skills
|
||||
|
||||
Project skills live under `.reasonix/skills/<name>/SKILL.md`.
|
||||
|
||||
| Skill | Use it to |
|
||||
| --- | --- |
|
||||
| `price-change` | record a vendor's price change in the price book |
|
||||
@@ -1,119 +0,0 @@
|
||||
// Command benchctl prints what the pricing and ablation contracts answer for
|
||||
// one model or one benchmark arm, so a reviewer can check a rate or an arm name
|
||||
// without writing a test.
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/pricing"
|
||||
)
|
||||
|
||||
var errUsage = errors.New("usage: benchctl rate|cost|tariff|arm [flags]")
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "benchctl:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func run(args []string, out io.Writer) error {
|
||||
if len(args) == 0 {
|
||||
return errUsage
|
||||
}
|
||||
switch args[0] {
|
||||
case "rate":
|
||||
return runRate(args[1:], out)
|
||||
case "cost":
|
||||
return runCost(args[1:], out)
|
||||
case "tariff":
|
||||
return runTariff(args[1:], out)
|
||||
case "arm":
|
||||
return runArm(args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("%w: unknown subcommand %q", errUsage, args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func runRate(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("rate", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
fmt.Fprintf(out, "%s/%s %s: cache-hit %g input %g output %g", *provider, *model, card.Currency, card.CacheHit, card.Input, card.Output)
|
||||
if card.CacheWrite != 0 {
|
||||
fmt.Fprintf(out, " cache-write %g", card.CacheWrite)
|
||||
}
|
||||
fmt.Fprintln(out, " per 1M")
|
||||
if entry, ok := pricing.LookupCatalog(*provider, *model, cur, ""); ok && entry.DocURL != "" {
|
||||
fmt.Fprintf(out, "source: %s\n", entry.DocURL)
|
||||
}
|
||||
fmt.Fprintf(out, "superseded generations: %d\n", len(pricing.SupersededRates(*provider, *model, cur)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runCost(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("cost", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
hit := fs.Int("hit", 0, "cached prompt tokens")
|
||||
miss := fs.Int("miss", 0, "uncached prompt tokens")
|
||||
completion := fs.Int("completion", 0, "completion tokens")
|
||||
write := fs.Int("write", 0, "uncached prompt tokens that were written to the cache")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
usage := pricing.UsageTokens{CacheHitTokens: *hit, CacheMissTokens: *miss, CompletionTokens: *completion, CacheWriteTokens: *write}
|
||||
amount := pricing.OriginalCostAmount(*card, usage)
|
||||
fmt.Fprintf(out, "%s %s\n", amount, pricing.NormalizeCurrency(card.Currency))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runArm(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("arm", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
spec := fs.String("ablate", "", "comma-separated modules to switch off: "+ablation.ModuleList())
|
||||
foldIndex := fs.String("fold-index", "", "fold-index scale: default, half, quarter or off")
|
||||
list := fs.Bool("list", false, "list the switchable modules")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *list {
|
||||
fmt.Fprintln(out, ablation.ModuleList())
|
||||
return nil
|
||||
}
|
||||
set, err := ablation.ParseArm(*spec, *foldIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(out, set.Arm())
|
||||
return nil
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module reasonix
|
||||
|
||||
go 1.26.0
|
||||
-37
@@ -1,37 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCacheWriteRateChargesWrittenTokens(t *testing.T) {
|
||||
card := RateCard{Input: 3, CacheWrite: 3.75, Currency: "USD"}
|
||||
usage := UsageTokens{CacheMissTokens: 1000000, CacheWriteTokens: 400000}
|
||||
if got := OriginalCostAmount(card, usage).String(); got != "3.3" {
|
||||
t.Fatalf("cost = %s, want 3.3", got)
|
||||
}
|
||||
card.CacheWrite = 0
|
||||
if got := OriginalCostAmount(card, usage).String(); got != "3" {
|
||||
t.Fatalf("cost without a cache-write rate = %s, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheWriteSeparatesFingerprintsAndCards(t *testing.T) {
|
||||
base := RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}
|
||||
withWrite := base
|
||||
withWrite.CacheWrite = 1.25
|
||||
if PricingFingerprint(base) == PricingFingerprint(withWrite) {
|
||||
t.Fatal("cache-write rate does not reach the fingerprint")
|
||||
}
|
||||
if sameCard(base, withWrite) {
|
||||
t.Fatal("cards differing in CacheWrite compare equal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicPricesCacheWrites(t *testing.T) {
|
||||
got := CurrentRate("anthropic", "claude-sonnet-4-5", "USD")
|
||||
if got == nil || got.CacheWrite != 3.75 {
|
||||
t.Fatalf("claude-sonnet-4-5 = %+v", got)
|
||||
}
|
||||
if OfficialProviderForEndpoint("https://api.anthropic.com/v1/messages") != "anthropic" {
|
||||
t.Fatal("api.anthropic.com is not recognised as Anthropic's endpoint")
|
||||
}
|
||||
}
|
||||
-222
@@ -1,222 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Official catalog metadata: currency, model, effective dates, documentation
|
||||
// sources, and price fingerprints. User-custom prices always win over catalog.
|
||||
|
||||
// CatalogEntry is one official list price for a model in a billing currency.
|
||||
type CatalogEntry struct {
|
||||
Provider string // deepseek | longcat | mimo | google
|
||||
Model string
|
||||
Currency string // ISO billing currency for this row
|
||||
CacheHit float64
|
||||
Input float64
|
||||
Output float64
|
||||
CacheWrite float64 // zero when the vendor bills cache writes at Input
|
||||
EffectiveFrom string // YYYY-MM-DD inclusive, empty = unbounded
|
||||
EffectiveTo string // YYYY-MM-DD exclusive, empty = unbounded
|
||||
DocURL string
|
||||
BillingMode string // payg | subscription_equivalent
|
||||
Notes string
|
||||
Fingerprint string // filled by Register
|
||||
// Peak is what the same tokens cost inside Window. Nil when the vendor
|
||||
// bills one rate around the clock, and then the fields above are the rate.
|
||||
Peak *RateCard
|
||||
Window *PeakWindow
|
||||
// CheckedOn is when a person last read DocURL and confirmed these numbers.
|
||||
// No vendor here serves prices over an API, so this date is all that stands
|
||||
// between a price change and a readout that is quietly wrong.
|
||||
CheckedOn string
|
||||
}
|
||||
|
||||
// deepseekPeak is the schedule DeepSeek publishes: peak on weekday mornings and
|
||||
// afternoons, and — since 2026-08-23 — never at a weekend. Chinese public
|
||||
// holidays bill off-peak in full, per the same page.
|
||||
var deepseekPeak = &PeakWindow{
|
||||
OffsetSeconds: beijing,
|
||||
Hours: [][2]int{{9, 12}, {14, 18}},
|
||||
WeekendOffPeak: "2026-08-23",
|
||||
HolidayOffPeak: chineseStatutoryHolidays,
|
||||
}
|
||||
|
||||
// chineseStatutoryHolidays are the State Council's 2026 public holidays, which
|
||||
// DeepSeek bills off-peak in full; extend it when the next notice is out. Make-up
|
||||
// workdays are absent: they fall on weekends, which are off-peak anyway.
|
||||
var chineseStatutoryHolidays = []string{
|
||||
// New Year's Day: Jan 1-3.
|
||||
"2026-01-01", "2026-01-02", "2026-01-03",
|
||||
// Spring Festival: Feb 15-23.
|
||||
"2026-02-15", "2026-02-16", "2026-02-17", "2026-02-18", "2026-02-19",
|
||||
"2026-02-20", "2026-02-21", "2026-02-22", "2026-02-23",
|
||||
// Qingming Festival: Apr 4-6.
|
||||
"2026-04-04", "2026-04-05", "2026-04-06",
|
||||
// Labor Day: May 1-5.
|
||||
"2026-05-01", "2026-05-02", "2026-05-03", "2026-05-04", "2026-05-05",
|
||||
// Dragon Boat Festival: Jun 19-21.
|
||||
"2026-06-19", "2026-06-20", "2026-06-21",
|
||||
// Mid-Autumn Festival: Sep 25-27.
|
||||
"2026-09-25", "2026-09-26", "2026-09-27",
|
||||
// National Day: Oct 1-7.
|
||||
"2026-10-01", "2026-10-02", "2026-10-03", "2026-10-04",
|
||||
"2026-10-05", "2026-10-06", "2026-10-07",
|
||||
}
|
||||
|
||||
// checkedOn is when the tables below were last read off the vendors' pages.
|
||||
const checkedOn = "2026-08-22"
|
||||
|
||||
// deepseekCheckedOn is when the DeepSeek rows were last read off that vendor's
|
||||
// page. Separate from checkedOn, because confirming one vendor's prices says
|
||||
// nothing about another's and a shared date would claim that it did.
|
||||
const deepseekCheckedOn = "2026-09-13"
|
||||
|
||||
// mimoCheckedOn is when the V2.6 rows were read off the vendor's own docs.
|
||||
const mimoCheckedOn = "2026-10-06"
|
||||
|
||||
// googleCheckedOn is when the Gemini rows were read off Google's pricing page.
|
||||
const googleCheckedOn = "2026-10-17"
|
||||
|
||||
// anthropicCheckedOn is when the Claude rows were read off Anthropic's pricing page.
|
||||
const anthropicCheckedOn = "2026-10-19"
|
||||
|
||||
// CatalogSourceURLs document public pricing pages.
|
||||
const (
|
||||
DocDeepSeekPricing = "https://api-docs.deepseek.com/quick_start/pricing"
|
||||
DocDeepSeekPricingCNY = "https://api-docs.deepseek.com/zh-cn/quick_start/pricing"
|
||||
DocLongCatPricingUSD = "https://longcat.chat/platform/docs/Pricing/LongCat-2.0.html"
|
||||
DocLongCatPricingCNY = "https://longcat.chat/platform/docs/zh/pricing/long-cat-2.0"
|
||||
DocMiMoPAYG = "https://mimo.mi.com/docs/price/pay-as-you-go"
|
||||
DocMiMoTokenPlan = "https://platform.xiaomimimo.com/token-plan"
|
||||
DocGeminiPricing = "https://ai.google.dev/gemini-api/docs/pricing"
|
||||
DocAnthropicPricing = "https://www.anthropic.com/pricing"
|
||||
)
|
||||
|
||||
// officialEndpointHosts maps a vendor's own API hostnames to the catalog
|
||||
// provider whose list prices apply there. Host equality is the test: a reseller
|
||||
// or proxy bills on its own terms even when it serves the same model, and an
|
||||
// entry merely named after a vendor is not evidence of the vendor's endpoint.
|
||||
var officialEndpointHosts = map[string]string{
|
||||
"api.anthropic.com": "anthropic",
|
||||
"api.deepseek.com": "deepseek",
|
||||
"api.longcat.chat": "longcat",
|
||||
"api.xiaomimimo.com": "mimo",
|
||||
"generativelanguage.googleapis.com": "google",
|
||||
"token-plan-cn.xiaomimimo.com": "mimo",
|
||||
"token-plan-sgp.xiaomimimo.com": "mimo",
|
||||
"token-plan-ams.xiaomimimo.com": "mimo",
|
||||
}
|
||||
|
||||
// OfficialProviderForEndpoint returns the catalog provider whose official
|
||||
// prices apply to baseURL, or "" when the address is not a vendor's own API.
|
||||
func OfficialProviderForEndpoint(baseURL string) string {
|
||||
u, err := url.Parse(strings.TrimSpace(baseURL))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return officialEndpointHosts[strings.ToLower(u.Hostname())]
|
||||
}
|
||||
|
||||
// OfficialCatalog is the built-in price book: the current generation of every
|
||||
// rate in officialRates, fingerprinted so a stored price can be recognised as
|
||||
// one of ours. User-custom prices always win over it.
|
||||
func OfficialCatalog() []CatalogEntry {
|
||||
entries := officialCatalogRows()
|
||||
for i := range entries {
|
||||
entries[i].Fingerprint = PricingFingerprint(RateCardFromCatalog(entries[i]))
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
// LookupCatalog finds an official entry. billingMode empty matches payg first.
|
||||
func LookupCatalog(provider, model, currency, billingMode string) (CatalogEntry, bool) {
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
model = strings.TrimSpace(model)
|
||||
currency = NormalizeCurrency(currency)
|
||||
billingMode = strings.TrimSpace(billingMode)
|
||||
if billingMode == "" {
|
||||
billingMode = BillingModePAYG
|
||||
}
|
||||
var fallback CatalogEntry
|
||||
foundFallback := false
|
||||
for _, e := range OfficialCatalog() {
|
||||
if e.Provider != provider || e.Model != model {
|
||||
continue
|
||||
}
|
||||
if NormalizeCurrency(e.Currency) != currency {
|
||||
continue
|
||||
}
|
||||
if e.BillingMode == billingMode {
|
||||
return e, true
|
||||
}
|
||||
if e.BillingMode == BillingModePAYG {
|
||||
fallback = e
|
||||
foundFallback = true
|
||||
}
|
||||
}
|
||||
if foundFallback {
|
||||
return fallback, true
|
||||
}
|
||||
return CatalogEntry{}, false
|
||||
}
|
||||
|
||||
// RateCardFromCatalog builds a RateCard from a catalog entry's base rate.
|
||||
func RateCardFromCatalog(e CatalogEntry) RateCard {
|
||||
return RateCard{
|
||||
CacheHit: e.CacheHit,
|
||||
Input: e.Input,
|
||||
Output: e.Output,
|
||||
CacheWrite: e.CacheWrite,
|
||||
Currency: e.Currency,
|
||||
}
|
||||
}
|
||||
|
||||
// RatesAt is what this entry charges at t: the peak card inside its window, the
|
||||
// base card everywhere else. An entry without a window charges one rate, and
|
||||
// then t does not matter.
|
||||
func (e CatalogEntry) RatesAt(t time.Time) RateCard {
|
||||
if e.Peak == nil || !e.Window.IsPeak(t) {
|
||||
return RateCardFromCatalog(e)
|
||||
}
|
||||
peak := *e.Peak
|
||||
peak.Currency = e.Currency
|
||||
return peak
|
||||
}
|
||||
|
||||
// InEffect reports whether this entry's dates cover the day t falls on.
|
||||
func (e CatalogEntry) InEffect(t time.Time) bool {
|
||||
day := t.UTC().Format("2006-01-02")
|
||||
if e.EffectiveFrom != "" && day < e.EffectiveFrom {
|
||||
return false
|
||||
}
|
||||
return e.EffectiveTo == "" || day < e.EffectiveTo
|
||||
}
|
||||
|
||||
// MatchesCatalog reports whether rates equal a known official entry.
|
||||
func MatchesCatalog(provider, model string, rates RateCard) (CatalogEntry, bool) {
|
||||
cur := NormalizeCurrency(rates.Currency)
|
||||
for _, e := range OfficialCatalog() {
|
||||
if e.Provider != strings.ToLower(strings.TrimSpace(provider)) {
|
||||
continue
|
||||
}
|
||||
if e.Model != strings.TrimSpace(model) {
|
||||
continue
|
||||
}
|
||||
if NormalizeCurrency(e.Currency) != cur {
|
||||
continue
|
||||
}
|
||||
// Either card identifies the vendor's own price. Which one a user
|
||||
// happens to have configured says nothing about when they are billing:
|
||||
// the hour decides that, and RatesAt is what reads it.
|
||||
if sameCard(RateCardFromCatalog(e), rates) {
|
||||
return e, true
|
||||
}
|
||||
if p := e.Peak; p != nil && sameCard(*p, rates) {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return CatalogEntry{}, false
|
||||
}
|
||||
-799
@@ -1,799 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Billing modes.
|
||||
const (
|
||||
BillingModePAYG = "payg"
|
||||
BillingModeSubscriptionEquivalent = "subscription_equivalent"
|
||||
)
|
||||
|
||||
// Valuation basis values.
|
||||
const (
|
||||
BasisIdentity = "identity"
|
||||
BasisOfficialTable = "official_table"
|
||||
// BasisFX is retained for decoding pre-v6 persisted quotes only. New
|
||||
// quotes never create FX valuations.
|
||||
BasisFX = "fx"
|
||||
)
|
||||
|
||||
// DisplayStatus describes whether a quote can satisfy its requested display.
|
||||
// It is deliberately separate from CostComplete: a known price-book amount
|
||||
// can exist even when no requested regional valuation is available.
|
||||
const (
|
||||
DisplayStatusMatched = "matched"
|
||||
DisplayStatusFallbackOriginal = "fallback_original"
|
||||
DisplayStatusBucketed = "bucketed"
|
||||
DisplayStatusUnavailable = "unavailable"
|
||||
)
|
||||
|
||||
// Aggregate modes describe how a session/run total was formed.
|
||||
const (
|
||||
AggregateModeSingleCurrency = "single_currency"
|
||||
AggregateModeCommonValuation = "common_valuation"
|
||||
AggregateModeCurrencyBuckets = "currency_buckets"
|
||||
)
|
||||
|
||||
// Rate bands name which side of a vendor's peak window a call was billed on.
|
||||
// An aggregate is mixed when its calls fell on both sides.
|
||||
const (
|
||||
RateBandPeak = "peak"
|
||||
RateBandOffPeak = "off_peak"
|
||||
RateBandMixed = "mixed"
|
||||
)
|
||||
|
||||
// DisplayRequest identifies how a caller selected the requested presentation
|
||||
// currency. The source is runtime context only; it is never persisted as a
|
||||
// provider price or wallet fact.
|
||||
type DisplayRequest struct {
|
||||
Currency string `json:"currency,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
DisplaySourceExplicit = "explicit"
|
||||
DisplaySourceWallet = "wallet"
|
||||
DisplaySourceAuto = "auto"
|
||||
)
|
||||
|
||||
func (r DisplayRequest) NormalizedCurrency() string {
|
||||
return NormalizeCurrency(r.Currency)
|
||||
}
|
||||
|
||||
// CostQuote is the host-side quoted cost for one model call (or an aggregate).
|
||||
// Original is the pricing-table currency fact; valuations hold identity or
|
||||
// official regional rate-card estimates. Selected is the display pick for the
|
||||
// current preference.
|
||||
type CostQuote struct {
|
||||
Original Money `json:"original"`
|
||||
// OriginalTotals is populated for mixed-currency aggregates. Individual
|
||||
// quotes and single-currency totals may omit it.
|
||||
OriginalTotals []Money `json:"originalTotals,omitempty"`
|
||||
Valuations map[string]Valuation `json:"valuations,omitempty"`
|
||||
Selected *Money `json:"selected,omitempty"`
|
||||
BillingMode string `json:"billingMode,omitempty"`
|
||||
Estimated bool `json:"estimated"`
|
||||
// CostComplete means usage and the price-book amount are known. It does not
|
||||
// imply that a requested display currency is available.
|
||||
CostComplete bool `json:"costComplete"`
|
||||
// Coverage is the same fact with the states a boolean cannot hold: an
|
||||
// aggregate that priced only some of its calls, and one that billed nothing.
|
||||
Coverage string `json:"coverage,omitempty"`
|
||||
// DisplayComplete means a single selected amount satisfies the display
|
||||
// request. Complete is retained as the old wire alias.
|
||||
DisplayComplete bool `json:"displayComplete"`
|
||||
Complete bool `json:"complete"`
|
||||
DisplayStatus string `json:"displayStatus,omitempty"`
|
||||
AggregateMode string `json:"aggregateMode,omitempty"`
|
||||
// ModelRef / UsageSource / PricingFingerprint identify the ledger key.
|
||||
ModelRef string `json:"modelRef,omitempty"`
|
||||
UsageSource string `json:"usageSource,omitempty"`
|
||||
PricingFingerprint string `json:"pricingFingerprint,omitempty"`
|
||||
RateDate string `json:"rateDate,omitempty"` // YYYY-MM-DD of FX used, if any
|
||||
IncompleteReason string `json:"incompleteReason,omitempty"`
|
||||
LegacyEstimate bool `json:"legacyEstimate,omitempty"`
|
||||
CatalogSource string `json:"catalogSource,omitempty"`
|
||||
// RateBand is set only for a rate confirmed as a vendor's own schedule; an
|
||||
// aggregate that includes any call without one carries none.
|
||||
RateBand string `json:"rateBand,omitempty"`
|
||||
}
|
||||
|
||||
// Valuation is one currency view of a cost fact.
|
||||
type Valuation struct {
|
||||
Money Money `json:"money"`
|
||||
Basis string `json:"basis"`
|
||||
Source string `json:"source"`
|
||||
AsOf string `json:"asOf"` // YYYY-MM-DD
|
||||
Rate *RateSnapshot `json:"rateSnapshot,omitempty"`
|
||||
Stale bool `json:"stale,omitempty"`
|
||||
}
|
||||
|
||||
// RateSnapshot records the FX observation used for a valuation.
|
||||
type RateSnapshot struct {
|
||||
Base string `json:"base"`
|
||||
Quote string `json:"quote"`
|
||||
Rate float64 `json:"rate"`
|
||||
Source string `json:"source"`
|
||||
AsOf string `json:"asOf"`
|
||||
Stale bool `json:"stale,omitempty"`
|
||||
}
|
||||
|
||||
// RateCard is the per-1M-token price used to compute original cost. It mirrors
|
||||
// provider.Pricing without importing that package (billing is a leaf).
|
||||
type RateCard struct {
|
||||
CacheHit float64 // per 1M cached prompt tokens
|
||||
Input float64 // per 1M uncached prompt tokens
|
||||
Output float64 // per 1M completion tokens
|
||||
// CacheWrite is per 1M tokens written to the cache; zero bills them at Input.
|
||||
CacheWrite float64
|
||||
Currency string // ISO or symbol; normalized on quote
|
||||
}
|
||||
|
||||
// UsageTokens is the token breakdown needed for cost. Mirrors provider.Usage
|
||||
// fields without importing provider.
|
||||
type UsageTokens struct {
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
CacheHitTokens int
|
||||
CacheMissTokens int
|
||||
CacheWriteTokens int
|
||||
CacheWriteBilledTokens float64
|
||||
Estimated bool
|
||||
}
|
||||
|
||||
// QuoteInput drives a single CostQuote computation.
|
||||
type QuoteInput struct {
|
||||
Usage UsageTokens
|
||||
Rates RateCard
|
||||
OccurredAt time.Time
|
||||
DisplayCurrency string // compatibility alias for Display.Currency
|
||||
Display DisplayRequest // runtime display request; empty means auto
|
||||
BillingMode string
|
||||
ModelRef string
|
||||
UsageSource string
|
||||
CatalogSource string
|
||||
PricingFingerprint string
|
||||
// ProviderKind is deepseek|longcat|mimo|… for official dual-table lookup.
|
||||
// When empty, ModelRef and Rates are used to infer a catalog match.
|
||||
ProviderKind string
|
||||
// ModelID is the bare model id (e.g. deepseek-v4-flash). Empty → parse ModelRef.
|
||||
ModelID string
|
||||
}
|
||||
|
||||
// OriginalCostAmount computes the fixed-point original cost from rates + tokens.
|
||||
// Mirrors the historic provider.Pricing.Cost semantics.
|
||||
func OriginalCostAmount(rates RateCard, u UsageTokens) Amount {
|
||||
hit := u.CacheHitTokens
|
||||
miss := u.CacheMissTokens
|
||||
if hit+miss == 0 && u.PromptTokens > 0 {
|
||||
miss = u.PromptTokens
|
||||
} else if miss == 0 && hit > 0 && u.PromptTokens > hit {
|
||||
miss = u.PromptTokens - hit
|
||||
}
|
||||
write := u.CacheWriteTokens
|
||||
write = max(write, 0)
|
||||
write = min(write, miss)
|
||||
billedWrite := 0.0
|
||||
if write > 0 {
|
||||
billedWrite = u.CacheWriteBilledTokens
|
||||
if billedWrite <= 0 {
|
||||
billedWrite = float64(write)
|
||||
}
|
||||
}
|
||||
writeRate := rates.Input
|
||||
if rates.CacheWrite > 0 {
|
||||
writeRate = rates.CacheWrite
|
||||
}
|
||||
total := (float64(hit)*rates.CacheHit +
|
||||
float64(miss-write)*rates.Input +
|
||||
billedWrite*writeRate +
|
||||
float64(u.CompletionTokens)*rates.Output) / 1e6
|
||||
return NewAmountFromFloat(total)
|
||||
}
|
||||
|
||||
// PricingFingerprint hashes a rate card for ledger keys.
|
||||
func PricingFingerprint(rates RateCard) string {
|
||||
cur := NormalizeCurrency(rates.Currency)
|
||||
raw := fmt.Sprintf("%s|%.12g|%.12g|%.12g", cur, rates.CacheHit, rates.Input, rates.Output)
|
||||
if rates.CacheWrite != 0 {
|
||||
// Appended only when set, so every card stored before the field keeps its key.
|
||||
raw += fmt.Sprintf("|w%.12g", rates.CacheWrite)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(raw))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// BuildQuote computes original cost and CNY/USD valuations.
|
||||
func BuildQuote(in QuoteInput) CostQuote {
|
||||
state := newQuoteBuildState(in)
|
||||
for _, target := range []string{"CNY", "USD"} {
|
||||
state.addTargetValuation(target)
|
||||
}
|
||||
state.selectDisplay()
|
||||
return state.quote
|
||||
}
|
||||
|
||||
type quoteBuildState struct {
|
||||
input QuoteInput
|
||||
currency string
|
||||
amount Amount
|
||||
occurred time.Time
|
||||
official CatalogEntry
|
||||
isOfficial bool
|
||||
valuationFailures map[string]string
|
||||
quote CostQuote
|
||||
}
|
||||
|
||||
func newQuoteBuildState(in QuoteInput) *quoteBuildState {
|
||||
currency := NormalizeCurrency(in.Rates.Currency)
|
||||
if currency == "" {
|
||||
currency = "CNY"
|
||||
}
|
||||
mode := strings.TrimSpace(in.BillingMode)
|
||||
if mode == "" {
|
||||
mode = BillingModePAYG
|
||||
}
|
||||
occurred := in.OccurredAt
|
||||
if occurred.IsZero() {
|
||||
occurred = time.Now().UTC()
|
||||
}
|
||||
fingerprint := strings.TrimSpace(in.PricingFingerprint)
|
||||
if fingerprint == "" {
|
||||
fingerprint = PricingFingerprint(in.Rates)
|
||||
}
|
||||
amount := OriginalCostAmount(in.Rates, in.Usage)
|
||||
q := CostQuote{
|
||||
Original: MoneyOf(amount, currency),
|
||||
Valuations: map[string]Valuation{},
|
||||
BillingMode: mode,
|
||||
Estimated: true,
|
||||
CostComplete: true,
|
||||
Coverage: CoverageComplete,
|
||||
DisplayComplete: true,
|
||||
Complete: true,
|
||||
DisplayStatus: DisplayStatusMatched,
|
||||
AggregateMode: AggregateModeSingleCurrency,
|
||||
ModelRef: strings.TrimSpace(in.ModelRef),
|
||||
UsageSource: strings.TrimSpace(in.UsageSource),
|
||||
PricingFingerprint: fingerprint,
|
||||
CatalogSource: strings.TrimSpace(in.CatalogSource),
|
||||
}
|
||||
q.Valuations[currency] = Valuation{
|
||||
Money: q.Original, Basis: BasisIdentity,
|
||||
Source: firstNonEmpty(in.CatalogSource, "rate_card"),
|
||||
AsOf: occurred.UTC().Format("2006-01-02"),
|
||||
}
|
||||
state := "eBuildState{
|
||||
input: in, currency: currency, amount: amount, occurred: occurred,
|
||||
valuationFailures: map[string]string{}, quote: q,
|
||||
}
|
||||
if !usageHasFacts(in.Usage) {
|
||||
state.markIncomplete("missing_price_or_usage")
|
||||
}
|
||||
state.matchOfficialCatalog()
|
||||
state.applyOfficialSchedule()
|
||||
return state
|
||||
}
|
||||
|
||||
// applyOfficialSchedule re-prices the turn at the vendor's rate for the hour it
|
||||
// actually happened. Only a rate confirmed as the vendor's own earns this: a
|
||||
// custom price is the user's own arrangement, and projecting someone else's
|
||||
// peak schedule onto it would invent a number nobody published.
|
||||
func (s *quoteBuildState) applyOfficialSchedule() {
|
||||
if !s.isOfficial || s.official.Peak == nil {
|
||||
return
|
||||
}
|
||||
s.quote.RateBand = RateBandOffPeak
|
||||
if s.official.Window.IsPeak(s.occurred) {
|
||||
s.quote.RateBand = RateBandPeak
|
||||
}
|
||||
rates := s.official.RatesAt(s.occurred)
|
||||
if rates.CacheHit == s.input.Rates.CacheHit && rates.Input == s.input.Rates.Input && rates.Output == s.input.Rates.Output {
|
||||
return
|
||||
}
|
||||
s.amount = OriginalCostAmount(rates, s.input.Usage)
|
||||
s.quote.Original = MoneyOf(s.amount, s.currency)
|
||||
s.quote.PricingFingerprint = PricingFingerprint(rates)
|
||||
s.quote.Valuations[s.currency] = Valuation{
|
||||
Money: s.quote.Original, Basis: BasisOfficialTable,
|
||||
Source: s.official.DocURL,
|
||||
AsOf: s.occurred.UTC().Format("2006-01-02"),
|
||||
}
|
||||
}
|
||||
|
||||
func quoteDisplayRequest(in QuoteInput) DisplayRequest {
|
||||
request := in.Display
|
||||
if request.Currency == "" && in.DisplayCurrency != "" {
|
||||
request.Currency = in.DisplayCurrency
|
||||
if request.Source == "" {
|
||||
request.Source = DisplaySourceExplicit
|
||||
}
|
||||
}
|
||||
if request.Source == "" {
|
||||
request.Source = DisplaySourceAuto
|
||||
}
|
||||
return request
|
||||
}
|
||||
|
||||
func usageHasFacts(u UsageTokens) bool {
|
||||
return u.PromptTokens > 0 || u.CompletionTokens > 0 || u.CacheHitTokens > 0 ||
|
||||
u.CacheMissTokens > 0 || u.CacheWriteTokens > 0 || u.CacheWriteBilledTokens > 0
|
||||
}
|
||||
|
||||
func (s *quoteBuildState) matchOfficialCatalog() {
|
||||
providerKind, modelID := resolveCatalogIdentity(s.input)
|
||||
s.official, s.isOfficial = MatchesCatalog(providerKind, modelID, s.input.Rates)
|
||||
if !s.isOfficial && providerKind != "" && modelID != "" {
|
||||
rates := s.input.Rates
|
||||
rates.Currency = s.currency
|
||||
s.official, s.isOfficial = MatchesCatalog(providerKind, modelID, rates)
|
||||
}
|
||||
if s.isOfficial && s.quote.CatalogSource == "" {
|
||||
s.quote.CatalogSource = s.official.DocURL
|
||||
}
|
||||
}
|
||||
|
||||
func (s *quoteBuildState) addTargetValuation(target string) {
|
||||
if target == s.currency {
|
||||
return
|
||||
}
|
||||
if s.addOfficialValuation(target) {
|
||||
return
|
||||
}
|
||||
// Runtime FX has intentionally been removed. A custom price that does not
|
||||
// match the official catalog remains in its original currency.
|
||||
s.valuationFailures[target] = "display_unavailable"
|
||||
}
|
||||
|
||||
func (s *quoteBuildState) addOfficialValuation(target string) bool {
|
||||
if !s.isOfficial {
|
||||
return false
|
||||
}
|
||||
peer, ok := LookupCatalog(
|
||||
s.official.Provider, s.official.Model, target, s.official.BillingMode,
|
||||
)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
s.quote.Valuations[target] = Valuation{
|
||||
Money: MoneyOf(OriginalCostAmount(peer.RatesAt(s.occurred), s.input.Usage), target),
|
||||
Basis: BasisOfficialTable, Source: peer.DocURL,
|
||||
AsOf: s.occurred.UTC().Format("2006-01-02"),
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *quoteBuildState) selectDisplay() {
|
||||
if !s.quote.CostComplete {
|
||||
s.quote.Selected = nil
|
||||
s.quote.DisplayComplete = false
|
||||
s.quote.Complete = false
|
||||
s.quote.DisplayStatus = DisplayStatusUnavailable
|
||||
return
|
||||
}
|
||||
display := quoteDisplayRequest(s.input).NormalizedCurrency()
|
||||
if display == "" {
|
||||
selected := s.quote.Original
|
||||
s.quote.Selected = &selected
|
||||
return
|
||||
}
|
||||
if valuation, ok := s.quote.Valuations[display]; ok {
|
||||
selected := valuation.Money
|
||||
s.quote.Selected = &selected
|
||||
return
|
||||
}
|
||||
// The original price-book amount is still a valid cost fact. Keep it as a
|
||||
// visible fallback and distinguish the display mismatch from no pricing.
|
||||
selected := s.quote.Original
|
||||
s.quote.Selected = &selected
|
||||
s.quote.DisplayComplete = false
|
||||
s.quote.Complete = false
|
||||
s.quote.DisplayStatus = DisplayStatusFallbackOriginal
|
||||
s.quote.IncompleteReason = firstNonEmpty(s.valuationFailures[display], "display_unavailable")
|
||||
}
|
||||
|
||||
func (s *quoteBuildState) markIncomplete(reason string) {
|
||||
s.quote.CostComplete = false
|
||||
s.quote.Coverage = CoverageIncomplete
|
||||
s.quote.DisplayComplete = false
|
||||
s.quote.Complete = false
|
||||
s.quote.DisplayStatus = DisplayStatusUnavailable
|
||||
if s.quote.IncompleteReason == "" {
|
||||
s.quote.IncompleteReason = reason
|
||||
}
|
||||
}
|
||||
|
||||
// SelectForDisplay returns the money for a display currency preference without
|
||||
// recomputing rates. Empty display returns original.
|
||||
func (q CostQuote) SelectForDisplay(display string) (Money, bool) {
|
||||
display = NormalizeCurrency(display)
|
||||
if display == "" {
|
||||
return q.Original, true
|
||||
}
|
||||
if v, ok := q.Valuations[display]; ok {
|
||||
return v.Money, true
|
||||
}
|
||||
if NormalizeCurrency(q.Original.Currency) == display {
|
||||
return q.Original, true
|
||||
}
|
||||
return Money{}, false
|
||||
}
|
||||
|
||||
// WithSelected returns a copy with Selected set for display.
|
||||
func (q CostQuote) WithSelected(display string) CostQuote {
|
||||
out := q
|
||||
if m, ok := q.SelectForDisplay(display); ok {
|
||||
out.Selected = &m
|
||||
if quoteHasCompleteCostFact(q) {
|
||||
out.Complete = true
|
||||
out.IncompleteReason = ""
|
||||
}
|
||||
} else {
|
||||
out.Selected = nil
|
||||
out.Complete = false
|
||||
if out.IncompleteReason == "" {
|
||||
out.IncompleteReason = "display_unavailable"
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// LegacyCostFloat returns the selected (or original) amount as float64 for
|
||||
// compatibility aliases cost / costUsd / total_cost.
|
||||
func (q CostQuote) LegacyCostFloat() float64 {
|
||||
if q.Selected != nil {
|
||||
return q.Selected.Float64()
|
||||
}
|
||||
return q.Original.Float64()
|
||||
}
|
||||
|
||||
// LegacyCurrencySymbol returns a display symbol for the selected/original currency.
|
||||
func (q CostQuote) LegacyCurrencySymbol() string {
|
||||
if q.Selected != nil && q.Selected.Currency != "" {
|
||||
return CurrencySymbol(q.Selected.Currency)
|
||||
}
|
||||
return CurrencySymbol(q.Original.Currency)
|
||||
}
|
||||
|
||||
// LegacyCurrencyCode returns the ISO code for selected/original.
|
||||
func (q CostQuote) LegacyCurrencyCode() string {
|
||||
if q.Selected != nil && q.Selected.Currency != "" {
|
||||
return NormalizeCurrency(q.Selected.Currency)
|
||||
}
|
||||
return NormalizeCurrency(q.Original.Currency)
|
||||
}
|
||||
|
||||
// AggregateQuotes combines occurrence-time quotes into a session/run total.
|
||||
// Every entry must have the requested valuation; partial totals stay unavailable.
|
||||
func AggregateQuotes(quotes []CostQuote, display string) CostQuote {
|
||||
if len(quotes) == 0 {
|
||||
return emptyAggregate(NormalizeCurrency(display))
|
||||
}
|
||||
accumulator := newQuoteAccumulator(display)
|
||||
for _, quote := range quotes {
|
||||
accumulator.add(quote)
|
||||
}
|
||||
return accumulator.finish()
|
||||
}
|
||||
|
||||
type quoteAccumulator struct {
|
||||
out CostQuote
|
||||
display string
|
||||
totals map[string]Amount
|
||||
originalCurrency string
|
||||
originalTotal Amount
|
||||
originalTotals map[string]Amount
|
||||
originalComplete bool
|
||||
coverage string
|
||||
displayComplete bool
|
||||
modes map[string]struct{}
|
||||
rateBands map[string]struct{}
|
||||
unbanded bool
|
||||
}
|
||||
|
||||
func emptyAggregate(display string) CostQuote {
|
||||
out := CostQuote{
|
||||
Original: MoneyOf(Zero, display), Valuations: map[string]Valuation{}, Estimated: true,
|
||||
CostComplete: false, DisplayComplete: false, Complete: false,
|
||||
Coverage: CoverageNone,
|
||||
DisplayStatus: DisplayStatusUnavailable, AggregateMode: AggregateModeSingleCurrency,
|
||||
IncompleteReason: "no_usage",
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func newQuoteAccumulator(display string) *quoteAccumulator {
|
||||
return "eAccumulator{
|
||||
out: CostQuote{Valuations: map[string]Valuation{}, Estimated: true},
|
||||
display: NormalizeCurrency(display), totals: map[string]Amount{}, originalTotals: map[string]Amount{},
|
||||
originalComplete: true, coverage: CoverageNone, displayComplete: true,
|
||||
modes: map[string]struct{}{}, rateBands: map[string]struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) add(quote CostQuote) {
|
||||
quote = NormalizeQuote(quote)
|
||||
a.coverage = FoldCoverage(a.coverage, quote.Coverage)
|
||||
if quote.Coverage != CoverageComplete && a.out.IncompleteReason == "" {
|
||||
a.out.IncompleteReason = quote.IncompleteReason
|
||||
}
|
||||
a.out.LegacyEstimate = a.out.LegacyEstimate || quote.LegacyEstimate
|
||||
if quote.BillingMode != "" {
|
||||
a.modes[quote.BillingMode] = struct{}{}
|
||||
}
|
||||
a.addRateBand(quote.RateBand)
|
||||
if quote.RateDate > a.out.RateDate {
|
||||
a.out.RateDate = quote.RateDate
|
||||
}
|
||||
originalCurrency := NormalizeCurrency(quote.Original.Currency)
|
||||
if len(quote.OriginalTotals) > 0 {
|
||||
for _, original := range quote.OriginalTotals {
|
||||
a.addOriginal(original, NormalizeCurrency(original.Currency))
|
||||
}
|
||||
} else {
|
||||
a.addOriginal(quote.Original, originalCurrency)
|
||||
}
|
||||
if a.display != "" {
|
||||
_, found := quote.Valuations[a.display]
|
||||
a.displayComplete = a.displayComplete && (found || originalCurrency == a.display)
|
||||
}
|
||||
originalIncluded := false
|
||||
for code, valuation := range quote.Valuations {
|
||||
code = NormalizeCurrency(code)
|
||||
if code == originalCurrency {
|
||||
originalIncluded = true
|
||||
}
|
||||
a.addValuation(code, valuation)
|
||||
}
|
||||
if originalCurrency != "" && !originalIncluded {
|
||||
a.addValuation(originalCurrency, Valuation{
|
||||
Money: quote.Original, Basis: BasisIdentity, Source: "aggregate",
|
||||
AsOf: time.Now().UTC().Format("2006-01-02"),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) addRateBand(band string) {
|
||||
switch band {
|
||||
case RateBandPeak, RateBandOffPeak:
|
||||
a.rateBands[band] = struct{}{}
|
||||
case RateBandMixed:
|
||||
a.rateBands[RateBandPeak], a.rateBands[RateBandOffPeak] = struct{}{}, struct{}{}
|
||||
default:
|
||||
a.unbanded = true
|
||||
}
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) rateBand() string {
|
||||
if a.unbanded {
|
||||
return ""
|
||||
}
|
||||
switch len(a.rateBands) {
|
||||
case 1:
|
||||
for band := range a.rateBands {
|
||||
return band
|
||||
}
|
||||
case 2:
|
||||
return RateBandMixed
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) addOriginal(original Money, currency string) {
|
||||
if currency != "" {
|
||||
a.originalTotals[currency] = a.originalTotals[currency].Add(original.AmountValue())
|
||||
}
|
||||
if a.originalCurrency == "" {
|
||||
a.originalCurrency = currency
|
||||
a.originalTotal = original.AmountValue()
|
||||
return
|
||||
}
|
||||
if currency != a.originalCurrency {
|
||||
a.originalComplete = false
|
||||
return
|
||||
}
|
||||
a.originalTotal = a.originalTotal.Add(original.AmountValue())
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) addValuation(code string, valuation Valuation) {
|
||||
if code == "" {
|
||||
return
|
||||
}
|
||||
a.totals[code] = a.totals[code].Add(valuation.Money.AmountValue())
|
||||
current, found := a.out.Valuations[code]
|
||||
if !found {
|
||||
current = valuation
|
||||
}
|
||||
current.Money = MoneyOf(a.totals[code], code)
|
||||
current.Stale = current.Stale || valuation.Stale
|
||||
a.out.Valuations[code] = current
|
||||
}
|
||||
|
||||
// costFactsComplete reads the accumulated coverage rather than tracking the
|
||||
// same fact a second time: two answers to one question drift.
|
||||
func (a *quoteAccumulator) costFactsComplete() bool {
|
||||
return a.coverage == CoverageComplete
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) finish() CostQuote {
|
||||
a.out.Coverage = a.coverage
|
||||
a.out.CostComplete = a.costFactsComplete()
|
||||
if a.originalComplete && a.originalCurrency != "" {
|
||||
a.out.Original = MoneyOf(a.originalTotal, a.originalCurrency)
|
||||
a.syncOriginalValuation()
|
||||
} else {
|
||||
a.out.Original = Money{Amount: "0"}
|
||||
}
|
||||
if len(a.originalTotals) > 1 {
|
||||
codes := make([]string, 0, len(a.originalTotals))
|
||||
for code := range a.originalTotals {
|
||||
codes = append(codes, code)
|
||||
}
|
||||
slices.Sort(codes)
|
||||
for _, code := range codes {
|
||||
a.out.OriginalTotals = append(a.out.OriginalTotals, MoneyOf(a.originalTotals[code], code))
|
||||
}
|
||||
}
|
||||
a.out.RateBand = a.rateBand()
|
||||
if len(a.modes) == 1 {
|
||||
for mode := range a.modes {
|
||||
a.out.BillingMode = mode
|
||||
}
|
||||
}
|
||||
if a.display == "" && a.originalComplete && a.costFactsComplete() {
|
||||
selected := a.out.Original
|
||||
a.out.Selected = &selected
|
||||
a.out.DisplayComplete = true
|
||||
a.out.Complete = true
|
||||
a.out.DisplayStatus = DisplayStatusMatched
|
||||
a.out.AggregateMode = AggregateModeSingleCurrency
|
||||
return a.out
|
||||
}
|
||||
valuation, found := a.out.Valuations[a.display]
|
||||
if found && a.displayComplete && a.costFactsComplete() {
|
||||
selected := valuation.Money
|
||||
a.out.Selected = &selected
|
||||
a.out.DisplayComplete = true
|
||||
a.out.Complete = true
|
||||
a.out.DisplayStatus = DisplayStatusMatched
|
||||
if len(a.originalTotals) > 1 {
|
||||
a.out.AggregateMode = AggregateModeCommonValuation
|
||||
} else {
|
||||
a.out.AggregateMode = AggregateModeSingleCurrency
|
||||
}
|
||||
return a.out
|
||||
}
|
||||
a.out.DisplayComplete = false
|
||||
a.out.Complete = false
|
||||
if !a.costFactsComplete() {
|
||||
a.out.DisplayStatus = DisplayStatusUnavailable
|
||||
a.out.IncompleteReason = firstNonEmpty(a.out.IncompleteReason, "incomplete_cost_fact")
|
||||
} else if a.originalComplete && a.originalCurrency != "" {
|
||||
selected := a.out.Original
|
||||
a.out.Selected = &selected
|
||||
a.out.DisplayStatus = DisplayStatusFallbackOriginal
|
||||
a.out.AggregateMode = AggregateModeSingleCurrency
|
||||
a.out.IncompleteReason = firstNonEmpty(a.out.IncompleteReason, "display_unavailable")
|
||||
} else {
|
||||
a.out.DisplayStatus = DisplayStatusBucketed
|
||||
a.out.AggregateMode = AggregateModeCurrencyBuckets
|
||||
a.out.IncompleteReason = firstNonEmpty(a.out.IncompleteReason, "currency_buckets")
|
||||
}
|
||||
return a.out
|
||||
}
|
||||
|
||||
// NormalizeQuote fills fields introduced after the first CostQuote wire shape.
|
||||
// It is used at every persistence and transport boundary so old telemetry and
|
||||
// old eventwire payloads remain safe without inventing a new amount.
|
||||
func NormalizeQuote(q CostQuote) CostQuote {
|
||||
if !q.CostComplete && !q.DisplayComplete && q.Complete {
|
||||
q.CostComplete = true
|
||||
q.DisplayComplete = true
|
||||
}
|
||||
if q.DisplayStatus != "" && q.DisplayStatus != DisplayStatusMatched && q.DisplayStatus != DisplayStatusFallbackOriginal && q.DisplayStatus != DisplayStatusBucketed && q.DisplayStatus != DisplayStatusUnavailable {
|
||||
q.DisplayStatus = ""
|
||||
}
|
||||
if q.RateBand != RateBandPeak && q.RateBand != RateBandOffPeak && q.RateBand != RateBandMixed {
|
||||
q.RateBand = ""
|
||||
}
|
||||
if q.AggregateMode != "" && q.AggregateMode != AggregateModeSingleCurrency && q.AggregateMode != AggregateModeCommonValuation && q.AggregateMode != AggregateModeCurrencyBuckets {
|
||||
q.AggregateMode = ""
|
||||
}
|
||||
if q.DisplayStatus == "" {
|
||||
switch {
|
||||
case q.Complete:
|
||||
q.DisplayStatus = DisplayStatusMatched
|
||||
case q.Original.Currency != "" && q.Original.Amount != "" && q.IncompleteReason != "no_price":
|
||||
q.DisplayStatus = DisplayStatusFallbackOriginal
|
||||
default:
|
||||
q.DisplayStatus = DisplayStatusUnavailable
|
||||
}
|
||||
}
|
||||
if q.AggregateMode == "" {
|
||||
if len(q.OriginalTotals) > 1 {
|
||||
q.AggregateMode = AggregateModeCurrencyBuckets
|
||||
} else if q.Selected != nil {
|
||||
q.AggregateMode = AggregateModeSingleCurrency
|
||||
}
|
||||
}
|
||||
if !ValidCoverage(q.Coverage) {
|
||||
q.Coverage = CoverageIncomplete
|
||||
if quoteHasCompleteCostFact(q) {
|
||||
q.Coverage = CoverageComplete
|
||||
}
|
||||
}
|
||||
q.Complete = q.DisplayComplete
|
||||
return q
|
||||
}
|
||||
|
||||
// quoteHasCompleteCostFact separates a known original-currency charge from a
|
||||
// display-only valuation failure must not poison an exact original total,
|
||||
// while unpriced and unrecoverable legacy records stay incomplete in every
|
||||
// display currency.
|
||||
func quoteHasCompleteCostFact(q CostQuote) bool {
|
||||
if q.CostComplete {
|
||||
return true
|
||||
}
|
||||
switch q.IncompleteReason {
|
||||
case "no_price", "missing_price_or_usage", "legacy_unrecoverable",
|
||||
"legacy_wiped_or_zero", "legacy_invalid_amount", "mixed_original_currencies",
|
||||
"incomplete_cost_fact":
|
||||
return false
|
||||
}
|
||||
if q.Complete {
|
||||
return true
|
||||
}
|
||||
currency := NormalizeCurrency(q.Original.Currency)
|
||||
if currency == "" {
|
||||
return false
|
||||
}
|
||||
valuation, ok := q.Valuations[currency]
|
||||
return ok && SameCurrency(valuation.Money.Currency, currency)
|
||||
}
|
||||
|
||||
func (a *quoteAccumulator) syncOriginalValuation() {
|
||||
valuation, found := a.out.Valuations[a.originalCurrency]
|
||||
if !found {
|
||||
valuation = Valuation{
|
||||
Basis: BasisIdentity, Source: "aggregate",
|
||||
AsOf: time.Now().UTC().Format("2006-01-02"),
|
||||
}
|
||||
}
|
||||
valuation.Money = a.out.Original
|
||||
a.out.Valuations[a.originalCurrency] = valuation
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, v := range values {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// resolveCatalogIdentity names which official table, if any, may price a quote.
|
||||
// The vendor comes only from ProviderKind, which the caller reads off the
|
||||
// endpoint's host; guessing it from the entry's name or the model id would bill
|
||||
// a relay as the vendor it resells. Unnamed means no official table — the user's
|
||||
// own rate card, which is right rather than merely plausible.
|
||||
func resolveCatalogIdentity(in QuoteInput) (provider, model string) {
|
||||
provider = strings.ToLower(strings.TrimSpace(in.ProviderKind))
|
||||
model = strings.TrimSpace(in.ModelID)
|
||||
if model == "" {
|
||||
ref := strings.TrimSpace(in.ModelRef)
|
||||
if i := strings.LastIndex(ref, "/"); i >= 0 && i+1 < len(ref) {
|
||||
model = ref[i+1:]
|
||||
} else {
|
||||
model = ref
|
||||
}
|
||||
}
|
||||
return provider, model
|
||||
}
|
||||
-358
@@ -1,358 +0,0 @@
|
||||
// rates.go — every rate this project has shipped for a vendor's model, and the
|
||||
// two questions that asks: what it costs now, and what it used to cost.
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// rateGeneration is one published pair. Peak is nil for a generation the vendor
|
||||
// billed at a single rate around the clock.
|
||||
type rateGeneration struct {
|
||||
Base RateCard
|
||||
Peak *RateCard
|
||||
// From is the first day (YYYY-MM-DD, UTC) this generation is billed; empty
|
||||
// means from when it shipped. An announced change is appended with its date.
|
||||
From string
|
||||
}
|
||||
|
||||
// today is the day a dated generation is read against; a test replaces it.
|
||||
var today = func() string { return time.Now().UTC().Format("2006-01-02") }
|
||||
|
||||
// currentIndex is the generation in force today: the last one whose date has
|
||||
// come. Rows with no dated generation answer with their last, as before.
|
||||
func (r officialRateRow) currentIndex() int {
|
||||
day := today()
|
||||
for i := len(r.Rates) - 1; i > 0; i-- {
|
||||
if r.Rates[i].From == "" || r.Rates[i].From <= day {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// officialRateRow is one vendor's model, in one currency, under one billing
|
||||
// mode, with every rate this project has shipped for it.
|
||||
type officialRateRow struct {
|
||||
Provider string
|
||||
Model string
|
||||
Currency string
|
||||
BillingMode string
|
||||
Window *PeakWindow
|
||||
DocURL string
|
||||
Notes string
|
||||
CheckedOn string
|
||||
// Rates is oldest first; the last entry is what this costs today.
|
||||
Rates []rateGeneration
|
||||
}
|
||||
|
||||
// officialRates is the price book. A price change appends a generation rather
|
||||
// than editing one: that is what leaves the superseded rate where a stored price
|
||||
// can still be recognised as this project's own default instead of a number its
|
||||
// user chose, which is the difference between refreshing one and losing it.
|
||||
var officialRates = []officialRateRow{
|
||||
{
|
||||
Provider: "deepseek", Model: "deepseek-flash", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, Window: deepseekPeak, DocURL: DocDeepSeekPricing, CheckedOn: deepseekCheckedOn,
|
||||
Rates: []rateGeneration{
|
||||
{Base: RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}},
|
||||
{Base: RateCard{CacheHit: 0.05, Input: 1.5, Output: 4.5, Currency: "CNY"},
|
||||
Peak: &RateCard{CacheHit: 0.10, Input: 3.0, Output: 9.0, Currency: "CNY"}},
|
||||
{Base: RateCard{CacheHit: 0.02, Input: 1, Output: 4, Currency: "CNY"},
|
||||
Peak: &RateCard{CacheHit: 0.04, Input: 2, Output: 8, Currency: "CNY"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Provider: "deepseek", Model: "deepseek-flash", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, Window: deepseekPeak, DocURL: DocDeepSeekPricing, CheckedOn: deepseekCheckedOn,
|
||||
Rates: []rateGeneration{
|
||||
{Base: RateCard{CacheHit: 0.0028, Input: 0.14, Output: 0.28, Currency: "USD"}},
|
||||
{Base: RateCard{CacheHit: 0.007, Input: 0.22, Output: 0.66, Currency: "USD"},
|
||||
Peak: &RateCard{CacheHit: 0.014, Input: 0.44, Output: 1.32, Currency: "USD"}},
|
||||
{Base: RateCard{CacheHit: 0.003, Input: 0.15, Output: 0.6, Currency: "USD"},
|
||||
Peak: &RateCard{CacheHit: 0.006, Input: 0.3, Output: 1.2, Currency: "USD"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Provider: "deepseek", Model: "deepseek-v4-pro", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, Window: deepseekPeak, DocURL: DocDeepSeekPricing, CheckedOn: deepseekCheckedOn,
|
||||
Rates: []rateGeneration{
|
||||
{Base: RateCard{CacheHit: 0.025, Input: 3, Output: 6, Currency: "CNY"}},
|
||||
{Base: RateCard{CacheHit: 0.15, Input: 4.5, Output: 13.5, Currency: "CNY"},
|
||||
Peak: &RateCard{CacheHit: 0.30, Input: 9.0, Output: 27.0, Currency: "CNY"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Provider: "deepseek", Model: "deepseek-v4-pro", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, Window: deepseekPeak, DocURL: DocDeepSeekPricing, CheckedOn: deepseekCheckedOn,
|
||||
Rates: []rateGeneration{
|
||||
{Base: RateCard{CacheHit: 0.003625, Input: 0.435, Output: 0.87, Currency: "USD"}},
|
||||
{Base: RateCard{CacheHit: 0.022, Input: 0.66, Output: 1.98, Currency: "USD"},
|
||||
Peak: &RateCard{CacheHit: 0.044, Input: 1.32, Output: 3.96, Currency: "USD"}},
|
||||
},
|
||||
},
|
||||
|
||||
// LongCat's launch discount, for which the vendor publishes no end date. List
|
||||
// price is CNY 0.10/5/20, USD 0.015/0.75/2.95 — what these revert to, and why
|
||||
// CheckedOn earns its place here.
|
||||
{
|
||||
Provider: "longcat", Model: "LongCat-2.0", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocLongCatPricingCNY, Notes: "launch_discount_no_end_date", CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.04, Input: 2, Output: 8, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "longcat", Model: "LongCat-2.0", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, DocURL: DocLongCatPricingUSD, Notes: "launch_discount_no_end_date", CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.006, Input: 0.30, Output: 1.20, Currency: "USD"}}},
|
||||
},
|
||||
|
||||
// Google: the two flash models double on 2027-01-01, hence the dated generation.
|
||||
// gemini-3.1-pro-preview is the rate up to 200k prompt tokens; gemini-4-argon has no price.
|
||||
{
|
||||
Provider: "google", Model: "gemini-3.8-flash", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, DocURL: DocGeminiPricing, CheckedOn: googleCheckedOn,
|
||||
Rates: []rateGeneration{
|
||||
{Base: RateCard{CacheHit: 0.075, Input: 0.75, Output: 3.75, Currency: "USD"}},
|
||||
{From: "2027-01-01", Base: RateCard{CacheHit: 0.15, Input: 1.50, Output: 7.50, Currency: "USD"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Provider: "google", Model: "gemini-3.6-flash", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, DocURL: DocGeminiPricing, CheckedOn: googleCheckedOn,
|
||||
Rates: []rateGeneration{
|
||||
{Base: RateCard{CacheHit: 0.075, Input: 0.75, Output: 3.75, Currency: "USD"}},
|
||||
{From: "2027-01-01", Base: RateCard{CacheHit: 0.15, Input: 1.50, Output: 7.50, Currency: "USD"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Provider: "google", Model: "gemini-3.5-flash-lite", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, DocURL: DocGeminiPricing, CheckedOn: googleCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.03, Input: 0.30, Output: 2.50, Currency: "USD"}}},
|
||||
},
|
||||
{
|
||||
Provider: "google", Model: "gemini-3.1-pro-preview", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, DocURL: DocGeminiPricing, Notes: "up_to_200k_prompt_tokens", CheckedOn: googleCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.20, Input: 2.00, Output: 12.00, Currency: "USD"}}},
|
||||
},
|
||||
|
||||
{
|
||||
Provider: "anthropic", Model: "claude-sonnet-4-5", Currency: "USD",
|
||||
BillingMode: BillingModePAYG, DocURL: DocAnthropicPricing, CheckedOn: anthropicCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.30, Input: 3, Output: 15, CacheWrite: 3.75, Currency: "USD"}}},
|
||||
},
|
||||
|
||||
// MiMo domestic PAYG; Token Plan bills the same rates as subscription_equivalent.
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.5-pro", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocMiMoPAYG, CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.025, Input: 3, Output: 6, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.5", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocMiMoPAYG, CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}}},
|
||||
},
|
||||
// Not on the vendor's price page as of CheckedOn. Kept because removing a rate
|
||||
// is what turns a still-configured model's cost silently into zero; a stale
|
||||
// rate at least reads as a number somebody can question.
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2-flash", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocMiMoPAYG, Notes: "not_on_vendor_page", CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.07, Input: 0.70, Output: 2.10, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.5-pro", Currency: "CNY",
|
||||
BillingMode: BillingModeSubscriptionEquivalent, DocURL: DocMiMoTokenPlan, Notes: "payg_equivalent_not_plan_bill", CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.025, Input: 3, Output: 6, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.5", Currency: "CNY",
|
||||
BillingMode: BillingModeSubscriptionEquivalent, DocURL: DocMiMoTokenPlan, Notes: "payg_equivalent_not_plan_bill", CheckedOn: checkedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.6-pro", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocMiMoPAYG, CheckedOn: mimoCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.025, Input: 3, Output: 6, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.6-flash", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocMiMoPAYG, CheckedOn: mimoCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.6-pro-ultraspeed", Currency: "CNY",
|
||||
BillingMode: BillingModePAYG, DocURL: DocMiMoPAYG, CheckedOn: mimoCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.25, Input: 30, Output: 60, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.6-pro", Currency: "CNY",
|
||||
BillingMode: BillingModeSubscriptionEquivalent, DocURL: DocMiMoTokenPlan, Notes: "payg_equivalent_not_plan_bill", CheckedOn: mimoCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.025, Input: 3, Output: 6, Currency: "CNY"}}},
|
||||
},
|
||||
{
|
||||
Provider: "mimo", Model: "mimo-v2.6-flash", Currency: "CNY",
|
||||
BillingMode: BillingModeSubscriptionEquivalent, DocURL: DocMiMoTokenPlan, Notes: "payg_equivalent_not_plan_bill", CheckedOn: mimoCheckedOn,
|
||||
Rates: []rateGeneration{{Base: RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}}},
|
||||
},
|
||||
}
|
||||
|
||||
// retiredDeepSeekFlashModels are names DeepSeek still answers as its current
|
||||
// flash model, at that model's price. Owned here because that is where the rates
|
||||
// they bill at live; config reads them through the accessor below.
|
||||
var retiredDeepSeekFlashModels = []string{"deepseek-v4-flash", "deepseek-v4-flash-vision-exp"}
|
||||
|
||||
// RetiredDeepSeekFlashModels copies the list on the way out: a caller sorting or
|
||||
// trimming the result would otherwise change what every other caller sees.
|
||||
func RetiredDeepSeekFlashModels() []string {
|
||||
return append([]string(nil), retiredDeepSeekFlashModels...)
|
||||
}
|
||||
|
||||
// RateModelFor answers which model's rate a name is billed at — a retired name
|
||||
// at the model that replaced it, an older spelling at the one it was renamed
|
||||
// from. Scoped to billing on purpose: two models can share a rate without being
|
||||
// the same model, and nothing outside a price lookup may read this as identity.
|
||||
func RateModelFor(provider, model string) string {
|
||||
model = strings.TrimSpace(model)
|
||||
switch strings.ToLower(strings.TrimSpace(provider)) {
|
||||
case "deepseek":
|
||||
if slices.Contains(retiredDeepSeekFlashModels, model) {
|
||||
return "deepseek-flash"
|
||||
}
|
||||
case "mimo":
|
||||
switch model {
|
||||
case "mimo-v2-pro":
|
||||
return "mimo-v2.5-pro"
|
||||
case "mimo-v2-omni":
|
||||
return "mimo-v2.5"
|
||||
}
|
||||
}
|
||||
return model
|
||||
}
|
||||
|
||||
// DefaultCurrency is what a vendor's rates are quoted in when an entry does not
|
||||
// say. A vendor publishing one currency leaves no choice to make; DeepSeek
|
||||
// prices in both, and USD is what an entry that never chose has been read as.
|
||||
func DefaultCurrency(provider string) string {
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
seen := map[string]bool{}
|
||||
for _, row := range officialRates {
|
||||
if row.Provider == provider {
|
||||
seen[row.Currency] = true
|
||||
}
|
||||
}
|
||||
if len(seen) == 1 {
|
||||
for currency := range seen {
|
||||
return currency
|
||||
}
|
||||
}
|
||||
return "USD"
|
||||
}
|
||||
|
||||
// officialRateRowsFor finds the rows for a model, which is more than one when a
|
||||
// vendor publishes the same rates under two billing modes.
|
||||
func officialRateRowsFor(provider, model, currency string) []officialRateRow {
|
||||
provider, currency = strings.ToLower(strings.TrimSpace(provider)), NormalizeCurrency(currency)
|
||||
model = RateModelFor(provider, model)
|
||||
var out []officialRateRow
|
||||
for _, row := range officialRates {
|
||||
if row.Provider == provider && row.Model == model && row.Currency == currency {
|
||||
out = append(out, row)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CurrentRate is what a vendor's model costs in a currency today — the off-peak
|
||||
// card, which is what a caller charging one rate should quote. Nil when this
|
||||
// catalog prices neither. A model published under two billing modes quotes the
|
||||
// pay-as-you-go rate, which is the one an unsubscribed caller is charged.
|
||||
func CurrentRate(provider, model, currency string) *RateCard {
|
||||
for _, row := range officialRateRowsFor(provider, model, currency) {
|
||||
if row.BillingMode == BillingModeSubscriptionEquivalent {
|
||||
continue
|
||||
}
|
||||
if len(row.Rates) > 0 {
|
||||
rate := row.Rates[row.currentIndex()].Base
|
||||
return &rate
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsOfficialCard reports whether card is a rate this project has shipped for the
|
||||
// model in the card's own currency: any generation, its off-peak or its peak
|
||||
// card. A card that is none of them is the user's, or damaged, and cannot be
|
||||
// valued against the vendor's other-currency table.
|
||||
func IsOfficialCard(provider, model string, card RateCard) bool {
|
||||
for _, row := range officialRateRowsFor(provider, model, card.Currency) {
|
||||
if row.BillingMode == BillingModeSubscriptionEquivalent {
|
||||
continue
|
||||
}
|
||||
for _, generation := range row.Rates {
|
||||
if sameCard(generation.Base, card) || generation.Peak != nil && sameCard(*generation.Peak, card) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func sameCard(a, b RateCard) bool {
|
||||
return a.CacheHit == b.CacheHit && a.Input == b.Input && a.Output == b.Output && a.CacheWrite == b.CacheWrite
|
||||
}
|
||||
|
||||
// SupersededRates are the rates shipped for a model before the current one. A
|
||||
// stored price matching one came from this project rather than from its user, so
|
||||
// a refresh may replace it; anything else is theirs and is left alone.
|
||||
func SupersededRates(provider, model, currency string) []RateCard {
|
||||
var out []RateCard
|
||||
for _, row := range officialRateRowsFor(provider, model, currency) {
|
||||
if row.BillingMode == BillingModeSubscriptionEquivalent || len(row.Rates) < 2 {
|
||||
continue
|
||||
}
|
||||
for _, generation := range row.Rates[:row.currentIndex()] {
|
||||
out = append(out, generation.Base)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// officialCatalogRows publishes the current generation of every row, plus the
|
||||
// retired DeepSeek names, which bill at the flash rate under their own id.
|
||||
func officialCatalogRows() []CatalogEntry {
|
||||
rows := make([]CatalogEntry, 0, len(officialRates)+len(retiredDeepSeekFlashModels)*2)
|
||||
for _, row := range officialRates {
|
||||
if len(row.Rates) == 0 {
|
||||
continue
|
||||
}
|
||||
current := row.Rates[row.currentIndex()]
|
||||
rows = append(rows, row.entry(row.Model, current))
|
||||
if row.Provider != "deepseek" || row.Model != "deepseek-flash" {
|
||||
continue
|
||||
}
|
||||
for _, retired := range retiredDeepSeekFlashModels {
|
||||
rows = append(rows, row.entry(retired, current))
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func (r officialRateRow) entry(model string, current rateGeneration) CatalogEntry {
|
||||
// Copied, so a caller holding a returned row cannot reach back into the
|
||||
// table every other caller reads.
|
||||
var peak *RateCard
|
||||
if current.Peak != nil {
|
||||
card := *current.Peak
|
||||
peak = &card
|
||||
}
|
||||
return CatalogEntry{
|
||||
Provider: r.Provider, Model: model, Currency: r.Currency,
|
||||
CacheHit: current.Base.CacheHit, Input: current.Base.Input, Output: current.Base.Output,
|
||||
CacheWrite: current.Base.CacheWrite,
|
||||
Peak: peak, Window: r.Window, DocURL: r.DocURL,
|
||||
BillingMode: r.BillingMode, Notes: r.Notes, CheckedOn: r.CheckedOn,
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
prompt = """Some providers bill tokens written into their prompt cache at a rate of their own. Add that to the pricing contract in this repository:
|
||||
|
||||
- `RateCard` gains a `CacheWrite` rate, per 1M tokens like the others. Zero means the card has no separate cache-write price, and cache-written tokens keep being charged at `Input` exactly as they are today.
|
||||
- When a card sets `CacheWrite`, `OriginalCostAmount` charges the billed cache-write units (`CacheWriteBilledTokens` when set, otherwise `CacheWriteTokens`) at `CacheWrite` instead of `Input`.
|
||||
- `PricingFingerprint` must tell apart two cards that differ only in `CacheWrite`, but a card without one must keep exactly the fingerprint it has today: stored ledgers are keyed by it.
|
||||
- Two cards that differ only in `CacheWrite` are different cards everywhere cards are compared, including official-card recognition and catalog matching.
|
||||
- `CatalogEntry` carries the rate too, and `RateCardFromCatalog` and `CatalogEntry.RatesAt` hand it back.
|
||||
- Add Anthropic to the official price book: `claude-sonnet-4-5`, pay-as-you-go in USD, cache hit 0.30, input 3, output 15 and cache write 3.75 per 1M tokens, documented at https://www.anthropic.com/pricing and checked 2026-10-19. `api.anthropic.com` is Anthropic's own endpoint.
|
||||
- `benchctl cost` gains a `-write` flag for cache-written tokens, and `benchctl rate` shows a card's cache-write rate when it has one.
|
||||
- Add tests for the new behaviour and update the README's pricing section."""
|
||||
class = "multi-file-feature"
|
||||
timeout_sec = 1200
|
||||
@@ -1,119 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
export GOFLAGS=-mod=mod GOTOOLCHAIN=local
|
||||
cat > internal/contract/pricing/zz_grader_test.go <<'GO'
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGraderCacheWriteCost(t *testing.T) {
|
||||
card := RateCard{CacheHit: 0.3, Input: 3, Output: 15, Currency: "USD"}
|
||||
priced := card
|
||||
priced.CacheWrite = 3.75
|
||||
cases := []struct {
|
||||
name string
|
||||
card RateCard
|
||||
usage UsageTokens
|
||||
want string
|
||||
}{
|
||||
{"no write, no rate", card, UsageTokens{CacheHitTokens: 200000, CacheMissTokens: 800000, CompletionTokens: 50000}, "3.21"},
|
||||
{"no write, rate set", priced, UsageTokens{CacheHitTokens: 200000, CacheMissTokens: 800000, CompletionTokens: 50000}, "3.21"},
|
||||
{"write at input", card, UsageTokens{CacheMissTokens: 1000000, CacheWriteTokens: 400000, CompletionTokens: 100000}, "4.5"},
|
||||
{"write at cache-write", priced, UsageTokens{CacheMissTokens: 1000000, CacheWriteTokens: 400000, CompletionTokens: 100000}, "4.8"},
|
||||
{"billed units at input", card, UsageTokens{CacheMissTokens: 1000000, CacheWriteTokens: 400000, CacheWriteBilledTokens: 500000}, "3.3"},
|
||||
{"billed units at cache-write", priced, UsageTokens{CacheMissTokens: 1000000, CacheWriteTokens: 400000, CacheWriteBilledTokens: 500000}, "3.675"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := OriginalCostAmount(tc.card, tc.usage).String(); got != tc.want {
|
||||
t.Errorf("%s: cost = %s, want %s", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraderFingerprintKeepsStoredKeys(t *testing.T) {
|
||||
cny := RateCard{CacheHit: 0.02, Input: 1, Output: 4, Currency: "CNY"}
|
||||
usd := RateCard{CacheHit: 0.3, Input: 3, Output: 15, Currency: "USD"}
|
||||
if got := PricingFingerprint(cny); got != "ba96037de26c3f79" {
|
||||
t.Errorf("fingerprint of a card without CacheWrite moved: %s", got)
|
||||
}
|
||||
if got := PricingFingerprint(usd); got != "e16b53b4599e5fda" {
|
||||
t.Errorf("fingerprint of a card without CacheWrite moved: %s", got)
|
||||
}
|
||||
withWrite := usd
|
||||
withWrite.CacheWrite = 3.75
|
||||
if PricingFingerprint(withWrite) == PricingFingerprint(usd) {
|
||||
t.Error("cards differing only in CacheWrite share a fingerprint")
|
||||
}
|
||||
other := usd
|
||||
other.CacheWrite = 6
|
||||
if PricingFingerprint(withWrite) == PricingFingerprint(other) {
|
||||
t.Error("two different CacheWrite rates share a fingerprint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraderCacheWriteIsPartOfCardIdentity(t *testing.T) {
|
||||
official := RateCard{CacheHit: 0.02, Input: 1, Output: 2, Currency: "CNY"}
|
||||
if !IsOfficialCard("mimo", "mimo-v2.5", official) {
|
||||
t.Fatal("the official mimo-v2.5 card is no longer recognised")
|
||||
}
|
||||
changed := official
|
||||
changed.CacheWrite = 1.25
|
||||
if IsOfficialCard("mimo", "mimo-v2.5", changed) {
|
||||
t.Error("a card with a cache-write rate the vendor never published is recognised as official")
|
||||
}
|
||||
if _, ok := MatchesCatalog("mimo", "mimo-v2.5", official); !ok {
|
||||
t.Fatal("the official mimo-v2.5 card no longer matches the catalog")
|
||||
}
|
||||
if _, ok := MatchesCatalog("mimo", "mimo-v2.5", changed); ok {
|
||||
t.Error("a card with an unpublished cache-write rate matches the catalog")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraderAnthropicIsPriced(t *testing.T) {
|
||||
want := RateCard{CacheHit: 0.30, Input: 3, Output: 15, CacheWrite: 3.75, Currency: "USD"}
|
||||
got := CurrentRate("anthropic", "claude-sonnet-4-5", "USD")
|
||||
if got == nil || *got != want {
|
||||
t.Fatalf("CurrentRate(anthropic, claude-sonnet-4-5) = %+v, want %+v", got, want)
|
||||
}
|
||||
if !IsOfficialCard("anthropic", "claude-sonnet-4-5", want) {
|
||||
t.Error("the Anthropic card is not recognised as official")
|
||||
}
|
||||
e, ok := LookupCatalog("anthropic", "claude-sonnet-4-5", "USD", "")
|
||||
if !ok || e.DocURL != "https://www.anthropic.com/pricing" || e.CheckedOn != "2026-10-19" || e.CacheWrite != 3.75 {
|
||||
t.Errorf("catalog entry = %+v, %v", e, ok)
|
||||
}
|
||||
if got := OfficialProviderForEndpoint("https://api.anthropic.com/v1/messages"); got != "anthropic" {
|
||||
t.Errorf("api.anthropic.com resolves to %q", got)
|
||||
}
|
||||
if got := DefaultCurrency("anthropic"); got != "USD" {
|
||||
t.Errorf("DefaultCurrency(anthropic) = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraderCatalogCarriesCacheWrite(t *testing.T) {
|
||||
e := CatalogEntry{Provider: "x", Model: "y", Currency: "USD", CacheHit: 0.1, Input: 1, Output: 2, CacheWrite: 1.25}
|
||||
if got := RateCardFromCatalog(e); got.CacheWrite != 1.25 || got.Input != 1 {
|
||||
t.Errorf("RateCardFromCatalog = %+v", got)
|
||||
}
|
||||
if got := e.RatesAt(time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)); got.CacheWrite != 1.25 {
|
||||
t.Errorf("RatesAt = %+v", got)
|
||||
}
|
||||
}
|
||||
GO
|
||||
gofmt_out="$(gofmt -l .)"
|
||||
[ -z "$gofmt_out" ] || { echo "gofmt needed: $gofmt_out"; exit 1; }
|
||||
go vet ./...
|
||||
go test ./... >/dev/null
|
||||
out="$(go run ./cmd/benchctl cost -provider mimo -model mimo-v2.5 -miss 1000000 -write 400000)"
|
||||
[ "$out" = "1 CNY" ] || { echo "benchctl cost -write: got $out"; exit 1; }
|
||||
out="$(go run ./cmd/benchctl rate -provider anthropic -model claude-sonnet-4-5)"
|
||||
case "$out" in *"cache-write 3.75"*) ;; *) echo "benchctl rate does not show the cache-write rate: $out"; exit 1 ;; esac
|
||||
out="$(go run ./cmd/benchctl rate -provider mimo -model mimo-v2.5)"
|
||||
case "$out" in *"cache-write"*) echo "benchctl rate shows a cache-write rate the card does not have: $out"; exit 1 ;; esac
|
||||
grep -lE 'CacheWrite:' internal/contract/pricing/*_test.go | grep -qv zz_grader_test.go \
|
||||
|| { echo "no test of the pricing package sets a CacheWrite rate"; exit 1; }
|
||||
awk '/^## Pricing/{on=1; next} /^## /{on=0} on' README.md | grep -q 'CacheWrite' \
|
||||
|| { echo "README's pricing section does not mention CacheWrite"; exit 1; }
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
---
|
||||
name: price-change
|
||||
description: Record a vendor's announced price change in the official price book without losing the rate it replaces.
|
||||
---
|
||||
|
||||
# Recording a price change
|
||||
|
||||
1. Find the vendor's rows in `officialRates` in
|
||||
`internal/contract/pricing/rates.go`. Each row is one model in one currency.
|
||||
2. **Append** a `rateGeneration` to the row's `Rates`; never edit an existing
|
||||
one. Set `From` to the first billed day (`YYYY-MM-DD`, UTC) when the change
|
||||
is announced ahead of time.
|
||||
3. Leave the earlier generation where it is: `SupersededRates` reads it so a
|
||||
stored price that matches it is recognised as ours and refreshed.
|
||||
4. Update the row's `CheckedOn` constant to the day you confirmed the page.
|
||||
5. Add a test beside `rates_test.go` that pins `CurrentRate` before and after
|
||||
the `From` day by replacing `today`.
|
||||
6. Run `make check`. `make pricecheck` needs the network and is run by hand.
|
||||
@@ -1,17 +0,0 @@
|
||||
.PHONY: check test vet fmt pricecheck
|
||||
|
||||
check: fmt vet test
|
||||
|
||||
fmt:
|
||||
@out="$$(gofmt -l .)"; if [ -n "$$out" ]; then echo "gofmt needed:"; echo "$$out"; exit 1; fi
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
# Reads each vendor's pricing page; this snapshot carries no network client.
|
||||
pricecheck:
|
||||
@echo "pricecheck: reaches each vendor's pricing page and needs the network; not available in this snapshot" >&2
|
||||
@exit 2
|
||||
@@ -1,69 +0,0 @@
|
||||
# Reasonix kernel snapshot
|
||||
|
||||
A trimmed slice of the Reasonix Go kernel: the contracts a benchmark report is
|
||||
priced and labelled with, a few leaf utilities, and `benchctl`, a small command
|
||||
that prints what those contracts answer.
|
||||
|
||||
```bash
|
||||
make check # gofmt, vet, test
|
||||
go run ./cmd/benchctl rate -model deepseek-flash -currency USD
|
||||
go run ./cmd/benchctl cost -provider mimo -model mimo-v2.5 -miss 1000000
|
||||
go run ./cmd/benchctl tariff -currency CNY -at 2026-10-12T02:00:00Z
|
||||
go run ./cmd/benchctl arm -ablate planner,evidence
|
||||
```
|
||||
|
||||
## Packages
|
||||
|
||||
| Package | Owns |
|
||||
| --- | --- |
|
||||
| `internal/contract/pricing` | the official price book, cost quotes, the spend ledger |
|
||||
| `internal/contract/ablation` | which subsystems a benchmark arm switches off, and the arm's name |
|
||||
| `internal/contract/agentgraph` | the vocabulary of one run's execution graph |
|
||||
| `internal/base/retrieval` | BM25 ranking over CJK-aware tokens |
|
||||
| `internal/base/rulelist` | splitting a one-line permission rule list |
|
||||
| `internal/base/tokencount` | a cheap token estimate |
|
||||
| `internal/base/neterr` | classifying transport errors |
|
||||
| `internal/base/nilutil` | typed-nil detection |
|
||||
| `cmd/benchctl` | prints rates, costs, peak schedules and arm names |
|
||||
|
||||
## Pricing
|
||||
|
||||
A `RateCard` is a price per million tokens in one currency:
|
||||
|
||||
| Field | Charges |
|
||||
| --- | --- |
|
||||
| `CacheHit` | prompt tokens served from the provider's cache |
|
||||
| `Input` | prompt tokens that missed the cache, including tokens written to it |
|
||||
| `Output` | completion tokens |
|
||||
|
||||
`OriginalCostAmount` turns a card and a `UsageTokens` breakdown into a
|
||||
fixed-point `Amount`; `PricingFingerprint` keys a card in the ledger, so a
|
||||
stored ledger entry stays attached to the card it was priced with.
|
||||
`CurrentRate` and `SupersededRates` answer from `officialRates`, which only
|
||||
ever grows: a price change appends a generation.
|
||||
|
||||
## Ablation
|
||||
|
||||
`ablation.Parse` reads a comma-separated list of modules to switch off, and
|
||||
`Set.Arm` names the result (`full` for the control). The switchable modules,
|
||||
in the order arm names use:
|
||||
|
||||
| Module | Off means |
|
||||
| --- | --- |
|
||||
| `evidence` | no evidence gate on completion claims |
|
||||
| `planner` | no two-model planning turn |
|
||||
| `subagent` | no delegation to workers |
|
||||
| `retrieval` | no retrieval over the folded transcript |
|
||||
| `compaction` | no context folding |
|
||||
| `upstream` | a dependency edge orders its endpoints without delivering the answer |
|
||||
| `recall-search` | recall can read a given address but cannot search for one |
|
||||
|
||||
The fold-index scale is a separate axis (`-fold-index`), not a module.
|
||||
|
||||
## Skills
|
||||
|
||||
Project skills live under `.reasonix/skills/<name>/SKILL.md`.
|
||||
|
||||
| Skill | Use it to |
|
||||
| --- | --- |
|
||||
| `price-change` | record a vendor's price change in the price book |
|
||||
@@ -1,51 +0,0 @@
|
||||
# Reasonix project memory
|
||||
|
||||
This checkout is a trimmed snapshot of the Reasonix kernel: the billing,
|
||||
ablation and graph contracts and a handful of leaf utilities, plus one small
|
||||
command that drives them. The conventions below are the project's own.
|
||||
|
||||
## Conventions
|
||||
|
||||
- Go kernel under `internal/`; each package owns one concern. A package's long
|
||||
explanation belongs in its `doc.go`, not spread across implementation files.
|
||||
- Judgements read structure, never wording: types, contracts, schemas. An
|
||||
error a caller must tell apart carries an identity — a sentinel, a typed
|
||||
code — never a sentence.
|
||||
- A metric is its definition, not its name. Before a number becomes a finding,
|
||||
state what it counts, from which anchor, over which population, and what it
|
||||
excludes.
|
||||
|
||||
## Comments
|
||||
|
||||
Default is none — the code is the truth. Write one only when the **why** is
|
||||
non-obvious: a hidden constraint, an invariant the type system cannot express,
|
||||
or an external-protocol quirk. A comment states the constraint that holds now;
|
||||
it does not narrate how the code got here. English, always.
|
||||
|
||||
- Declaration doc: ≤5 lines. Package comment: ≤8 lines, or ≤40 in a `doc.go`.
|
||||
- Every other comment: ≤3 lines. Struct-field and trailing `//`: 1 line.
|
||||
- One responsibility per file; 800 lines is the ceiling.
|
||||
|
||||
## Vendor rates
|
||||
|
||||
Official list prices live in one place: `officialRates` in
|
||||
`internal/contract/pricing/rates.go`, oldest generation first, the last being
|
||||
what a model charges today. A price change is an **append, never an edit** —
|
||||
the superseded rate is what lets an installed config be recognised as holding
|
||||
one of ours and brought forward. Callers read these through
|
||||
`pricing.CurrentRate` and `pricing.SupersededRates` and restate none of them.
|
||||
|
||||
`make pricecheck` compares that table against each vendor's own page. It needs
|
||||
the network, so it is not part of `make check`.
|
||||
|
||||
## Before every commit
|
||||
|
||||
```bash
|
||||
make check # gofmt -l, go vet ./..., go test ./...
|
||||
```
|
||||
|
||||
## Change discipline
|
||||
|
||||
List callers, consumers of the same state and sibling features before editing.
|
||||
Fix at the owning layer; a fix that breaks a neighbouring behaviour is a failed
|
||||
fix. Write tests first; they must fail without the change.
|
||||
@@ -1,115 +0,0 @@
|
||||
// Command benchctl prints what the pricing and ablation contracts answer for
|
||||
// one model or one benchmark arm, so a reviewer can check a rate or an arm name
|
||||
// without writing a test.
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/pricing"
|
||||
)
|
||||
|
||||
var errUsage = errors.New("usage: benchctl rate|cost|tariff|arm [flags]")
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "benchctl:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func run(args []string, out io.Writer) error {
|
||||
if len(args) == 0 {
|
||||
return errUsage
|
||||
}
|
||||
switch args[0] {
|
||||
case "rate":
|
||||
return runRate(args[1:], out)
|
||||
case "cost":
|
||||
return runCost(args[1:], out)
|
||||
case "tariff":
|
||||
return runTariff(args[1:], out)
|
||||
case "arm":
|
||||
return runArm(args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("%w: unknown subcommand %q", errUsage, args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func runRate(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("rate", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
fmt.Fprintf(out, "%s/%s %s: cache-hit %g input %g output %g per 1M\n",
|
||||
*provider, *model, card.Currency, card.CacheHit, card.Input, card.Output)
|
||||
if entry, ok := pricing.LookupCatalog(*provider, *model, cur, ""); ok && entry.DocURL != "" {
|
||||
fmt.Fprintf(out, "source: %s\n", entry.DocURL)
|
||||
}
|
||||
fmt.Fprintf(out, "superseded generations: %d\n", len(pricing.SupersededRates(*provider, *model, cur)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runCost(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("cost", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
hit := fs.Int("hit", 0, "cached prompt tokens")
|
||||
miss := fs.Int("miss", 0, "uncached prompt tokens")
|
||||
completion := fs.Int("completion", 0, "completion tokens")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
usage := pricing.UsageTokens{CacheHitTokens: *hit, CacheMissTokens: *miss, CompletionTokens: *completion}
|
||||
amount := pricing.OriginalCostAmount(*card, usage)
|
||||
fmt.Fprintf(out, "%s %s\n", amount, pricing.NormalizeCurrency(card.Currency))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runArm(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("arm", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
spec := fs.String("ablate", "", "comma-separated modules to switch off: "+ablation.ModuleList())
|
||||
foldIndex := fs.String("fold-index", "", "fold-index scale: default, half, quarter or off")
|
||||
list := fs.Bool("list", false, "list the switchable modules")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *list {
|
||||
fmt.Fprintln(out, ablation.ModuleList())
|
||||
return nil
|
||||
}
|
||||
set, err := ablation.ParseArm(*spec, *foldIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(out, set.Arm())
|
||||
return nil
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func runOut(t *testing.T, args ...string) string {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
if err := run(args, &out); err != nil {
|
||||
t.Fatalf("run %v: %v", args, err)
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func TestRateNamesTheCurrentCard(t *testing.T) {
|
||||
got := runOut(t, "rate", "-provider", "mimo", "-model", "mimo-v2.5")
|
||||
if !strings.Contains(got, "mimo/mimo-v2.5 CNY: cache-hit 0.02 input 1 output 2 per 1M") {
|
||||
t.Fatalf("rate output = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateRefusesAnUnpricedModel(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if err := run([]string{"rate", "-provider", "nobody", "-model", "x"}, &out); err == nil {
|
||||
t.Fatal("an unpriced model must fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCostChargesTheCurrentCard(t *testing.T) {
|
||||
got := runOut(t, "cost", "-provider", "mimo", "-model", "mimo-v2.5", "-miss", "1000000", "-completion", "1000000")
|
||||
if strings.TrimSpace(got) != "3 CNY" {
|
||||
t.Fatalf("cost output = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArmNamesTheConfiguration(t *testing.T) {
|
||||
if got := strings.TrimSpace(runOut(t, "arm")); got != "full" {
|
||||
t.Fatalf("control arm = %q", got)
|
||||
}
|
||||
if got := strings.TrimSpace(runOut(t, "arm", "-ablate", "planner,evidence", "-fold-index", "half")); got != "no-evidence+no-planner+index-half" {
|
||||
t.Fatalf("arm = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownSubcommandIsAUsageError(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if err := run([]string{"nope"}, &out); !errors.Is(err, errUsage) {
|
||||
t.Fatalf("err = %v, want errUsage", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTariffReadsTheVendorsSchedule(t *testing.T) {
|
||||
got := runOut(t, "tariff", "-provider", "mimo", "-model", "mimo-v2.5")
|
||||
if !strings.Contains(got, "official card: true") || !strings.Contains(got, "one rate around the clock") {
|
||||
t.Fatalf("tariff output = %q", got)
|
||||
}
|
||||
peak := runOut(t, "tariff", "-currency", "CNY", "-at", "2026-10-12T02:00:00Z")
|
||||
if !strings.Contains(peak, "peak=true until 2026-10-12T04:00:00Z") {
|
||||
t.Fatalf("deepseek tariff at a weekday peak = %q", peak)
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/pricing"
|
||||
)
|
||||
|
||||
func runTariff(args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("tariff", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
provider := fs.String("provider", "deepseek", "vendor")
|
||||
model := fs.String("model", "deepseek-flash", "model id")
|
||||
currency := fs.String("currency", "", "currency; empty reads the vendor's default")
|
||||
at := fs.String("at", "", "instant to read the schedule at, RFC 3339; empty is now")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
when := time.Now()
|
||||
if *at != "" {
|
||||
t, err := time.Parse(time.RFC3339, *at)
|
||||
if err != nil {
|
||||
return fmt.Errorf("-at: %w", err)
|
||||
}
|
||||
when = t
|
||||
}
|
||||
cur := *currency
|
||||
if cur == "" {
|
||||
cur = pricing.DefaultCurrency(*provider)
|
||||
}
|
||||
card := pricing.CurrentRate(*provider, *model, cur)
|
||||
if card == nil {
|
||||
return fmt.Errorf("no official rate for %s/%s in %s", *provider, *model, cur)
|
||||
}
|
||||
fmt.Fprintf(out, "official card: %v\n", pricing.IsOfficialCard(*provider, *model, *card))
|
||||
tariff, ok := pricing.LiveTariff(*provider, *model, *card, when)
|
||||
switch {
|
||||
case !ok:
|
||||
fmt.Fprintln(out, "schedule: one rate around the clock")
|
||||
case !tariff.Changes:
|
||||
fmt.Fprintf(out, "schedule: peak=%v, no change ahead\n", tariff.Peak)
|
||||
default:
|
||||
fmt.Fprintf(out, "schedule: peak=%v until %s\n", tariff.Peak, tariff.ChangesAt.UTC().Format(time.RFC3339))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module reasonix
|
||||
|
||||
go 1.26.0
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
//go:build !windows
|
||||
|
||||
package neterr
|
||||
|
||||
import "syscall"
|
||||
|
||||
var resetErrors = []error{syscall.ECONNRESET, syscall.ECONNABORTED, syscall.EPIPE}
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
package neterr
|
||||
|
||||
import "syscall"
|
||||
|
||||
// syscall.ECONNRESET and friends exist on Windows only as APPLICATION_ERROR
|
||||
// placeholders, so errors.Is against them never matches a real socket failure.
|
||||
// Winsock's own numbering carries the identities the runtime returns.
|
||||
var resetErrors = []error{syscall.WSAECONNRESET, syscall.WSAECONNABORTED, syscall.ERROR_BROKEN_PIPE}
|
||||
@@ -1,47 +0,0 @@
|
||||
// Package neterr answers what a transport error is from the identity the
|
||||
// operating system gave it, never from the sentence it rendered. A system
|
||||
// error's text arrives in the machine's own language and is reworded between
|
||||
// releases, so a word list that matches on an English host silently stops
|
||||
// matching elsewhere. The judgement lives here once because three packages
|
||||
// were each carrying their own copy of it.
|
||||
package neterr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
)
|
||||
|
||||
// IsConnReset reports whether err is a connection-level drop (peer reset,
|
||||
// truncated body, closed socket) as opposed to a protocol or caller error. A
|
||||
// stream cut this way mid-body can be replayed from scratch, unlike a decode
|
||||
// or 4xx error. The common trigger is a local proxy (v2rayN/sing-box)
|
||||
// idle-closing a long-lived SSE connection during a reasoner's first-token gap.
|
||||
func IsConnReset(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) ||
|
||||
errors.Is(err, net.ErrClosed) || isAny(err, resetErrors) {
|
||||
return true
|
||||
}
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(err, &dnsErr) {
|
||||
return false
|
||||
}
|
||||
var netErr net.Error
|
||||
return errors.As(err, &netErr)
|
||||
}
|
||||
|
||||
func isAny(err error, targets []error) bool {
|
||||
for _, target := range targets {
|
||||
if errors.Is(err, target) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
-43
@@ -1,43 +0,0 @@
|
||||
package neterr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsConnReset(t *testing.T) {
|
||||
if IsConnReset(nil) {
|
||||
t.Error("nil is not a conn reset")
|
||||
}
|
||||
if IsConnReset(context.Canceled) || IsConnReset(context.DeadlineExceeded) {
|
||||
t.Error("ctx cancel/deadline must not look like a recoverable reset")
|
||||
}
|
||||
if IsConnReset(errors.New("decode stream: invalid character")) {
|
||||
t.Error("a plain protocol error must not be treated as a conn reset")
|
||||
}
|
||||
for _, err := range []error{
|
||||
io.ErrUnexpectedEOF,
|
||||
&net.OpError{Op: "read", Err: resetErrors[0]},
|
||||
fmt.Errorf("read stream: %w", &net.OpError{Op: "read", Err: errors.New("wsarecv: forcibly closed")}),
|
||||
} {
|
||||
if !IsConnReset(err) {
|
||||
t.Errorf("want conn reset for %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsConnResetExcludesDNS(t *testing.T) {
|
||||
for _, dns := range []*net.DNSError{
|
||||
{Err: "no such host", Name: "api.deepseek.com", IsNotFound: true},
|
||||
{Err: "i/o timeout", Name: "api.deepseek.com", IsTimeout: true, IsTemporary: true},
|
||||
} {
|
||||
err := fmt.Errorf("dial: %w", &net.OpError{Op: "dial", Err: dns})
|
||||
if IsConnReset(err) {
|
||||
t.Errorf("DNS failure %v is not a connection reset", dns)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
package nilutil
|
||||
|
||||
import "reflect"
|
||||
|
||||
// IsNil reports whether v is nil or an interface holding a typed nil value.
|
||||
func IsNil(v any) bool {
|
||||
if v == nil {
|
||||
return true
|
||||
}
|
||||
rv := reflect.ValueOf(v)
|
||||
switch rv.Kind() {
|
||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||
return rv.IsNil()
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
-31
@@ -1,31 +0,0 @@
|
||||
package nilutil
|
||||
|
||||
import "testing"
|
||||
|
||||
type sampleInterface interface {
|
||||
Do()
|
||||
}
|
||||
|
||||
type sampleImpl struct{}
|
||||
|
||||
func (*sampleImpl) Do() {}
|
||||
|
||||
func TestIsNilDetectsTypedNilInterface(t *testing.T) {
|
||||
var p *sampleImpl
|
||||
var i sampleInterface = p
|
||||
|
||||
if !IsNil(i) {
|
||||
t.Fatal("IsNil should detect typed nil interface")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsNilRejectsConcreteValues(t *testing.T) {
|
||||
var p sampleInterface = &sampleImpl{}
|
||||
|
||||
if IsNil(p) {
|
||||
t.Fatal("IsNil should not reject a concrete interface value")
|
||||
}
|
||||
if IsNil("x") {
|
||||
t.Fatal("IsNil should not reject non-nil scalar values")
|
||||
}
|
||||
}
|
||||
-218
@@ -1,218 +0,0 @@
|
||||
package retrieval
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Tokens lowercases Latin words and splits CJK runs into overlapping bigrams
|
||||
// (a lone CJK rune stays a single term) — the standard CJK indexing unit
|
||||
// (Lucene's CJKAnalyzer, SQLite FTS): a bigram only matches a real two-rune
|
||||
// subsequence where per-rune unigrams matched scattered common characters.
|
||||
// Intentionally a local, dependency-free approximation of FTS matching.
|
||||
func Tokens(s string) []string {
|
||||
var out []string
|
||||
var b strings.Builder
|
||||
var prev rune
|
||||
cjkRun := 0
|
||||
flush := func() {
|
||||
if b.Len() == 0 {
|
||||
return
|
||||
}
|
||||
out = append(out, b.String())
|
||||
b.Reset()
|
||||
}
|
||||
endCJK := func() {
|
||||
if cjkRun == 1 {
|
||||
out = append(out, string(prev))
|
||||
}
|
||||
cjkRun = 0
|
||||
}
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case isCJK(r):
|
||||
flush()
|
||||
if cjkRun > 0 {
|
||||
out = append(out, string([]rune{prev, r}))
|
||||
}
|
||||
prev = r
|
||||
cjkRun++
|
||||
case unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_':
|
||||
endCJK()
|
||||
b.WriteRune(unicode.ToLower(r))
|
||||
default:
|
||||
flush()
|
||||
endCJK()
|
||||
}
|
||||
}
|
||||
flush()
|
||||
endCJK()
|
||||
return out
|
||||
}
|
||||
|
||||
func isCJK(r rune) bool {
|
||||
return unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul)
|
||||
}
|
||||
|
||||
// Unique returns terms in first-seen order.
|
||||
func Unique(in []string) []string {
|
||||
seen := map[string]bool{}
|
||||
out := make([]string, 0, len(in))
|
||||
for _, s := range in {
|
||||
if s == "" || seen[s] {
|
||||
continue
|
||||
}
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Counts returns a term-frequency map.
|
||||
func Counts(terms []string) map[string]int {
|
||||
counts := map[string]int{}
|
||||
for _, term := range terms {
|
||||
counts[term]++
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
// BM25Score scores a document against query terms.
|
||||
func BM25Score(counts map[string]int, length int, queryTerms []string, df map[string]int, totalDocs int, avgLen float64) float64 {
|
||||
const (
|
||||
k1 = 1.2
|
||||
b = 0.75
|
||||
)
|
||||
if length <= 0 || totalDocs <= 0 {
|
||||
return 0
|
||||
}
|
||||
if avgLen <= 0 {
|
||||
avgLen = 1
|
||||
}
|
||||
var score float64
|
||||
docLen := float64(length)
|
||||
for _, term := range queryTerms {
|
||||
tf := counts[term]
|
||||
if tf == 0 {
|
||||
continue
|
||||
}
|
||||
termDF := df[term]
|
||||
if termDF == 0 {
|
||||
continue
|
||||
}
|
||||
idf := math.Log(1 + (float64(totalDocs)-float64(termDF)+0.5)/(float64(termDF)+0.5))
|
||||
freq := float64(tf)
|
||||
score += idf * (freq * (k1 + 1)) / (freq + k1*(1-b+b*docLen/avgLen))
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
// DocumentFrequency counts how many documents contain each term.
|
||||
func DocumentFrequency(docs []map[string]int) map[string]int {
|
||||
df := map[string]int{}
|
||||
for _, counts := range docs {
|
||||
for term := range counts {
|
||||
df[term]++
|
||||
}
|
||||
}
|
||||
return df
|
||||
}
|
||||
|
||||
// KeepTopRelativeScore keeps the best item and drops trailing items whose score
|
||||
// falls below ratio * topScore. Callers must pass items already sorted best
|
||||
// first. This mirrors SQLite FTS/BM25 search UIs that over-fetch, then trim
|
||||
// common-word-only noise without imposing an absolute score threshold.
|
||||
func KeepTopRelativeScore[T any](items []T, ratio float64, score func(T) float64) []T {
|
||||
if len(items) == 0 || ratio <= 0 {
|
||||
return items
|
||||
}
|
||||
top := score(items[0])
|
||||
if top <= 0 {
|
||||
return items
|
||||
}
|
||||
cutoff := top * ratio
|
||||
out := items[:0]
|
||||
for i, item := range items {
|
||||
if i == 0 || score(item) >= cutoff {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// QueryTerms normalizes a search string and reports an error when nothing
|
||||
// searchable remains.
|
||||
func QueryTerms(query string) ([]string, error) {
|
||||
terms := Unique(Tokens(strings.TrimSpace(query)))
|
||||
if len(terms) == 0 {
|
||||
return nil, fmt.Errorf("query must contain at least one letter or number")
|
||||
}
|
||||
return terms, nil
|
||||
}
|
||||
|
||||
// MakeSnippet returns a whitespace-compacted excerpt centered near the query.
|
||||
func MakeSnippet(text, query string, terms []string, maxRunes int) string {
|
||||
text = CompactWhitespace(text)
|
||||
if maxRunes <= 0 || utf8.RuneCountInString(text) <= maxRunes {
|
||||
return text
|
||||
}
|
||||
lower := strings.ToLower(text)
|
||||
query = strings.ToLower(strings.TrimSpace(query))
|
||||
idx := -1
|
||||
if query != "" {
|
||||
idx = strings.Index(lower, query)
|
||||
}
|
||||
if idx < 0 {
|
||||
for _, term := range terms {
|
||||
runes := []rune(term)
|
||||
if len(runes) == 1 && !isCJK(runes[0]) {
|
||||
continue
|
||||
}
|
||||
if i := strings.Index(lower, term); i >= 0 {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
idx = 0
|
||||
}
|
||||
return snippetAround(text, idx, maxRunes)
|
||||
}
|
||||
|
||||
func snippetAround(text string, byteIdx, maxRunes int) string {
|
||||
if byteIdx < 0 {
|
||||
byteIdx = 0
|
||||
}
|
||||
if byteIdx > len(text) {
|
||||
byteIdx = len(text)
|
||||
}
|
||||
for byteIdx > 0 && byteIdx < len(text) && !utf8.RuneStart(text[byteIdx]) {
|
||||
byteIdx--
|
||||
}
|
||||
runes := []rune(text)
|
||||
pos := utf8.RuneCountInString(text[:byteIdx])
|
||||
start := max(pos-maxRunes/2, 0)
|
||||
end := start + maxRunes
|
||||
if end > len(runes) {
|
||||
end = len(runes)
|
||||
start = max(end-maxRunes, 0)
|
||||
}
|
||||
prefix := ""
|
||||
suffix := ""
|
||||
if start > 0 {
|
||||
prefix = "..."
|
||||
}
|
||||
if end < len(runes) {
|
||||
suffix = "..."
|
||||
}
|
||||
return prefix + string(runes[start:end]) + suffix
|
||||
}
|
||||
|
||||
// CompactWhitespace collapses runs of whitespace into one ASCII space.
|
||||
func CompactWhitespace(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
-77
@@ -1,77 +0,0 @@
|
||||
package retrieval
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestTokensHandlesLatinAndCJK(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
in string
|
||||
want []string
|
||||
}{
|
||||
{"BM25 检索 cache-first", []string{"bm25", "检索", "cache", "first"}},
|
||||
{"数据库迁移", []string{"数据", "据库", "库迁", "迁移"}},
|
||||
{"库", []string{"库"}},
|
||||
{"用pnpm装依赖", []string{"用", "pnpm", "装依", "依赖"}},
|
||||
} {
|
||||
got := Tokens(tc.in)
|
||||
if strings.Join(got, ",") != strings.Join(tc.want, ",") {
|
||||
t.Fatalf("Tokens(%q) = %#v, want %#v", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBM25ScoreRanksMatchingDocument(t *testing.T) {
|
||||
query := Unique(Tokens("prompt cache"))
|
||||
doc1 := Counts(Tokens("prompt cache cache stability"))
|
||||
doc2 := Counts(Tokens("dashboard colors"))
|
||||
df := DocumentFrequency([]map[string]int{doc1, doc2})
|
||||
score1 := BM25Score(doc1, 4, query, df, 2, 3)
|
||||
score2 := BM25Score(doc2, 2, query, df, 2, 3)
|
||||
if score1 <= score2 {
|
||||
t.Fatalf("matching score %.3f should exceed unrelated score %.3f", score1, score2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeepTopRelativeScoreKeepsTopAndDropsWeakTail(t *testing.T) {
|
||||
items := []struct {
|
||||
name string
|
||||
score float64
|
||||
}{
|
||||
{name: "top", score: 10},
|
||||
{name: "near", score: 2},
|
||||
{name: "noise", score: 1.4},
|
||||
{name: "zero", score: 0},
|
||||
}
|
||||
got := KeepTopRelativeScore(items, 0.15, func(item struct {
|
||||
name string
|
||||
score float64
|
||||
}) float64 {
|
||||
return item.score
|
||||
})
|
||||
if len(got) != 2 || got[0].name != "top" || got[1].name != "near" {
|
||||
t.Fatalf("KeepTopRelativeScore() = %#v, want top and near", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMakeSnippetHandlesMultibyteBoundary(t *testing.T) {
|
||||
text := strings.Repeat("前缀", 80) + "稳定结论 synthesis cache " + strings.Repeat("后缀", 80)
|
||||
out := MakeSnippet(text, "synthesis cache", QueryTermsForTest(t, "synthesis cache"), 60)
|
||||
if !strings.Contains(out, "synthesis cache") {
|
||||
t.Fatalf("snippet missing query: %q", out)
|
||||
}
|
||||
if strings.ContainsRune(out, utf8.RuneError) {
|
||||
t.Fatalf("snippet contains replacement rune: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func QueryTermsForTest(t *testing.T, query string) []string {
|
||||
t.Helper()
|
||||
terms, err := QueryTerms(query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return terms
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
// Retrieval V2, shadow-only: BM25F field weighting, code-symbol splitting,
|
||||
// and mixed CJK uni+bigrams. Nothing here serves the model — V2 rankings ride
|
||||
// the recall audit next to production's, and MemoryBench decides if V2 ever
|
||||
// takes over. Weights are candidates under measurement, not tuned truths.
|
||||
package retrieval
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// TokensV2 extends Tokens with code-symbol structure and CJK unigrams:
|
||||
// CamelCase and snake_case words also emit their segments (FooBar → foobar,
|
||||
// foo, bar), and CJK runs emit unigrams alongside bigrams so one-character
|
||||
// queries still match. The V1 form of every token is preserved, so V2 recall
|
||||
// is a superset of V1's vocabulary.
|
||||
func TokensV2(s string) []string {
|
||||
var out []string
|
||||
var word []rune
|
||||
var prev rune
|
||||
cjkRun := 0
|
||||
flushWord := func() {
|
||||
if len(word) == 0 {
|
||||
return
|
||||
}
|
||||
lower := strings.ToLower(string(word))
|
||||
out = append(out, lower)
|
||||
for _, seg := range splitCodeSymbol(word) {
|
||||
if seg != lower {
|
||||
out = append(out, seg)
|
||||
}
|
||||
}
|
||||
word = word[:0]
|
||||
}
|
||||
endCJK := func() { cjkRun = 0 }
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case isCJK(r):
|
||||
flushWord()
|
||||
out = append(out, string(r))
|
||||
if cjkRun > 0 {
|
||||
out = append(out, string([]rune{prev, r}))
|
||||
}
|
||||
prev = r
|
||||
cjkRun++
|
||||
case unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_':
|
||||
endCJK()
|
||||
word = append(word, r)
|
||||
default:
|
||||
flushWord()
|
||||
endCJK()
|
||||
}
|
||||
}
|
||||
flushWord()
|
||||
return out
|
||||
}
|
||||
|
||||
// splitCodeSymbol yields the lowercase segments of a code identifier:
|
||||
// CamelCase humps, snake_case parts, and letter/digit boundaries.
|
||||
func splitCodeSymbol(word []rune) []string {
|
||||
var segments []string
|
||||
var seg []rune
|
||||
flush := func() {
|
||||
if len(seg) > 1 { // single letters are noise, not signal
|
||||
segments = append(segments, strings.ToLower(string(seg)))
|
||||
}
|
||||
seg = seg[:0]
|
||||
}
|
||||
for i, r := range word {
|
||||
boundary := r == '_' ||
|
||||
(i > 0 && unicode.IsUpper(r) && !unicode.IsUpper(word[i-1])) ||
|
||||
(i > 0 && unicode.IsDigit(r) != unicode.IsDigit(word[i-1]))
|
||||
if boundary {
|
||||
flush()
|
||||
}
|
||||
if r != '_' {
|
||||
seg = append(seg, r)
|
||||
}
|
||||
}
|
||||
flush()
|
||||
if len(segments) < 2 {
|
||||
return nil // no structure worth indexing beyond the whole word
|
||||
}
|
||||
return segments
|
||||
}
|
||||
|
||||
// FieldedDoc is one document split into weighted fields for BM25F.
|
||||
type FieldedDoc struct {
|
||||
ID string
|
||||
Fields map[string]string
|
||||
}
|
||||
|
||||
// v2FieldWeights follow the BM25F intuition: identity fields say what a fact
|
||||
// IS, the body says everything it mentions. Candidates under measurement.
|
||||
var v2FieldWeights = map[string]float64{
|
||||
"name": 2.5, "title": 2.5, "keywords": 2.0, "subject": 2.0,
|
||||
"description": 1.5, "body": 1.0,
|
||||
}
|
||||
|
||||
// V2Hit is one shadow-ranked document.
|
||||
type V2Hit struct {
|
||||
ID string
|
||||
Score float64
|
||||
}
|
||||
|
||||
// RankV2 scores docs against a query with BM25F over TokensV2: term
|
||||
// frequencies accumulate per field scaled by field weight, then the standard
|
||||
// saturation applies once per term (the F in BM25F), with length
|
||||
// normalization over the weighted document length.
|
||||
func RankV2(query string, docs []FieldedDoc) []V2Hit {
|
||||
queryTerms := Unique(TokensV2(query))
|
||||
if len(queryTerms) == 0 || len(docs) == 0 {
|
||||
return nil
|
||||
}
|
||||
type indexed struct {
|
||||
id string
|
||||
tf map[string]float64
|
||||
length float64
|
||||
}
|
||||
corpus := make([]indexed, 0, len(docs))
|
||||
df := map[string]int{}
|
||||
var totalLen float64
|
||||
for _, doc := range docs {
|
||||
ix := indexed{id: doc.ID, tf: map[string]float64{}}
|
||||
seen := map[string]bool{}
|
||||
for field, text := range doc.Fields {
|
||||
weight, ok := v2FieldWeights[field]
|
||||
if !ok {
|
||||
weight = 1.0
|
||||
}
|
||||
for _, term := range TokensV2(text) {
|
||||
ix.tf[term] += weight
|
||||
ix.length += weight
|
||||
seen[term] = true
|
||||
}
|
||||
}
|
||||
for term := range seen {
|
||||
df[term]++
|
||||
}
|
||||
totalLen += ix.length
|
||||
corpus = append(corpus, ix)
|
||||
}
|
||||
avgLen := totalLen / float64(len(corpus))
|
||||
if avgLen <= 0 {
|
||||
avgLen = 1
|
||||
}
|
||||
const k1, b = 1.2, 0.75
|
||||
var hits []V2Hit
|
||||
for _, doc := range corpus {
|
||||
var score float64
|
||||
for _, term := range queryTerms {
|
||||
tf := doc.tf[term]
|
||||
if tf == 0 || df[term] == 0 {
|
||||
continue
|
||||
}
|
||||
idf := math.Log(1 + (float64(len(corpus))-float64(df[term])+0.5)/(float64(df[term])+0.5))
|
||||
score += idf * (tf * (k1 + 1)) / (tf + k1*(1-b+b*doc.length/avgLen))
|
||||
}
|
||||
if score > 0 {
|
||||
hits = append(hits, V2Hit{ID: doc.id, Score: score})
|
||||
}
|
||||
}
|
||||
SortHitsDesc(hits)
|
||||
return hits
|
||||
}
|
||||
|
||||
// SortHitsDesc orders shadow hits best-first with a stable ID tiebreak.
|
||||
func SortHitsDesc(hits []V2Hit) {
|
||||
for i := 1; i < len(hits); i++ {
|
||||
for j := i; j > 0 && (hits[j].Score > hits[j-1].Score ||
|
||||
(hits[j].Score == hits[j-1].Score && hits[j].ID < hits[j-1].ID)); j-- {
|
||||
hits[j], hits[j-1] = hits[j-1], hits[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
-47
@@ -1,47 +0,0 @@
|
||||
package retrieval
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTokensV2CodeSymbolsAndCJK(t *testing.T) {
|
||||
got := strings.Join(TokensV2("RetryPolicy.MaxBackoffMs ticket_1827 数据库"), " ")
|
||||
for _, want := range []string{"retrypolicy", "retry", "policy", "maxbackoffms", "max", "backoff", "ticket_1827", "ticket", "1827", "数", "据", "库", "数据", "据库"} {
|
||||
if !strings.Contains(" "+got+" ", " "+want+" ") {
|
||||
t.Fatalf("TokensV2 missing %q in %q", want, got)
|
||||
}
|
||||
}
|
||||
if strings.Contains(" "+got+" ", " ms ") {
|
||||
// "Ms" splits to a single meaningful segment only when len > 1; "ms" is
|
||||
// fine — assert single letters never appear instead.
|
||||
t.Log("ms segment present (acceptable)")
|
||||
}
|
||||
for _, tok := range TokensV2("A B C") {
|
||||
if len(tok) == 0 {
|
||||
t.Fatal("empty token emitted")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRankV2FieldWeightsBeatBodyMentions(t *testing.T) {
|
||||
docs := []FieldedDoc{
|
||||
{ID: "titled", Fields: map[string]string{"title": "Package manager", "body": "We migrated to pnpm."}},
|
||||
{ID: "mention", Fields: map[string]string{"title": "Sprint notes", "body": "Someone asked about the package manager once; also weather, lunch, and a package of stickers."}},
|
||||
}
|
||||
hits := RankV2("package manager", docs)
|
||||
if len(hits) != 2 || hits[0].ID != "titled" {
|
||||
t.Fatalf("field-weighted doc must outrank body mentions: %+v", hits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRankV2FindsCodeSymbolBySegment(t *testing.T) {
|
||||
docs := []FieldedDoc{
|
||||
{ID: "sym", Fields: map[string]string{"body": "Configure RetryPolicy.MaxBackoffMs in policy/retry.toml."}},
|
||||
{ID: "other", Fields: map[string]string{"body": "Unrelated fact about deploys."}},
|
||||
}
|
||||
hits := RankV2("max backoff configuration", docs)
|
||||
if len(hits) == 0 || hits[0].ID != "sym" {
|
||||
t.Fatalf("segment query must find the camel-case symbol: %+v", hits)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
// Package rulelist splits a list of permission rules written on one line.
|
||||
//
|
||||
// Rules are separated by commas or whitespace, and a separator inside
|
||||
// parentheses belongs to the rule: `Bash(git add:*) Read` is two rules. Skill
|
||||
// front matter and the --allowed-tools flag share it, so one spelling means
|
||||
// one thing in both.
|
||||
package rulelist
|
||||
-56
@@ -1,56 +0,0 @@
|
||||
package rulelist
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrUnclosed: an opening parenthesis is never closed.
|
||||
ErrUnclosed = errors.New("unclosed '('")
|
||||
// ErrUnexpectedClose: a closing parenthesis has no opening one.
|
||||
ErrUnexpectedClose = errors.New("unexpected ')'")
|
||||
)
|
||||
|
||||
// Split returns the rules in value, in order, trimmed and non-empty.
|
||||
func Split(value string) ([]string, error) {
|
||||
var rules []string
|
||||
start, depth := -1, 0
|
||||
flush := func(end int) {
|
||||
if start < 0 {
|
||||
return
|
||||
}
|
||||
if rule := strings.TrimSpace(value[start:end]); rule != "" {
|
||||
rules = append(rules, rule)
|
||||
}
|
||||
start = -1
|
||||
}
|
||||
for i, r := range value {
|
||||
switch r {
|
||||
case '(':
|
||||
if start < 0 {
|
||||
start = i
|
||||
}
|
||||
depth++
|
||||
case ')':
|
||||
if depth == 0 {
|
||||
return nil, ErrUnexpectedClose
|
||||
}
|
||||
depth--
|
||||
default:
|
||||
if depth == 0 && (r == ',' || unicode.IsSpace(r)) {
|
||||
flush(i)
|
||||
continue
|
||||
}
|
||||
if start < 0 {
|
||||
start = i
|
||||
}
|
||||
}
|
||||
}
|
||||
if depth != 0 {
|
||||
return nil, ErrUnclosed
|
||||
}
|
||||
flush(len(value))
|
||||
return rules, nil
|
||||
}
|
||||
-40
@@ -1,40 +0,0 @@
|
||||
package rulelist
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSplitSeparatesOnCommasAndWhitespaceOutsideParentheses(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
in string
|
||||
want []string
|
||||
}{
|
||||
{"", nil},
|
||||
{" ", nil},
|
||||
{"Read", []string{"Read"}},
|
||||
{"read_file, grep,bash", []string{"read_file", "grep", "bash"}},
|
||||
{"Read Grep Bash", []string{"Read", "Grep", "Bash"}},
|
||||
{"Bash(git add:*) Read", []string{"Bash(git add:*)", "Read"}},
|
||||
{"Bash(git add:*, git commit:*), Read", []string{"Bash(git add:*, git commit:*)", "Read"}},
|
||||
{"bash,", []string{"bash"}},
|
||||
} {
|
||||
got, err := Split(tc.in)
|
||||
if err != nil {
|
||||
t.Fatalf("Split(%q): %v", tc.in, err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("Split(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitReportsUnbalancedParenthesesByIdentity(t *testing.T) {
|
||||
if _, err := Split("Bash(git add"); !errors.Is(err, ErrUnclosed) {
|
||||
t.Errorf("unclosed = %v", err)
|
||||
}
|
||||
if _, err := Split("Bash)"); !errors.Is(err, ErrUnexpectedClose) {
|
||||
t.Errorf("unexpected close = %v", err)
|
||||
}
|
||||
}
|
||||
-25
@@ -1,25 +0,0 @@
|
||||
// Package tokencount estimates how many tokens a string costs a model, without
|
||||
// carrying a tokenizer. Anything that has to size text before a provider counts
|
||||
// it — a cache-shape diagnostic, a tool call's context footprint — needs one
|
||||
// answer to that question rather than a private heuristic per caller.
|
||||
package tokencount
|
||||
|
||||
import "unicode/utf8"
|
||||
|
||||
// Text estimates the token count of s. The split is by script because the two
|
||||
// behave nothing alike — Latin text and code trend near four bytes per token,
|
||||
// CJK closer to one token per character — and either rule applied to the whole
|
||||
// string is wrong by ~4x on the other half. A real tokenizer would be exact and
|
||||
// would mean a vocabulary per model, re-run over whole transcripts every turn.
|
||||
func Text(s string) int {
|
||||
narrow, wide := 0, 0
|
||||
for _, r := range s {
|
||||
if r < utf8.RuneSelf {
|
||||
narrow++
|
||||
continue
|
||||
}
|
||||
wide++
|
||||
}
|
||||
// Round up: a long run of short strings must not sum to nothing.
|
||||
return (narrow+3)/4 + wide
|
||||
}
|
||||
-27
@@ -1,27 +0,0 @@
|
||||
package tokencount
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestTextDoesNotUndercountCJK(t *testing.T) {
|
||||
// The reason this package exists: bytes/4 reads a Chinese transcript as a
|
||||
// third of its real size, so a planner sizing context off it compacts late.
|
||||
const cjk = "把这个仓库跑一遍测试，把失败的那几个定位到具体文件"
|
||||
runes := len([]rune(cjk))
|
||||
if got := Text(cjk); got != runes {
|
||||
t.Fatalf("Text(cjk) = %d, want one token per rune (%d)", got, runes)
|
||||
}
|
||||
if byBytes := len(cjk) / 4; Text(cjk) <= byBytes {
|
||||
t.Fatalf("Text(cjk) = %d, must exceed the bytes/4 reading (%d)", Text(cjk), byBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextRoundsASCIIUpAndKeepsEmptyFree(t *testing.T) {
|
||||
if got := Text(""); got != 0 {
|
||||
t.Fatalf("Text(\"\") = %d, want 0", got)
|
||||
}
|
||||
// Five bytes is more than one token's worth; truncating would let a long
|
||||
// run of short strings sum to nothing.
|
||||
if got := Text("abcde"); got != 2 {
|
||||
t.Fatalf("Text(\"abcde\") = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
-200
@@ -1,200 +0,0 @@
|
||||
// Package ablation switches individual Reasonix subsystems off so a benchmark
|
||||
// can attribute a change in solve rate to one of them.
|
||||
package ablation
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Module string
|
||||
|
||||
const (
|
||||
Evidence Module = "evidence"
|
||||
Planner Module = "planner"
|
||||
Subagent Module = "subagent"
|
||||
Retrieval Module = "retrieval"
|
||||
Compaction Module = "compaction"
|
||||
// Upstream off means a fleet dependency edge orders its endpoints without
|
||||
// delivering the dependency's answer, so a dependent must re-derive it.
|
||||
Upstream Module = "upstream"
|
||||
// RecallSearch off leaves recall able to read an address it was given and
|
||||
// unable to look one up, which is what the tool could do before the folded
|
||||
// region became searchable.
|
||||
RecallSearch Module = "recall-search"
|
||||
)
|
||||
|
||||
// FoldIndexScale sizes the model-visible fold index against its shipped budget.
|
||||
// It is a bench axis, not a module: the question it answers is what an ambient
|
||||
// address list is worth, and four steps answer that where a free-form fraction
|
||||
// would turn a measurement into a tuning knob.
|
||||
type FoldIndexScale string
|
||||
|
||||
const (
|
||||
FoldIndexDefault FoldIndexScale = ""
|
||||
FoldIndexHalf FoldIndexScale = "half"
|
||||
FoldIndexQuarter FoldIndexScale = "quarter"
|
||||
FoldIndexOff FoldIndexScale = "off"
|
||||
)
|
||||
|
||||
// Ratio scales a budget. The default returns it unchanged.
|
||||
func (f FoldIndexScale) Ratio() float64 {
|
||||
switch f {
|
||||
case FoldIndexHalf:
|
||||
return 0.5
|
||||
case FoldIndexQuarter:
|
||||
return 0.25
|
||||
case FoldIndexOff:
|
||||
return 0
|
||||
default:
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
// ParseFoldIndexScale reads a bench flag value. "" and "default" ship as-is.
|
||||
func ParseFoldIndexScale(spec string) (FoldIndexScale, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(spec)) {
|
||||
case "", "default", "full":
|
||||
return FoldIndexDefault, nil
|
||||
case string(FoldIndexHalf):
|
||||
return FoldIndexHalf, nil
|
||||
case string(FoldIndexQuarter):
|
||||
return FoldIndexQuarter, nil
|
||||
case string(FoldIndexOff), "none", "0":
|
||||
return FoldIndexOff, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown fold-index scale %q (want default, half, quarter, or off)", spec)
|
||||
}
|
||||
}
|
||||
|
||||
// Modules returns every switchable module in the order arm names use.
|
||||
func Modules() []Module {
|
||||
return []Module{Evidence, Planner, Subagent, Retrieval, Compaction, Upstream, RecallSearch}
|
||||
}
|
||||
|
||||
// Set is the group of modules disabled for a run. The zero value is the
|
||||
// control arm: everything on.
|
||||
type Set struct {
|
||||
off map[Module]bool
|
||||
// foldIndex is an axis rather than a switch, so it lives beside the module
|
||||
// set instead of inside it. It still reaches Arm(): a run whose index was
|
||||
// quartered must not report the same identity as the control.
|
||||
foldIndex FoldIndexScale
|
||||
}
|
||||
|
||||
// WithFoldIndex returns this set at the given index scale.
|
||||
func (s Set) WithFoldIndex(scale FoldIndexScale) Set {
|
||||
s.foldIndex = scale
|
||||
return s
|
||||
}
|
||||
|
||||
// FoldIndex is the scale this run sizes its fold index at.
|
||||
func (s Set) FoldIndex() FoldIndexScale { return s.foldIndex }
|
||||
|
||||
// Parse reads a spec such as "evidence,planner". "" and "none" mean the control
|
||||
// arm; "all" disables every module.
|
||||
func Parse(spec string) (Set, error) {
|
||||
spec = strings.TrimSpace(spec)
|
||||
if spec == "" || strings.EqualFold(spec, "none") {
|
||||
return Set{}, nil
|
||||
}
|
||||
if strings.EqualFold(spec, "all") {
|
||||
return New(Modules()...), nil
|
||||
}
|
||||
known := map[Module]bool{}
|
||||
for _, m := range Modules() {
|
||||
known[m] = true
|
||||
}
|
||||
var mods []Module
|
||||
for _, field := range strings.FieldsFunc(spec, func(r rune) bool { return r == ',' || r == ' ' }) {
|
||||
m := Module(strings.ToLower(strings.TrimSpace(field)))
|
||||
if !known[m] {
|
||||
return Set{}, fmt.Errorf("unknown ablation module %q (want %s, or none/all)", field, ModuleList())
|
||||
}
|
||||
mods = append(mods, m)
|
||||
}
|
||||
return New(mods...), nil
|
||||
}
|
||||
|
||||
// ParseArm reads both bench axes at once: the module spec and the fold-index
|
||||
// scale. One call so a frontend cannot wire the first and forget the second,
|
||||
// which would silently run an index arm under the control arm's name.
|
||||
func ParseArm(modules, foldIndex string) (Set, error) {
|
||||
set, err := Parse(modules)
|
||||
if err != nil {
|
||||
return Set{}, err
|
||||
}
|
||||
scale, err := ParseFoldIndexScale(foldIndex)
|
||||
if err != nil {
|
||||
return Set{}, err
|
||||
}
|
||||
return set.WithFoldIndex(scale), nil
|
||||
}
|
||||
|
||||
// ModuleList names every switchable module for flag help and error text, so
|
||||
// those cannot drift from Modules() the way a hand-kept list does.
|
||||
func ModuleList() string { return joinModules(Modules(), ", ") }
|
||||
|
||||
// New returns a Set with the given modules disabled.
|
||||
func New(mods ...Module) Set {
|
||||
if len(mods) == 0 {
|
||||
return Set{}
|
||||
}
|
||||
off := make(map[Module]bool, len(mods))
|
||||
for _, m := range mods {
|
||||
off[m] = true
|
||||
}
|
||||
return Set{off: off}
|
||||
}
|
||||
|
||||
func (s Set) Off(m Module) bool { return s.off[m] }
|
||||
|
||||
func (s Set) Empty() bool { return len(s.off) == 0 && s.foldIndex == FoldIndexDefault }
|
||||
|
||||
// Arm is the published name of this configuration: "full" for the control arm,
|
||||
// otherwise "no-evidence+no-planner". Stable across runs so results from
|
||||
// different machines group by the same key.
|
||||
func (s Set) Arm() string {
|
||||
if s.Empty() {
|
||||
return "full"
|
||||
}
|
||||
parts := make([]string, 0, len(s.off)+1)
|
||||
for _, m := range s.disabled() {
|
||||
parts = append(parts, "no-"+string(m))
|
||||
}
|
||||
if s.foldIndex != FoldIndexDefault {
|
||||
parts = append(parts, "index-"+string(s.foldIndex))
|
||||
}
|
||||
return strings.Join(parts, "+")
|
||||
}
|
||||
|
||||
// String round-trips back through Parse. It carries modules only — the fold
|
||||
// index axis has its own flag and its own parser.
|
||||
func (s Set) String() string {
|
||||
if s.Empty() {
|
||||
return "none"
|
||||
}
|
||||
return joinModules(s.disabled(), ",")
|
||||
}
|
||||
|
||||
func (s Set) disabled() []Module {
|
||||
order := map[Module]int{}
|
||||
for i, m := range Modules() {
|
||||
order[m] = i
|
||||
}
|
||||
out := make([]Module, 0, len(s.off))
|
||||
for m := range s.off {
|
||||
out = append(out, m)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return order[out[i]] < order[out[j]] })
|
||||
return out
|
||||
}
|
||||
|
||||
func joinModules(mods []Module, sep string) string {
|
||||
parts := make([]string, len(mods))
|
||||
for i, m := range mods {
|
||||
parts[i] = string(m)
|
||||
}
|
||||
return strings.Join(parts, sep)
|
||||
}
|
||||
-108
@@ -1,108 +0,0 @@
|
||||
package ablation
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseAcceptsSpecFormsAndRejectsUnknownModules(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
spec string
|
||||
arm string
|
||||
}{
|
||||
{"", "full"},
|
||||
{"none", "full"},
|
||||
{"NONE", "full"},
|
||||
{"evidence", "no-evidence"},
|
||||
{"planner,evidence", "no-evidence+no-planner"},
|
||||
{"planner evidence", "no-evidence+no-planner"},
|
||||
{" Evidence , Retrieval ", "no-evidence+no-retrieval"},
|
||||
{"all", "no-evidence+no-planner+no-subagent+no-retrieval+no-compaction+no-upstream+no-recall-search"},
|
||||
} {
|
||||
set, err := Parse(tc.spec)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse(%q): %v", tc.spec, err)
|
||||
}
|
||||
if got := set.Arm(); got != tc.arm {
|
||||
t.Errorf("Parse(%q).Arm() = %q, want %q", tc.spec, got, tc.arm)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := Parse("evidence,planer"); err == nil {
|
||||
t.Fatal("a misspelled module must fail loudly, not silently run the control arm")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArmNameIsIndependentOfSpecOrder(t *testing.T) {
|
||||
a, _ := Parse("retrieval,evidence,planner")
|
||||
b, _ := Parse("planner,retrieval,evidence")
|
||||
if a.Arm() != b.Arm() {
|
||||
t.Fatalf("arm names diverge by input order: %q vs %q", a.Arm(), b.Arm())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStringRoundTripsThroughParse(t *testing.T) {
|
||||
for _, spec := range []string{"none", "evidence", "evidence,compaction", "all"} {
|
||||
set, err := Parse(spec)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse(%q): %v", spec, err)
|
||||
}
|
||||
again, err := Parse(set.String())
|
||||
if err != nil {
|
||||
t.Fatalf("Parse(%q): %v", set.String(), err)
|
||||
}
|
||||
if again.Arm() != set.Arm() {
|
||||
t.Errorf("%q round-tripped to %q", set.Arm(), again.Arm())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The index axis is not a module, but a run that moved it is not the control
|
||||
// arm either: results keyed by arm name would otherwise merge with it.
|
||||
func TestFoldIndexScaleNamesItsOwnArm(t *testing.T) {
|
||||
var s Set
|
||||
if got := s.WithFoldIndex(FoldIndexQuarter).Arm(); got != "index-quarter" {
|
||||
t.Errorf("Arm() = %q, want index-quarter", got)
|
||||
}
|
||||
if got := New(Evidence).WithFoldIndex(FoldIndexOff).Arm(); got != "no-evidence+index-off" {
|
||||
t.Errorf("Arm() = %q, want no-evidence+index-off", got)
|
||||
}
|
||||
if s.WithFoldIndex(FoldIndexQuarter).Empty() {
|
||||
t.Error("a quartered index reported itself as the control arm")
|
||||
}
|
||||
if !s.WithFoldIndex(FoldIndexDefault).Empty() {
|
||||
t.Error("the default scale is the control arm")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFoldIndexScaleRatios(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
spec string
|
||||
scale FoldIndexScale
|
||||
ratio float64
|
||||
}{
|
||||
{"", FoldIndexDefault, 1}, {"default", FoldIndexDefault, 1},
|
||||
{"half", FoldIndexHalf, 0.5}, {"QUARTER", FoldIndexQuarter, 0.25},
|
||||
{"off", FoldIndexOff, 0}, {"0", FoldIndexOff, 0},
|
||||
} {
|
||||
got, err := ParseFoldIndexScale(tc.spec)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFoldIndexScale(%q): %v", tc.spec, err)
|
||||
}
|
||||
if got != tc.scale || got.Ratio() != tc.ratio {
|
||||
t.Errorf("ParseFoldIndexScale(%q) = %q (ratio %v), want %q (%v)", tc.spec, got, got.Ratio(), tc.scale, tc.ratio)
|
||||
}
|
||||
}
|
||||
if _, err := ParseFoldIndexScale("0.37"); err == nil {
|
||||
t.Fatal("a free-form fraction was accepted; the axis is four steps on purpose")
|
||||
}
|
||||
}
|
||||
|
||||
func TestZeroValueIsTheControlArm(t *testing.T) {
|
||||
var s Set
|
||||
if !s.Empty() || s.Arm() != "full" {
|
||||
t.Fatalf("zero Set = %q, want the full control arm", s.Arm())
|
||||
}
|
||||
for _, m := range Modules() {
|
||||
if s.Off(m) {
|
||||
t.Errorf("zero Set disabled %s", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
// Package agentgraph is the vocabulary of one run's execution graph: the nodes
|
||||
// a turn handed work to, and the typed edges between them.
|
||||
//
|
||||
// It exists because that structure was being re-derived in five places and
|
||||
// survived in none. A fleet's dependency DAG died with the call that built it,
|
||||
// the persisted sidecar recorded a parent link and nothing else, and the event
|
||||
// stream implied edges through id prefixes. A reader asking why a node waited
|
||||
// could not tell an unmet dependency from a delivered answer, because neither
|
||||
// had ever been written down as anything a reader could tell apart.
|
||||
//
|
||||
// Nothing here knows what an agent, a tool, or a provider is. A producer
|
||||
// publishes a Delta of what it has just proven; every consumer folds deltas
|
||||
// with the same Apply, so the graph a headless run reports is the graph the
|
||||
// desktop draws.
|
||||
package agentgraph
|
||||
-219
@@ -1,219 +0,0 @@
|
||||
package agentgraph
|
||||
|
||||
import "slices"
|
||||
|
||||
// NodeKind separates the three things a node can be. A group is the call that
|
||||
// owns a fan-out, a worker is one delegated run, and an external node is work
|
||||
// this run did not do — named only because an edge points at it.
|
||||
type NodeKind string
|
||||
|
||||
const (
|
||||
KindGroup NodeKind = "group"
|
||||
KindWorker NodeKind = "worker"
|
||||
KindExternal NodeKind = "external"
|
||||
)
|
||||
|
||||
// NodeState is where one node stands. It is the node's own lifecycle and not a
|
||||
// tool call's: an adopted node never ran and still holds an answer, and a
|
||||
// skipped one never started because something upstream did not finish.
|
||||
type NodeState string
|
||||
|
||||
const (
|
||||
StatePending NodeState = "pending"
|
||||
// StateQueued is ready and waiting for a session concurrency slot. It is
|
||||
// not StatePending: nothing upstream is missing, and the only thing between
|
||||
// this node and its first step is the scheduler's ceiling.
|
||||
StateQueued NodeState = "queued"
|
||||
StateRunning NodeState = "running"
|
||||
StateCompleted NodeState = "completed"
|
||||
StateAdopted NodeState = "adopted"
|
||||
StateFailed NodeState = "failed"
|
||||
StateCancelled NodeState = "cancelled"
|
||||
StateSkipped NodeState = "skipped"
|
||||
)
|
||||
|
||||
// Answered reports whether this state holds a final answer a dependent may
|
||||
// start from: one the node produced, or one adopted in its place.
|
||||
func (s NodeState) Answered() bool { return s == StateCompleted || s == StateAdopted }
|
||||
|
||||
// Terminal reports whether the node will not change state again.
|
||||
func (s NodeState) Terminal() bool {
|
||||
return s != StatePending && s != StateQueued && s != StateRunning
|
||||
}
|
||||
|
||||
// Grant is what a node was allowed to touch while it ran. Empty means the
|
||||
// question does not apply — a group is not a run, and an adopted node never got
|
||||
// one — which a bool could not say apart from "may write".
|
||||
type Grant string
|
||||
|
||||
const (
|
||||
GrantRead Grant = "read"
|
||||
GrantWrite Grant = "write"
|
||||
)
|
||||
|
||||
// WaitCause is why a queued node is not running. Queued says a node is ready and
|
||||
// held back; it does not say by what, and the answers have different remedies —
|
||||
// two are session settings and the third is a path or an edge.
|
||||
type WaitCause string
|
||||
|
||||
const (
|
||||
// WaitSlots is the session's total sub-agent ceiling.
|
||||
WaitSlots WaitCause = "slots"
|
||||
// WaitWriters is the session's concurrent-writer ceiling, which binds a
|
||||
// writer while total capacity is still free.
|
||||
WaitWriters WaitCause = "writers"
|
||||
// WaitClaim is a write path someone else already holds. No ceiling change
|
||||
// releases it: only disjoint paths, or an edge that orders the two.
|
||||
WaitClaim WaitCause = "claim"
|
||||
)
|
||||
|
||||
// EdgeKind is what one edge means. The node pair alone says nothing: waiting
|
||||
// for an answer and receiving one are separate facts, and an arm that orders
|
||||
// two nodes while delivering nothing between them is the case a single edge
|
||||
// type cannot describe.
|
||||
type EdgeKind string
|
||||
|
||||
const (
|
||||
// Spawn is structural: To was created by From.
|
||||
Spawn EdgeKind = "spawn"
|
||||
// Depends orders: To may not start until From has answered.
|
||||
Depends EdgeKind = "depends"
|
||||
// Context carries: From's answer opened To's run.
|
||||
Context EdgeKind = "context"
|
||||
// Adopt reuses: From's existing answer stood in for running To, which
|
||||
// therefore cost nothing. From is usually outside this graph.
|
||||
Adopt EdgeKind = "adopt"
|
||||
)
|
||||
|
||||
// Node is one participant in a run. Zero fields mean "not learned yet", which
|
||||
// is what lets a producer report an outcome without restating the identity it
|
||||
// declared when the node entered the graph.
|
||||
type Node struct {
|
||||
ID string `json:"id"`
|
||||
ParentID string `json:"parentId,omitempty"`
|
||||
Kind NodeKind `json:"kind,omitempty"`
|
||||
State NodeState `json:"state,omitempty"`
|
||||
Label string `json:"label,omitempty"`
|
||||
Profile string `json:"profile,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Effort string `json:"effort,omitempty"`
|
||||
Grant Grant `json:"grant,omitempty"`
|
||||
// Wait is what held this node out of a slot. Like the stamps it survives the
|
||||
// run starting: the state says it is running now, this says what it waited on.
|
||||
Wait WaitCause `json:"wait,omitempty"`
|
||||
// Ref is the persisted transcript this node's answer reads back from.
|
||||
Ref string `json:"ref,omitempty"`
|
||||
// QueuedAt, StartedAt and EndedAt are unix milliseconds, zero while unknown.
|
||||
// QueuedAt to StartedAt is slot wait, which only the scheduler can measure.
|
||||
QueuedAt int64 `json:"queuedAt,omitempty"`
|
||||
StartedAt int64 `json:"startedAt,omitempty"`
|
||||
EndedAt int64 `json:"endedAt,omitempty"`
|
||||
Err string `json:"err,omitempty"`
|
||||
}
|
||||
|
||||
// Edge is one typed relation between two node ids. From may name a node outside
|
||||
// this graph: an adopted answer's source is work some earlier run did.
|
||||
type Edge struct {
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
Kind EdgeKind `json:"kind"`
|
||||
}
|
||||
|
||||
// Delta is one publication of what a producer has just proven. It is additive:
|
||||
// a node named again updates the one already there rather than forking into a
|
||||
// second, and a repeated edge is the same edge.
|
||||
type Delta struct {
|
||||
Nodes []Node `json:"nodes,omitempty"`
|
||||
Edges []Edge `json:"edges,omitempty"`
|
||||
}
|
||||
|
||||
// Empty reports whether this delta carries nothing worth publishing.
|
||||
func (d Delta) Empty() bool { return len(d.Nodes) == 0 && len(d.Edges) == 0 }
|
||||
|
||||
// Graph is the folded result. Nodes and edges keep first-seen order so two
|
||||
// consumers fed the same deltas draw the same picture. Lookups scan rather than
|
||||
// index: a graph is bounded by one call's fan-out, and a map would have to be
|
||||
// rebuilt every time the value crossed a wire.
|
||||
type Graph struct {
|
||||
Nodes []Node `json:"nodes,omitempty"`
|
||||
Edges []Edge `json:"edges,omitempty"`
|
||||
}
|
||||
|
||||
// Apply folds one delta into the graph.
|
||||
func (g *Graph) Apply(d Delta) {
|
||||
for _, n := range d.Nodes {
|
||||
g.upsert(n)
|
||||
}
|
||||
for _, e := range d.Edges {
|
||||
g.link(e)
|
||||
}
|
||||
}
|
||||
|
||||
// Node returns the node with this id.
|
||||
func (g *Graph) Node(id string) (Node, bool) {
|
||||
if i := g.indexOf(id); i >= 0 {
|
||||
return g.Nodes[i], true
|
||||
}
|
||||
return Node{}, false
|
||||
}
|
||||
|
||||
func (g *Graph) indexOf(id string) int {
|
||||
return slices.IndexFunc(g.Nodes, func(n Node) bool { return n.ID == id })
|
||||
}
|
||||
|
||||
func (g *Graph) upsert(n Node) {
|
||||
if n.ID == "" {
|
||||
return
|
||||
}
|
||||
if i := g.indexOf(n.ID); i >= 0 {
|
||||
g.Nodes[i] = merge(g.Nodes[i], n)
|
||||
return
|
||||
}
|
||||
g.Nodes = append(g.Nodes, n)
|
||||
}
|
||||
|
||||
// merge keeps what the graph already knew wherever the update says nothing, so
|
||||
// an outcome-only update cannot blank the identity an earlier delta declared.
|
||||
func merge(old, update Node) Node {
|
||||
out := update
|
||||
out.ParentID = orText(update.ParentID, old.ParentID)
|
||||
out.Kind = NodeKind(orText(string(update.Kind), string(old.Kind)))
|
||||
out.State = NodeState(orText(string(update.State), string(old.State)))
|
||||
out.Label = orText(update.Label, old.Label)
|
||||
out.Profile = orText(update.Profile, old.Profile)
|
||||
out.Model = orText(update.Model, old.Model)
|
||||
out.Effort = orText(update.Effort, old.Effort)
|
||||
out.Ref = orText(update.Ref, old.Ref)
|
||||
out.Err = orText(update.Err, old.Err)
|
||||
out.Grant = Grant(orText(string(update.Grant), string(old.Grant)))
|
||||
out.Wait = WaitCause(orText(string(update.Wait), string(old.Wait)))
|
||||
out.QueuedAt = orStamp(update.QueuedAt, old.QueuedAt)
|
||||
out.StartedAt = orStamp(update.StartedAt, old.StartedAt)
|
||||
out.EndedAt = orStamp(update.EndedAt, old.EndedAt)
|
||||
return out
|
||||
}
|
||||
|
||||
// link adds an edge once. A self-edge or a half-named one is a producer bug the
|
||||
// graph refuses to record rather than render as a loop nothing can explain.
|
||||
func (g *Graph) link(e Edge) {
|
||||
if e.From == "" || e.To == "" || e.Kind == "" || e.From == e.To {
|
||||
return
|
||||
}
|
||||
if !slices.Contains(g.Edges, e) {
|
||||
g.Edges = append(g.Edges, e)
|
||||
}
|
||||
}
|
||||
|
||||
func orText(update, old string) string {
|
||||
if update != "" {
|
||||
return update
|
||||
}
|
||||
return old
|
||||
}
|
||||
|
||||
func orStamp(update, old int64) int64 {
|
||||
if update != 0 {
|
||||
return update
|
||||
}
|
||||
return old
|
||||
}
|
||||
-153
@@ -1,153 +0,0 @@
|
||||
package agentgraph
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestApplyUpsertsNodesByID(t *testing.T) {
|
||||
var g Graph
|
||||
g.Apply(Delta{Nodes: []Node{
|
||||
{ID: "a", Kind: KindWorker, State: StatePending, Label: "research", Profile: "explore", Grant: GrantRead},
|
||||
}})
|
||||
// An outcome-only update is what a producer sends once the node settles.
|
||||
g.Apply(Delta{Nodes: []Node{{ID: "a", State: StateCompleted, Ref: "sub-1", EndedAt: 42}}})
|
||||
|
||||
if len(g.Nodes) != 1 {
|
||||
t.Fatalf("nodes = %d, want 1 (the update forked the node)", len(g.Nodes))
|
||||
}
|
||||
n, ok := g.Node("a")
|
||||
if !ok {
|
||||
t.Fatal("node a missing")
|
||||
}
|
||||
if n.State != StateCompleted || n.Ref != "sub-1" || n.EndedAt != 42 {
|
||||
t.Fatalf("update did not land: %+v", n)
|
||||
}
|
||||
if n.Label != "research" || n.Profile != "explore" || n.Kind != KindWorker || n.Grant != GrantRead {
|
||||
t.Fatalf("outcome-only update blanked declared identity: %+v", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyIgnoresUnusableRecords(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
delta Delta
|
||||
}{
|
||||
{"node without id", Delta{Nodes: []Node{{State: StateRunning}}}},
|
||||
{"edge without source", Delta{Edges: []Edge{{To: "b", Kind: Depends}}}},
|
||||
{"edge without target", Delta{Edges: []Edge{{From: "a", Kind: Depends}}}},
|
||||
{"edge without kind", Delta{Edges: []Edge{{From: "a", To: "b"}}}},
|
||||
{"self edge", Delta{Edges: []Edge{{From: "a", To: "a", Kind: Depends}}}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var g Graph
|
||||
g.Apply(c.delta)
|
||||
if len(g.Nodes) != 0 || len(g.Edges) != 0 {
|
||||
t.Fatalf("recorded %+v", g)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyKeepsEdgesDistinctByKind(t *testing.T) {
|
||||
var g Graph
|
||||
// depends and context join the same pair and are not the same fact: an arm
|
||||
// that orders without delivering emits the first and not the second.
|
||||
edges := []Edge{
|
||||
{From: "a", To: "b", Kind: Depends},
|
||||
{From: "a", To: "b", Kind: Context},
|
||||
{From: "a", To: "b", Kind: Depends},
|
||||
}
|
||||
g.Apply(Delta{Edges: edges})
|
||||
if len(g.Edges) != 2 {
|
||||
t.Fatalf("edges = %v, want depends and context once each", g.Edges)
|
||||
}
|
||||
for _, want := range []Edge{{From: "a", To: "b", Kind: Depends}, {From: "a", To: "b", Kind: Context}} {
|
||||
if !slices.Contains(g.Edges, want) {
|
||||
t.Fatalf("missing %+v in %v", want, g.Edges)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyIsOrderStable(t *testing.T) {
|
||||
build := func() Graph {
|
||||
var g Graph
|
||||
for _, id := range []string{"a", "b", "c"} {
|
||||
g.Apply(Delta{Nodes: []Node{{ID: id, Kind: KindWorker, State: StatePending}}})
|
||||
}
|
||||
g.Apply(Delta{Nodes: []Node{{ID: "b", State: StateRunning}}})
|
||||
return g
|
||||
}
|
||||
first, second := build(), build()
|
||||
if len(first.Nodes) != 3 {
|
||||
t.Fatalf("nodes = %d, want 3", len(first.Nodes))
|
||||
}
|
||||
for i := range first.Nodes {
|
||||
if first.Nodes[i] != second.Nodes[i] {
|
||||
t.Fatalf("node %d diverged: %+v vs %+v", i, first.Nodes[i], second.Nodes[i])
|
||||
}
|
||||
}
|
||||
if first.Nodes[1].ID != "b" || first.Nodes[1].State != StateRunning {
|
||||
t.Fatalf("an update reordered the graph: %+v", first.Nodes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStateVocabulary(t *testing.T) {
|
||||
cases := []struct {
|
||||
state NodeState
|
||||
answered bool
|
||||
terminal bool
|
||||
}{
|
||||
{StatePending, false, false},
|
||||
// Queued is not terminal: the node has not run, and a reader that treats
|
||||
// it as settled stops waiting for the answer it is still going to get.
|
||||
{StateQueued, false, false},
|
||||
{StateRunning, false, false},
|
||||
{StateCompleted, true, true},
|
||||
{StateAdopted, true, true},
|
||||
{StateFailed, false, true},
|
||||
{StateCancelled, false, true},
|
||||
{StateSkipped, false, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.state.Answered(); got != c.answered {
|
||||
t.Errorf("%s.Answered() = %v, want %v", c.state, got, c.answered)
|
||||
}
|
||||
if got := c.state.Terminal(); got != c.terminal {
|
||||
t.Errorf("%s.Terminal() = %v, want %v", c.state, got, c.terminal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A graph crosses a wire before it is drawn, so the folded value has to survive
|
||||
// the round trip that every consumer outside this process reads it through.
|
||||
func TestGraphSurvivesJSONRoundTrip(t *testing.T) {
|
||||
var g Graph
|
||||
g.Apply(Delta{
|
||||
Nodes: []Node{
|
||||
{ID: "grp", Kind: KindGroup, State: StateRunning, Label: "fleet(2)"},
|
||||
{ID: "grp/1", ParentID: "grp", Kind: KindWorker, State: StateCompleted, Model: "m", StartedAt: 1, EndedAt: 2},
|
||||
{ID: "old-ref", Kind: KindExternal, State: StateCompleted},
|
||||
},
|
||||
Edges: []Edge{
|
||||
{From: "grp", To: "grp/1", Kind: Spawn},
|
||||
{From: "old-ref", To: "grp/1", Kind: Adopt},
|
||||
},
|
||||
})
|
||||
raw, err := json.Marshal(g)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var back Graph
|
||||
if err := json.Unmarshal(raw, &back); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(back.Edges, g.Edges) {
|
||||
t.Fatalf("edges = %v, want %v", back.Edges, g.Edges)
|
||||
}
|
||||
if !slices.Equal(back.Nodes, g.Nodes) {
|
||||
t.Fatalf("nodes = %v, want %v", back.Nodes, g.Nodes)
|
||||
}
|
||||
}
|
||||
-220
@@ -1,220 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Official catalog metadata: currency, model, effective dates, documentation
|
||||
// sources, and price fingerprints. User-custom prices always win over catalog.
|
||||
|
||||
// CatalogEntry is one official list price for a model in a billing currency.
|
||||
type CatalogEntry struct {
|
||||
Provider string // deepseek | longcat | mimo | google
|
||||
Model string
|
||||
Currency string // ISO billing currency for this row
|
||||
CacheHit float64
|
||||
Input float64
|
||||
Output float64
|
||||
EffectiveFrom string // YYYY-MM-DD inclusive, empty = unbounded
|
||||
EffectiveTo string // YYYY-MM-DD exclusive, empty = unbounded
|
||||
DocURL string
|
||||
BillingMode string // payg | subscription_equivalent
|
||||
Notes string
|
||||
Fingerprint string // filled by Register
|
||||
// Peak is what the same tokens cost inside Window. Nil when the vendor
|
||||
// bills one rate around the clock, and then the fields above are the rate.
|
||||
Peak *RateCard
|
||||
Window *PeakWindow
|
||||
// CheckedOn is when a person last read DocURL and confirmed these numbers.
|
||||
// No vendor here serves prices over an API, so this date is all that stands
|
||||
// between a price change and a readout that is quietly wrong.
|
||||
CheckedOn string
|
||||
}
|
||||
|
||||
// deepseekPeak is the schedule DeepSeek publishes: peak on weekday mornings and
|
||||
// afternoons, and — since 2026-08-23 — never at a weekend. Chinese public
|
||||
// holidays bill off-peak in full, per the same page.
|
||||
var deepseekPeak = &PeakWindow{
|
||||
OffsetSeconds: beijing,
|
||||
Hours: [][2]int{{9, 12}, {14, 18}},
|
||||
WeekendOffPeak: "2026-08-23",
|
||||
HolidayOffPeak: chineseStatutoryHolidays,
|
||||
}
|
||||
|
||||
// chineseStatutoryHolidays are the State Council's 2026 public holidays, which
|
||||
// DeepSeek bills off-peak in full; extend it when the next notice is out. Make-up
|
||||
// workdays are absent: they fall on weekends, which are off-peak anyway.
|
||||
var chineseStatutoryHolidays = []string{
|
||||
// New Year's Day: Jan 1-3.
|
||||
"2026-01-01", "2026-01-02", "2026-01-03",
|
||||
// Spring Festival: Feb 15-23.
|
||||
"2026-02-15", "2026-02-16", "2026-02-17", "2026-02-18", "2026-02-19",
|
||||
"2026-02-20", "2026-02-21", "2026-02-22", "2026-02-23",
|
||||
// Qingming Festival: Apr 4-6.
|
||||
"2026-04-04", "2026-04-05", "2026-04-06",
|
||||
// Labor Day: May 1-5.
|
||||
"2026-05-01", "2026-05-02", "2026-05-03", "2026-05-04", "2026-05-05",
|
||||
// Dragon Boat Festival: Jun 19-21.
|
||||
"2026-06-19", "2026-06-20", "2026-06-21",
|
||||
// Mid-Autumn Festival: Sep 25-27.
|
||||
"2026-09-25", "2026-09-26", "2026-09-27",
|
||||
// National Day: Oct 1-7.
|
||||
"2026-10-01", "2026-10-02", "2026-10-03", "2026-10-04",
|
||||
"2026-10-05", "2026-10-06", "2026-10-07",
|
||||
}
|
||||
|
||||
// checkedOn is when the tables below were last read off the vendors' pages.
|
||||
const checkedOn = "2026-08-22"
|
||||
|
||||
// deepseekCheckedOn is when the DeepSeek rows were last read off that vendor's
|
||||
// page. Separate from checkedOn, because confirming one vendor's prices says
|
||||
// nothing about another's and a shared date would claim that it did.
|
||||
const deepseekCheckedOn = "2026-09-13"
|
||||
|
||||
// mimoCheckedOn is when the V2.6 rows were read off the vendor's own docs.
|
||||
const mimoCheckedOn = "2026-10-06"
|
||||
|
||||
// googleCheckedOn is when the Gemini rows were read off Google's pricing page.
|
||||
const googleCheckedOn = "2026-10-17"
|
||||
|
||||
// CatalogSourceURLs document public pricing pages.
|
||||
const (
|
||||
DocDeepSeekPricing = "https://api-docs.deepseek.com/quick_start/pricing"
|
||||
DocDeepSeekPricingCNY = "https://api-docs.deepseek.com/zh-cn/quick_start/pricing"
|
||||
DocLongCatPricingUSD = "https://longcat.chat/platform/docs/Pricing/LongCat-2.0.html"
|
||||
DocLongCatPricingCNY = "https://longcat.chat/platform/docs/zh/pricing/long-cat-2.0"
|
||||
DocMiMoPAYG = "https://mimo.mi.com/docs/price/pay-as-you-go"
|
||||
DocMiMoTokenPlan = "https://platform.xiaomimimo.com/token-plan"
|
||||
DocGeminiPricing = "https://ai.google.dev/gemini-api/docs/pricing"
|
||||
)
|
||||
|
||||
// officialEndpointHosts maps a vendor's own API hostnames to the catalog
|
||||
// provider whose list prices apply there. Host equality is the test: a reseller
|
||||
// or proxy bills on its own terms even when it serves the same model, and an
|
||||
// entry merely named after a vendor is not evidence of the vendor's endpoint.
|
||||
var officialEndpointHosts = map[string]string{
|
||||
"api.deepseek.com": "deepseek",
|
||||
"api.longcat.chat": "longcat",
|
||||
"api.xiaomimimo.com": "mimo",
|
||||
"generativelanguage.googleapis.com": "google",
|
||||
"token-plan-cn.xiaomimimo.com": "mimo",
|
||||
"token-plan-sgp.xiaomimimo.com": "mimo",
|
||||
"token-plan-ams.xiaomimimo.com": "mimo",
|
||||
}
|
||||
|
||||
// OfficialProviderForEndpoint returns the catalog provider whose official
|
||||
// prices apply to baseURL, or "" when the address is not a vendor's own API.
|
||||
func OfficialProviderForEndpoint(baseURL string) string {
|
||||
u, err := url.Parse(strings.TrimSpace(baseURL))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return officialEndpointHosts[strings.ToLower(u.Hostname())]
|
||||
}
|
||||
|
||||
// OfficialCatalog is the built-in price book: the current generation of every
|
||||
// rate in officialRates, fingerprinted so a stored price can be recognised as
|
||||
// one of ours. User-custom prices always win over it.
|
||||
func OfficialCatalog() []CatalogEntry {
|
||||
entries := officialCatalogRows()
|
||||
for i := range entries {
|
||||
entries[i].Fingerprint = PricingFingerprint(RateCard{
|
||||
CacheHit: entries[i].CacheHit,
|
||||
Input: entries[i].Input,
|
||||
Output: entries[i].Output,
|
||||
Currency: entries[i].Currency,
|
||||
})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
// LookupCatalog finds an official entry. billingMode empty matches payg first.
|
||||
func LookupCatalog(provider, model, currency, billingMode string) (CatalogEntry, bool) {
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
model = strings.TrimSpace(model)
|
||||
currency = NormalizeCurrency(currency)
|
||||
billingMode = strings.TrimSpace(billingMode)
|
||||
if billingMode == "" {
|
||||
billingMode = BillingModePAYG
|
||||
}
|
||||
var fallback CatalogEntry
|
||||
foundFallback := false
|
||||
for _, e := range OfficialCatalog() {
|
||||
if e.Provider != provider || e.Model != model {
|
||||
continue
|
||||
}
|
||||
if NormalizeCurrency(e.Currency) != currency {
|
||||
continue
|
||||
}
|
||||
if e.BillingMode == billingMode {
|
||||
return e, true
|
||||
}
|
||||
if e.BillingMode == BillingModePAYG {
|
||||
fallback = e
|
||||
foundFallback = true
|
||||
}
|
||||
}
|
||||
if foundFallback {
|
||||
return fallback, true
|
||||
}
|
||||
return CatalogEntry{}, false
|
||||
}
|
||||
|
||||
// RateCardFromCatalog builds a RateCard from a catalog entry's base rate.
|
||||
func RateCardFromCatalog(e CatalogEntry) RateCard {
|
||||
return RateCard{
|
||||
CacheHit: e.CacheHit,
|
||||
Input: e.Input,
|
||||
Output: e.Output,
|
||||
Currency: e.Currency,
|
||||
}
|
||||
}
|
||||
|
||||
// RatesAt is what this entry charges at t: the peak card inside its window, the
|
||||
// base card everywhere else. An entry without a window charges one rate, and
|
||||
// then t does not matter.
|
||||
func (e CatalogEntry) RatesAt(t time.Time) RateCard {
|
||||
if e.Peak == nil || !e.Window.IsPeak(t) {
|
||||
return RateCardFromCatalog(e)
|
||||
}
|
||||
peak := *e.Peak
|
||||
peak.Currency = e.Currency
|
||||
return peak
|
||||
}
|
||||
|
||||
// InEffect reports whether this entry's dates cover the day t falls on.
|
||||
func (e CatalogEntry) InEffect(t time.Time) bool {
|
||||
day := t.UTC().Format("2006-01-02")
|
||||
if e.EffectiveFrom != "" && day < e.EffectiveFrom {
|
||||
return false
|
||||
}
|
||||
return e.EffectiveTo == "" || day < e.EffectiveTo
|
||||
}
|
||||
|
||||
// MatchesCatalog reports whether rates equal a known official entry.
|
||||
func MatchesCatalog(provider, model string, rates RateCard) (CatalogEntry, bool) {
|
||||
cur := NormalizeCurrency(rates.Currency)
|
||||
for _, e := range OfficialCatalog() {
|
||||
if e.Provider != strings.ToLower(strings.TrimSpace(provider)) {
|
||||
continue
|
||||
}
|
||||
if e.Model != strings.TrimSpace(model) {
|
||||
continue
|
||||
}
|
||||
if NormalizeCurrency(e.Currency) != cur {
|
||||
continue
|
||||
}
|
||||
// Either card identifies the vendor's own price. Which one a user
|
||||
// happens to have configured says nothing about when they are billing:
|
||||
// the hour decides that, and RatesAt is what reads it.
|
||||
if e.CacheHit == rates.CacheHit && e.Input == rates.Input && e.Output == rates.Output {
|
||||
return e, true
|
||||
}
|
||||
if p := e.Peak; p != nil && p.CacheHit == rates.CacheHit && p.Input == rates.Input && p.Output == rates.Output {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return CatalogEntry{}, false
|
||||
}
|
||||
-95
@@ -1,95 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Every catalog provider must declare at least one official endpoint, or its
|
||||
// rows become unreachable: a rate card is only matched against the catalog
|
||||
// after the entry's host identifies the vendor.
|
||||
func TestEveryCatalogProviderHasAnOfficialEndpoint(t *testing.T) {
|
||||
declared := map[string]bool{}
|
||||
for _, host := range officialEndpointHosts {
|
||||
declared[host] = true
|
||||
}
|
||||
for _, entry := range OfficialCatalog() {
|
||||
if !declared[entry.Provider] {
|
||||
t.Errorf("catalog provider %q has no official endpoint host; its prices can never match", entry.Provider)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfficialProviderForEndpoint(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
baseURL string
|
||||
want string
|
||||
}{
|
||||
{"official deepseek", "https://api.deepseek.com", "deepseek"},
|
||||
{"official deepseek anthropic path", "https://api.deepseek.com/anthropic", "deepseek"},
|
||||
{"official longcat", "https://api.longcat.chat/openai/v1", "longcat"},
|
||||
{"official mimo region", "https://token-plan-sgp.xiaomimimo.com/v1", "mimo"},
|
||||
{"uppercase host", "https://API.DeepSeek.com/v1", "deepseek"},
|
||||
{"proxy that resells the model", "https://my-deepseek-proxy.internal/v1", ""},
|
||||
{"vendor name in the path only", "https://gateway.example.com/deepseek/v1", ""},
|
||||
{"lookalike registrable domain", "https://api.deepseek.com.evil.test/v1", ""},
|
||||
{"empty", "", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := OfficialProviderForEndpoint(tc.baseURL); got != tc.want {
|
||||
t.Fatalf("OfficialProviderForEndpoint(%q) = %q, want %q", tc.baseURL, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// pricesGoStaleAfter is how long a hand-copied rate is trusted before someone
|
||||
// reads the vendor's page again. No vendor here serves prices over an API, so
|
||||
// the alternative to this going red on a quiet day is learning about a change
|
||||
// from a user whose cost was wrong for months. Bumping CheckedOn without
|
||||
// opening DocURL defeats it; what this buys is that the question gets asked.
|
||||
const pricesGoStaleAfter = 90 * 24 * time.Hour
|
||||
|
||||
func TestCatalogPricesHaveBeenCheckedRecently(t *testing.T) {
|
||||
var undated, stale []string
|
||||
// This snapshot never sees a vendor page again, so staleness is read at the
|
||||
// day it was taken rather than the day the suite runs.
|
||||
cutoff := time.Date(2026, 10, 19, 0, 0, 0, 0, time.UTC).Add(-pricesGoStaleAfter)
|
||||
for _, e := range OfficialCatalog() {
|
||||
where := e.Provider + "/" + e.Model + " " + e.Currency + " (" + e.DocURL + ")"
|
||||
if e.CheckedOn == "" {
|
||||
undated = append(undated, where)
|
||||
continue
|
||||
}
|
||||
on, err := time.Parse("2006-01-02", e.CheckedOn)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: CheckedOn %q is not YYYY-MM-DD", where, e.CheckedOn)
|
||||
}
|
||||
if on.Before(cutoff) {
|
||||
stale = append(stale, where+" last checked "+e.CheckedOn)
|
||||
}
|
||||
}
|
||||
if len(undated) > 0 {
|
||||
t.Errorf("catalog rows with no CheckedOn — read the vendor's page and date them:\n %s",
|
||||
strings.Join(undated, "\n "))
|
||||
}
|
||||
if len(stale) > 0 {
|
||||
t.Errorf("catalog rows older than %d days. Open each DocURL, confirm or correct the rate, then set CheckedOn to today:\n %s",
|
||||
int(pricesGoStaleAfter.Hours()/24), strings.Join(stale, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// A row that names a peak card must say when it applies, and one that names a
|
||||
// window must have a card to charge inside it. Half of the pair silently prices
|
||||
// every hour the same, which is the bug the pair exists to prevent.
|
||||
func TestPeakRatesAndWindowsComeInPairs(t *testing.T) {
|
||||
for _, e := range OfficialCatalog() {
|
||||
where := e.Provider + "/" + e.Model + " " + e.Currency
|
||||
if (e.Peak == nil) != (e.Window == nil) {
|
||||
t.Errorf("%s: Peak and Window must be set together (peak=%v window=%v)", where, e.Peak != nil, e.Window != nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
-48
@@ -1,48 +0,0 @@
|
||||
package pricing
|
||||
|
||||
// Coverage says how much of the usage behind a quote carries a complete price
|
||||
// fact. CostComplete answers that for one call; an aggregate has two answers
|
||||
// more, because it can price some of its calls and not others, and it can have
|
||||
// billed nothing at all. A session that has not spent yet is not a session
|
||||
// whose price is unknown.
|
||||
const (
|
||||
CoverageNone = "none"
|
||||
CoverageComplete = "complete"
|
||||
CoveragePartial = "partial"
|
||||
CoverageIncomplete = "incomplete"
|
||||
)
|
||||
|
||||
// ValidCoverage reports whether c is one of the four identities.
|
||||
func ValidCoverage(c string) bool {
|
||||
switch c {
|
||||
case CoverageNone, CoverageComplete, CoveragePartial, CoverageIncomplete:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FoldCoverage returns the coverage of a and b together: the join over complete
|
||||
// and incomplete, with none as the identity and partial as the top. Order cannot
|
||||
// reach the answer, and a run that has priced everything so far cannot fold back
|
||||
// to complete once an unpriced call joins it. A value outside the four folds as
|
||||
// none; NormalizeQuote repairs a decoded quote before it can reach here.
|
||||
func FoldCoverage(a, b string) string {
|
||||
a, b = coverageOrNone(a), coverageOrNone(b)
|
||||
switch {
|
||||
case a == CoverageNone:
|
||||
return b
|
||||
case b == CoverageNone:
|
||||
return a
|
||||
case a == b:
|
||||
return a
|
||||
default:
|
||||
return CoveragePartial
|
||||
}
|
||||
}
|
||||
|
||||
func coverageOrNone(c string) string {
|
||||
if ValidCoverage(c) {
|
||||
return c
|
||||
}
|
||||
return CoverageNone
|
||||
}
|
||||
-190
@@ -1,190 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var allCoverages = []string{CoverageNone, CoverageComplete, CoveragePartial, CoverageIncomplete}
|
||||
|
||||
// The contract the aggregate has to satisfy, stated as the pairs themselves.
|
||||
func TestFoldCoverageContract(t *testing.T) {
|
||||
cases := []struct{ a, b, want string }{
|
||||
{CoverageNone, CoverageNone, CoverageNone},
|
||||
{CoverageComplete, CoverageNone, CoverageComplete},
|
||||
{CoverageComplete, CoverageComplete, CoverageComplete},
|
||||
{CoverageComplete, CoverageIncomplete, CoveragePartial},
|
||||
{CoverageIncomplete, CoverageComplete, CoveragePartial},
|
||||
{CoverageIncomplete, CoverageIncomplete, CoverageIncomplete},
|
||||
{CoveragePartial, CoverageComplete, CoveragePartial},
|
||||
{CoveragePartial, CoverageIncomplete, CoveragePartial},
|
||||
{CoveragePartial, CoverageNone, CoveragePartial},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := FoldCoverage(c.a, c.b); got != c.want {
|
||||
t.Errorf("fold(%s, %s) = %s, want %s", c.a, c.b, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Arrival order is not a fact about the session, so it may not reach the
|
||||
// answer. Exhaustive rather than sampled: there are only sixty-four triples.
|
||||
func TestFoldCoverageIsOrderIndependent(t *testing.T) {
|
||||
for _, a := range allCoverages {
|
||||
for _, b := range allCoverages {
|
||||
if FoldCoverage(a, b) != FoldCoverage(b, a) {
|
||||
t.Errorf("fold(%s, %s) != fold(%s, %s)", a, b, b, a)
|
||||
}
|
||||
if FoldCoverage(a, a) != a {
|
||||
t.Errorf("fold(%s, %s) = %s, want %s", a, a, FoldCoverage(a, a), a)
|
||||
}
|
||||
for _, c := range allCoverages {
|
||||
left := FoldCoverage(FoldCoverage(a, b), c)
|
||||
right := FoldCoverage(a, FoldCoverage(b, c))
|
||||
if left != right {
|
||||
t.Errorf("fold is not associative on (%s, %s, %s): %s vs %s", a, b, c, left, right)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The failure a boolean invites: the last quote deciding for the ones before it.
|
||||
func TestCompleteNeverSurvivesALaterIncomplete(t *testing.T) {
|
||||
for _, a := range allCoverages {
|
||||
got := FoldCoverage(FoldCoverage(a, CoverageComplete), CoverageIncomplete)
|
||||
if got == CoverageComplete {
|
||||
t.Errorf("fold(%s, complete, incomplete) = complete", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFoldCoverageTreatsAnUnnamedStateAsNothing(t *testing.T) {
|
||||
if got := FoldCoverage("", CoverageComplete); got != CoverageComplete {
|
||||
t.Fatalf("fold(\"\", complete) = %s", got)
|
||||
}
|
||||
if got := FoldCoverage("mostly", CoverageIncomplete); got != CoverageIncomplete {
|
||||
t.Fatalf("fold(unnamed, incomplete) = %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func pricedQuote(t *testing.T, perMillion float64) CostQuote {
|
||||
t.Helper()
|
||||
q := BuildQuote(QuoteInput{
|
||||
Usage: UsageTokens{PromptTokens: 1000},
|
||||
Rates: RateCard{Input: perMillion, Currency: "USD"},
|
||||
})
|
||||
if q.Coverage != CoverageComplete {
|
||||
t.Fatalf("a priced call is not complete: %+v", q)
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
func unpricedQuote(t *testing.T) CostQuote {
|
||||
t.Helper()
|
||||
q := NormalizeQuote(CostQuote{
|
||||
Estimated: true, DisplayStatus: DisplayStatusUnavailable, IncompleteReason: "no_price",
|
||||
})
|
||||
if q.Coverage != CoverageIncomplete {
|
||||
t.Fatalf("an unpriced call is not incomplete: %+v", q)
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
func TestAggregateCoverageAcrossCalls(t *testing.T) {
|
||||
priced, unpriced := pricedQuote(t, 1), unpricedQuote(t)
|
||||
cases := []struct {
|
||||
name string
|
||||
quotes []CostQuote
|
||||
want string
|
||||
}{
|
||||
{"no usage", nil, CoverageNone},
|
||||
{"one priced", []CostQuote{priced}, CoverageComplete},
|
||||
{"two priced", []CostQuote{priced, priced}, CoverageComplete},
|
||||
{"priced then unpriced", []CostQuote{priced, unpriced}, CoveragePartial},
|
||||
{"unpriced then priced", []CostQuote{unpriced, priced}, CoveragePartial},
|
||||
{"unpriced only", []CostQuote{unpriced}, CoverageIncomplete},
|
||||
{"unpriced twice", []CostQuote{unpriced, unpriced}, CoverageIncomplete},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := AggregateQuotes(c.quotes, "USD")
|
||||
if got.Coverage != c.want {
|
||||
t.Errorf("%s: coverage = %s, want %s", c.name, got.Coverage, c.want)
|
||||
}
|
||||
// The boolean is a reading of the coverage, not a second answer.
|
||||
if got.CostComplete != (c.want == CoverageComplete) {
|
||||
t.Errorf("%s: costComplete = %v against coverage %s", c.name, got.CostComplete, got.Coverage)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A session's completeness is a fact about its calls, so the order they were
|
||||
// billed in may not change it — at the aggregate, not only in the fold.
|
||||
func TestAggregateCoverageIgnoresArrivalOrder(t *testing.T) {
|
||||
priced, unpriced := pricedQuote(t, 1), unpricedQuote(t)
|
||||
forward := AggregateQuotes([]CostQuote{priced, priced, unpriced}, "USD")
|
||||
backward := AggregateQuotes([]CostQuote{unpriced, priced, priced}, "USD")
|
||||
if forward.Coverage != backward.Coverage || forward.Coverage != CoveragePartial {
|
||||
t.Fatalf("order changed coverage: %s vs %s", forward.Coverage, backward.Coverage)
|
||||
}
|
||||
}
|
||||
|
||||
// Zero is an amount, not a completeness state. Both sessions total zero; only
|
||||
// one of them has been billed for anything.
|
||||
func TestKnownZeroIsCompleteAndNoUsageIsNot(t *testing.T) {
|
||||
free := AggregateQuotes([]CostQuote{pricedQuote(t, 0)}, "USD")
|
||||
if free.Coverage != CoverageComplete || !free.CostComplete {
|
||||
t.Fatalf("a free call priced at zero is not complete: %+v", free)
|
||||
}
|
||||
if amount := free.Original.AmountValue().Float64(); amount != 0 {
|
||||
t.Fatalf("a free call cost %v", amount)
|
||||
}
|
||||
empty := AggregateQuotes(nil, "USD")
|
||||
if empty.Coverage != CoverageNone {
|
||||
t.Fatalf("no usage = %s", empty.Coverage)
|
||||
}
|
||||
if amount := empty.Original.AmountValue().Float64(); amount != 0 {
|
||||
t.Fatalf("an empty session cost %v", amount)
|
||||
}
|
||||
if free.Coverage == empty.Coverage {
|
||||
t.Fatal("a billed zero and an unbilled session share one identity")
|
||||
}
|
||||
}
|
||||
|
||||
// A quote persisted before coverage existed still has to name one, and the
|
||||
// name has to be the one its own cost fact supports.
|
||||
func TestNormalizeFillsCoverageForAQuoteThatPredatesIt(t *testing.T) {
|
||||
old := NormalizeQuote(CostQuote{Original: MoneyOf(NewAmountFromFloat(1), "USD"), Complete: true})
|
||||
if old.Coverage != CoverageComplete {
|
||||
t.Fatalf("an old complete quote = %s", old.Coverage)
|
||||
}
|
||||
if got := NormalizeQuote(CostQuote{Coverage: "mostly", IncompleteReason: "no_price"}); got.Coverage != CoverageIncomplete {
|
||||
t.Fatalf("an unnamed coverage survived normalization: %s", got.Coverage)
|
||||
}
|
||||
kept := NormalizeQuote(CostQuote{Coverage: CoveragePartial, IncompleteReason: "no_price"})
|
||||
if kept.Coverage != CoveragePartial {
|
||||
t.Fatalf("normalization overwrote a coverage the quote already had: %s", kept.Coverage)
|
||||
}
|
||||
}
|
||||
|
||||
// The ledger merges by pricing key before anything aggregates, so it is a
|
||||
// second fold. An entry that has seen an unpriced call may not read complete
|
||||
// again because a later call arrived in another currency.
|
||||
func TestLedgerEntryKeepsIncompletenessAcrossBucketing(t *testing.T) {
|
||||
l := NewLedger()
|
||||
unpriced := unpricedQuote(t)
|
||||
unpriced.ModelRef, unpriced.UsageSource = "m", "executor"
|
||||
cny := BuildQuote(QuoteInput{Usage: UsageTokens{PromptTokens: 1000}, Rates: RateCard{Input: 1, Currency: "CNY"}, ModelRef: "m", UsageSource: "executor"})
|
||||
usd := BuildQuote(QuoteInput{Usage: UsageTokens{PromptTokens: 1000}, Rates: RateCard{Input: 1, Currency: "USD"}, ModelRef: "m", UsageSource: "executor"})
|
||||
usd.PricingFingerprint, cny.PricingFingerprint = unpriced.PricingFingerprint, unpriced.PricingFingerprint
|
||||
for _, q := range []CostQuote{unpriced, cny, usd} {
|
||||
l.Add(q, UsageTokens{PromptTokens: 1000}, time.Time{})
|
||||
}
|
||||
total := l.Total("USD")
|
||||
if total.Coverage != CoveragePartial {
|
||||
t.Fatalf("ledger total coverage = %s, want partial", total.Coverage)
|
||||
}
|
||||
if total.CostComplete {
|
||||
t.Fatal("an entry that saw an unpriced call reads as complete")
|
||||
}
|
||||
}
|
||||
-68
@@ -1,68 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import "testing"
|
||||
|
||||
// https://ai.google.dev/gemini-api/docs/pricing (read 2026-10-17): per 1M tokens,
|
||||
// paid tier. gemini-3.8-flash is 0.75 in / 3.75 out / 0.075 cached through
|
||||
// 2026-12-31 and 1.50 / 7.50 / 0.15 from 2027-01-01.
|
||||
func TestAGeminiRateStepIsDatedNotEdited(t *testing.T) {
|
||||
was := today
|
||||
defer func() { today = was }()
|
||||
for _, tc := range []struct {
|
||||
day string
|
||||
cache, input, output float64
|
||||
}{
|
||||
{"2026-10-17", 0.075, 0.75, 3.75},
|
||||
{"2026-12-31", 0.075, 0.75, 3.75},
|
||||
{"2027-01-01", 0.15, 1.50, 7.50},
|
||||
{"2028-03-01", 0.15, 1.50, 7.50},
|
||||
} {
|
||||
today = func() string { return tc.day }
|
||||
got := CurrentRate("google", "gemini-3.8-flash", "USD")
|
||||
if got == nil || got.CacheHit != tc.cache || got.Input != tc.input || got.Output != tc.output {
|
||||
t.Errorf("%s: rate = %+v, want %v/%v/%v", tc.day, got, tc.cache, tc.input, tc.output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRateInForceIsNotListedAsSuperseded(t *testing.T) {
|
||||
was := today
|
||||
defer func() { today = was }()
|
||||
today = func() string { return "2026-10-17" }
|
||||
if sup := SupersededRates("google", "gemini-3.8-flash", "USD"); len(sup) != 0 {
|
||||
t.Fatalf("before the step nothing is superseded, got %+v", sup)
|
||||
}
|
||||
today = func() string { return "2027-02-01" }
|
||||
sup := SupersededRates("google", "gemini-3.8-flash", "USD")
|
||||
if len(sup) != 1 || sup[0].Input != 0.75 {
|
||||
t.Fatalf("after the step the intro rate is superseded, got %+v", sup)
|
||||
}
|
||||
if !IsOfficialCard("google", "gemini-3.8-flash", RateCard{CacheHit: 0.075, Input: 0.75, Output: 3.75, Currency: "USD"}) {
|
||||
t.Fatal("a stored intro card must still read as ours after the step")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoogleModelsAreBilledAtGoogleOnlyOnGoogleHosts(t *testing.T) {
|
||||
if OfficialProviderForEndpoint("https://generativelanguage.googleapis.com/v1beta") != "google" {
|
||||
t.Fatal("Google's own host does not read as Google")
|
||||
}
|
||||
if OfficialProviderForEndpoint("https://relay.example.com/v1beta") != "" {
|
||||
t.Fatal("a relay was billed at Google's list price")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryGoogleRowNamesItsSource(t *testing.T) {
|
||||
n := 0
|
||||
for _, e := range OfficialCatalog() {
|
||||
if e.Provider != "google" {
|
||||
continue
|
||||
}
|
||||
n++
|
||||
if e.DocURL != DocGeminiPricing || e.CheckedOn != googleCheckedOn || e.Input <= 0 || e.Output <= e.Input {
|
||||
t.Errorf("row %+v", e)
|
||||
}
|
||||
}
|
||||
if n != 4 {
|
||||
t.Fatalf("%d Google rows, want 4 (gemini-4-argon shows no price)", n)
|
||||
}
|
||||
}
|
||||
-238
@@ -1,238 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LedgerEntry is one occurrence-time cost fact. Ledger keys include model,
|
||||
// usage source, pricing fingerprint, and legacy rate date so model switches and
|
||||
// sub-agents never collapse into a single scalar.
|
||||
type LedgerEntry struct {
|
||||
Key string `json:"key"`
|
||||
ModelRef string `json:"modelRef"`
|
||||
UsageSource string `json:"usageSource"`
|
||||
PricingFingerprint string `json:"pricingFingerprint"`
|
||||
RateDate string `json:"rateDate,omitempty"`
|
||||
OccurredAt time.Time `json:"occurredAt"`
|
||||
Quote CostQuote `json:"quote"`
|
||||
// Token totals for this bucket (summed).
|
||||
PromptTokens int `json:"promptTokens"`
|
||||
CompletionTokens int `json:"completionTokens"`
|
||||
TotalTokens int `json:"totalTokens"`
|
||||
CacheHitTokens int `json:"cacheHitTokens"`
|
||||
CacheMissTokens int `json:"cacheMissTokens"`
|
||||
// CacheUnknownTokens is prompt the endpoint reported without saying how much
|
||||
// was cached. It is counted as a miss in CacheMissTokens for pricing and kept
|
||||
// here so a hit rate can leave it out instead of being pulled down by it.
|
||||
CacheUnknownTokens int `json:"cacheUnknownTokens,omitempty"`
|
||||
RequestCount int `json:"requestCount"`
|
||||
}
|
||||
|
||||
// LedgerKey builds the stable aggregation key.
|
||||
func LedgerKey(modelRef, usageSource, pricingFingerprint, rateDate string) string {
|
||||
return strings.Join([]string{
|
||||
strings.TrimSpace(modelRef),
|
||||
strings.TrimSpace(usageSource),
|
||||
strings.TrimSpace(pricingFingerprint),
|
||||
strings.TrimSpace(rateDate),
|
||||
}, "|")
|
||||
}
|
||||
|
||||
// Ledger accumulates occurrence-time quotes. Switching display currency only
|
||||
// re-selects valuations already stored; it does not reprice or clear history.
|
||||
type Ledger struct {
|
||||
Version int `json:"version"`
|
||||
Entries map[string]LedgerEntry `json:"entries"`
|
||||
}
|
||||
|
||||
// LedgerVersion is the current persisted ledger schema.
|
||||
const LedgerVersion = 1
|
||||
|
||||
// NewLedger returns an empty ledger.
|
||||
func NewLedger() *Ledger {
|
||||
return &Ledger{Version: LedgerVersion, Entries: map[string]LedgerEntry{}}
|
||||
}
|
||||
|
||||
// Add merges a quote into the ledger under its natural key.
|
||||
func (l *Ledger) Add(q CostQuote, tokens UsageTokens, occurred time.Time) {
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
if l.Entries == nil {
|
||||
l.Entries = map[string]LedgerEntry{}
|
||||
}
|
||||
if l.Version == 0 {
|
||||
l.Version = LedgerVersion
|
||||
}
|
||||
if occurred.IsZero() {
|
||||
occurred = time.Now().UTC()
|
||||
}
|
||||
q = NormalizeQuote(q)
|
||||
source := q.UsageSource
|
||||
if source == "" {
|
||||
source = "executor"
|
||||
}
|
||||
key := LedgerKey(q.ModelRef, source, q.PricingFingerprint, q.RateDate)
|
||||
ent, ok := l.Entries[key]
|
||||
if !ok {
|
||||
ent = LedgerEntry{
|
||||
Key: key,
|
||||
ModelRef: q.ModelRef,
|
||||
UsageSource: source,
|
||||
PricingFingerprint: q.PricingFingerprint,
|
||||
RateDate: q.RateDate,
|
||||
OccurredAt: occurred,
|
||||
Quote: q,
|
||||
}
|
||||
// Fresh quote valuations are kept; we re-aggregate Original via sums.
|
||||
ent.Quote.Valuations = cloneValuations(q.Valuations)
|
||||
} else {
|
||||
// Sum original when same currency; otherwise retain deterministic
|
||||
// per-currency buckets. Once a bucketed entry exists, keep adding into
|
||||
// those buckets so later same-currency calls are not lost.
|
||||
if len(ent.Quote.OriginalTotals) > 0 {
|
||||
ent.Quote.OriginalTotals = mergeOriginalTotals(ent.Quote.OriginalTotals, q)
|
||||
ent.Quote.DisplayComplete = false
|
||||
ent.Quote.Complete = false
|
||||
ent.Quote.DisplayStatus = DisplayStatusBucketed
|
||||
ent.Quote.AggregateMode = AggregateModeCurrencyBuckets
|
||||
ent.Quote.IncompleteReason = "mixed_original_currencies"
|
||||
} else {
|
||||
sum, err := AddMoney(ent.Quote.Original, q.Original)
|
||||
if err != nil {
|
||||
ent.Quote.OriginalTotals = mergeOriginalTotals([]Money{ent.Quote.Original}, q)
|
||||
ent.Quote.DisplayComplete = false
|
||||
ent.Quote.Complete = false
|
||||
ent.Quote.DisplayStatus = DisplayStatusBucketed
|
||||
ent.Quote.AggregateMode = AggregateModeCurrencyBuckets
|
||||
ent.Quote.IncompleteReason = "mixed_original_currencies"
|
||||
} else {
|
||||
ent.Quote.Original = sum
|
||||
}
|
||||
}
|
||||
for code, v := range q.Valuations {
|
||||
code = NormalizeCurrency(code)
|
||||
if prev, ok := ent.Quote.Valuations[code]; ok {
|
||||
added, err := AddMoney(prev.Money, v.Money)
|
||||
if err == nil {
|
||||
prev.Money = added
|
||||
if v.Stale {
|
||||
prev.Stale = true
|
||||
}
|
||||
ent.Quote.Valuations[code] = prev
|
||||
}
|
||||
} else {
|
||||
if ent.Quote.Valuations == nil {
|
||||
ent.Quote.Valuations = map[string]Valuation{}
|
||||
}
|
||||
ent.Quote.Valuations[code] = v
|
||||
}
|
||||
}
|
||||
if q.Estimated {
|
||||
ent.Quote.Estimated = true
|
||||
}
|
||||
if !q.Complete {
|
||||
ent.Quote.DisplayComplete = false
|
||||
ent.Quote.Complete = false
|
||||
if ent.Quote.IncompleteReason == "" {
|
||||
ent.Quote.IncompleteReason = q.IncompleteReason
|
||||
}
|
||||
}
|
||||
ent.Quote.Coverage = FoldCoverage(ent.Quote.Coverage, q.Coverage)
|
||||
ent.Quote.CostComplete = ent.Quote.Coverage == CoverageComplete
|
||||
if ent.Quote.DisplayStatus != DisplayStatusBucketed {
|
||||
ent.Quote.DisplayStatus = q.DisplayStatus
|
||||
}
|
||||
}
|
||||
ent.PromptTokens += tokens.PromptTokens
|
||||
ent.CompletionTokens += tokens.CompletionTokens
|
||||
ent.TotalTokens += tokens.PromptTokens + tokens.CompletionTokens
|
||||
if tokens.CacheHitTokens+tokens.CacheMissTokens > 0 {
|
||||
ent.CacheHitTokens += tokens.CacheHitTokens
|
||||
ent.CacheMissTokens += tokens.CacheMissTokens
|
||||
} else {
|
||||
ent.CacheMissTokens += tokens.PromptTokens
|
||||
ent.CacheUnknownTokens += tokens.PromptTokens
|
||||
}
|
||||
ent.RequestCount++
|
||||
if occurred.After(ent.OccurredAt) {
|
||||
ent.OccurredAt = occurred
|
||||
}
|
||||
l.Entries[key] = ent
|
||||
}
|
||||
|
||||
func mergeOriginalTotals(existing []Money, q CostQuote) []Money {
|
||||
amounts := map[string]Amount{}
|
||||
add := func(m Money) {
|
||||
currency := NormalizeCurrency(m.Currency)
|
||||
if currency == "" {
|
||||
return
|
||||
}
|
||||
amounts[currency] = amounts[currency].Add(m.AmountValue())
|
||||
}
|
||||
for _, m := range existing {
|
||||
add(m)
|
||||
}
|
||||
if len(q.OriginalTotals) > 0 {
|
||||
for _, m := range q.OriginalTotals {
|
||||
add(m)
|
||||
}
|
||||
} else {
|
||||
add(q.Original)
|
||||
}
|
||||
codes := slices.Sorted(maps.Keys(amounts))
|
||||
out := make([]Money, 0, len(codes))
|
||||
for _, code := range codes {
|
||||
out = append(out, MoneyOf(amounts[code], code))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneValuations(in map[string]Valuation) map[string]Valuation {
|
||||
if len(in) == 0 {
|
||||
return map[string]Valuation{}
|
||||
}
|
||||
out := make(map[string]Valuation, len(in))
|
||||
maps.Copy(out, in)
|
||||
return out
|
||||
}
|
||||
|
||||
// Total returns the aggregate CostQuote for a display currency.
|
||||
func (l *Ledger) Total(display string) CostQuote {
|
||||
if l == nil || len(l.Entries) == 0 {
|
||||
return AggregateQuotes(nil, display)
|
||||
}
|
||||
quotes := make([]CostQuote, 0, len(l.Entries))
|
||||
for _, ent := range l.Entries {
|
||||
q := ent.Quote
|
||||
// Refresh valuation moneys already summed in entries.
|
||||
quotes = append(quotes, q)
|
||||
}
|
||||
// Stable order for determinism.
|
||||
sort.Slice(quotes, func(i, j int) bool {
|
||||
return quotes[i].ModelRef+quotes[i].UsageSource < quotes[j].ModelRef+quotes[j].UsageSource
|
||||
})
|
||||
return AggregateQuotes(quotes, display)
|
||||
}
|
||||
|
||||
// SelectDisplay rebinds Selected on every entry and the total without
|
||||
// recomputing occurrence-time pricing facts.
|
||||
func (l *Ledger) SelectDisplay(display string) CostQuote {
|
||||
return l.Total(display)
|
||||
}
|
||||
|
||||
// EntriesBySource groups ledger entries by usage source for status panels.
|
||||
func (l *Ledger) EntriesBySource() map[string][]LedgerEntry {
|
||||
out := map[string][]LedgerEntry{}
|
||||
if l == nil {
|
||||
return out
|
||||
}
|
||||
for _, ent := range l.Entries {
|
||||
out[ent.UsageSource] = append(out[ent.UsageSource], ent)
|
||||
}
|
||||
return out
|
||||
}
|
||||
-88
@@ -1,88 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LegacyUsageRecord is the minimum shape needed to migrate old telemetry.
|
||||
type LegacyUsageRecord struct {
|
||||
SessionCost float64
|
||||
SessionCurrency string
|
||||
// EndedAt is retained for historical ordering and identity valuation dates.
|
||||
EndedAt time.Time
|
||||
// Tokens optional for ledger completeness.
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
ModelRef string
|
||||
UsageSource string
|
||||
}
|
||||
|
||||
// MigrateLegacyUsage builds a CostQuote from pre-CostQuote telemetry.
|
||||
// Records that cannot be valued are marked legacy_estimate; wiped mixed-currency
|
||||
// zeros are not reconstructed from current price tables.
|
||||
func MigrateLegacyUsage(rec LegacyUsageRecord) CostQuote {
|
||||
cur := NormalizeCurrency(rec.SessionCurrency)
|
||||
if cur == "" {
|
||||
// Historic defaults often used ¥ symbol without code.
|
||||
if strings.Contains(rec.SessionCurrency, "¥") || strings.Contains(rec.SessionCurrency, "￥") {
|
||||
cur = "CNY"
|
||||
} else if strings.Contains(rec.SessionCurrency, "$") {
|
||||
cur = "USD"
|
||||
}
|
||||
}
|
||||
// Wiped mixed-currency totals (legacy code zeroed them) and empty rows must
|
||||
// not be reconstructed from current price tables.
|
||||
if rec.SessionCost <= 0 || cur == "" {
|
||||
reason := "legacy_unrecoverable"
|
||||
if rec.SessionCost == 0 && strings.TrimSpace(rec.SessionCurrency) != "" {
|
||||
// Explicit zero with a currency often means a prior mixed-currency wipe.
|
||||
reason = "legacy_wiped_or_zero"
|
||||
} else if rec.SessionCost < 0 {
|
||||
reason = "legacy_invalid_amount"
|
||||
}
|
||||
return CostQuote{
|
||||
Original: MoneyOf(Zero, cur),
|
||||
Estimated: true,
|
||||
CostComplete: false,
|
||||
Coverage: CoverageIncomplete,
|
||||
DisplayComplete: false,
|
||||
Complete: false,
|
||||
DisplayStatus: DisplayStatusUnavailable,
|
||||
LegacyEstimate: true,
|
||||
IncompleteReason: reason,
|
||||
ModelRef: rec.ModelRef,
|
||||
UsageSource: rec.UsageSource,
|
||||
}
|
||||
}
|
||||
amount := NewAmountFromFloat(rec.SessionCost)
|
||||
q := CostQuote{
|
||||
Original: MoneyOf(amount, cur),
|
||||
Valuations: map[string]Valuation{},
|
||||
Estimated: true,
|
||||
CostComplete: true,
|
||||
Coverage: CoverageComplete,
|
||||
DisplayComplete: true,
|
||||
Complete: true,
|
||||
DisplayStatus: DisplayStatusMatched,
|
||||
AggregateMode: AggregateModeSingleCurrency,
|
||||
LegacyEstimate: true,
|
||||
ModelRef: rec.ModelRef,
|
||||
UsageSource: rec.UsageSource,
|
||||
BillingMode: BillingModePAYG,
|
||||
}
|
||||
ended := rec.EndedAt
|
||||
if ended.IsZero() {
|
||||
ended = time.Now().UTC()
|
||||
}
|
||||
asOf := ended.UTC().Format("2006-01-02")
|
||||
q.Valuations[cur] = Valuation{
|
||||
Money: q.Original,
|
||||
Basis: BasisIdentity,
|
||||
Source: "legacy_telemetry",
|
||||
AsOf: asOf,
|
||||
}
|
||||
m := q.Original
|
||||
q.Selected = &m
|
||||
return q
|
||||
}
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
package pricing
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMigrateLegacyUsageIdentityOnly(t *testing.T) {
|
||||
q := MigrateLegacyUsage(LegacyUsageRecord{SessionCost: 1.25, SessionCurrency: "USD"})
|
||||
if !q.CostComplete || !q.DisplayComplete || !q.Complete || q.Selected == nil {
|
||||
t.Fatalf("migrated quote = %+v", q)
|
||||
}
|
||||
if len(q.Valuations) != 1 || q.Valuations["USD"].Basis != BasisIdentity {
|
||||
t.Fatalf("migrated valuations = %+v", q.Valuations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateLegacyWipedZeroDoesNotGuess(t *testing.T) {
|
||||
q := MigrateLegacyUsage(LegacyUsageRecord{SessionCurrency: "CNY"})
|
||||
if q.CostComplete || q.Complete || q.DisplayStatus != DisplayStatusUnavailable || q.IncompleteReason != "legacy_wiped_or_zero" {
|
||||
t.Fatalf("wiped quote = %+v", q)
|
||||
}
|
||||
}
|
||||
Loaded 100 of 3719 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user