Compare commits

..
2372 changed files with 115396 additions and 61335 deletions

No files matched your search

-7
View File
@@ -1,7 +0,0 @@
# Logins allowed to open pull requests, one per line, matched case-insensitively.
# Repository owners, members and collaborators, and bots, are always allowed.
lancy69
chuiwenwei
xxoingr
kevin9327
imcreazy
-3
View File
@@ -2,9 +2,6 @@
Target branch: 2.x features and fixes go to `studio`. 1.x takes bug fixes,
provider/API compatibility, release/updater, security and platform stability
only, on `main-v2`.
Pull requests are limited to maintainers and invited contributors; one from
anyone else is closed automatically. Open an issue instead.
-->
## Summary
@@ -0,0 +1,24 @@
[
{
"package": "reasonix/internal/runtime/delegation",
"tests": [
"TestAncestorCopyAdmissionCoversUnpublishedRun",
"TestAncestorCopyAdmissionCoversUnpublishedRun/continue_then_continue",
"TestAncestorCopyAdmissionCoversUnpublishedRun/continue_then_legacy",
"TestAncestorCopyAdmissionCoversUnpublishedRun/legacy_then_continue",
"TestAncestorCopyAdmissionCoversUnpublishedRun/legacy_then_legacy",
"TestAncestorCopyAdmissionAllowsDifferentParents",
"TestAncestorCopyAdmissionReleasesAbandonedPreparation",
"TestAncestorCopyAdmissionReleasesAfterInvalidCopy",
"TestAncestorCopyAdmissionAllowsDifferentSources",
"TestAncestorCopyAdmissionUsesNearestSourceIdentity",
"TestAncestorCopyAdmissionReleasesAfterBusyCurrentReference",
"TestTaskAncestorCopyAdmissionBeforeRunningMetadata",
"TestTaskAncestorCopyAdmissionBeforeRunningMetadata/continue_from",
"TestTaskAncestorCopyAdmissionBeforeRunningMetadata/fork_from",
"TestTaskAncestorCopyAdmissionReleasesUnstartedBackground",
"TestTaskAncestorCopyAdmissionReleasesUnstartedBackground/continue_from",
"TestTaskAncestorCopyAdmissionReleasesUnstartedBackground/fork_from"
]
}
]
@@ -0,0 +1,26 @@
[
{
"package": "reasonix/internal/runtime/agent",
"tests": [
"TestCarriedProofInvalidatesPermissionChange",
"TestCarriedProofFileStateCompatibility",
"TestCarriedProofFileStateCompatibility/legacy_file",
"TestCarriedProofFileStateCompatibility/legacy_absent",
"TestCarriedProofFileStateCompatibility/future",
"TestCarriedProofFileStateCompatibility/current_file",
"TestCarriedProofFileStateCompatibility/current_absent",
"TestCarriedProofRejectsDirectoryState",
"TestCarriedProofStandsOnlyForUnchangedContent",
"TestCarriedProofSeesADeletion"
]
},
{
"package": "reasonix/internal/assembly/boot",
"tests": [
"TestEffectCarriedProofTracksFileState",
"TestEffectCarriedProofTracksFileState/unchanged",
"TestEffectCarriedProofTracksFileState/content",
"TestEffectCarriedProofTracksFileState/mode"
]
}
]
-293
View File
@@ -1,293 +0,0 @@
name: Android
# The Android shell under `android/` ships from the same tag the desktop line
# publishes, and lands on the same release page.
#
# It is a separate workflow rather than a job inside release-studio.yml for two
# reasons. The toolchains share nothing — Go, the Android SDK, a Gradle wrapper —
# so a runner carrying both would install three toolchains to build one artifact.
# And `signing-contract` pins which workflows may reach the SignPath credentials;
# this one never touches them, and keeping it out of that graph is what keeps the
# fingerprint stable.
#
# A failure here does not hold back the desktop release: the two are independent
# by construction, and an APK is worth less than the desktop builds it sits
# beside.
on:
push:
tags: ['studio-v*']
workflow_dispatch:
inputs:
tag:
description: "Existing studio tag to build for and attach to (for example studio-v2.51.0)"
required: false
type: string
ref:
description: "A branch to build as a trial run of this workflow; nothing is attached to any release"
required: false
type: string
permissions:
contents: write # upload the APK to the release the tag belongs to
concurrency:
group: android-${{ inputs.tag || inputs.ref || github.ref_name }}
cancel-in-progress: false
jobs:
resolve:
name: resolve the tag
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
tag: ${{ steps.release.outputs.tag }}
version: ${{ steps.release.outputs.version }}
sha: ${{ steps.release.outputs.sha }}
candidate: ${{ steps.release.outputs.candidate }}
publish: ${{ steps.release.outputs.publish }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
persist-credentials: false
# The same tag grammar release-studio.yml enforces, spelled the same way:
# two workflows that disagree about which tags name a release would publish
# one artifact for a tag the other refused.
- name: Resolve tag and version
id: release
env:
IN_TAG: ${{ inputs.tag }}
IN_REF: ${{ inputs.ref }}
REF_NAME: ${{ github.ref_name }}
run: |
# A trial run builds a branch as a throwaway version and publishes nothing.
if [ -n "$IN_REF" ] && [ -z "$IN_TAG" ]; then
sha="$(git rev-parse --verify "origin/$IN_REF^{commit}")"
{ echo "tag="; echo "version=v0.0.0-dev.${sha:0:7}"; echo "sha=$sha"
echo "candidate=false"; echo "publish=false"; } >> "$GITHUB_OUTPUT"
echo "trial build of $IN_REF at $sha"
exit 0
fi
tag="${IN_TAG:-$REF_NAME}"
semver='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-([0-9A-Za-z-]+)(\.[0-9A-Za-z-]+)*)?$'
if [[ ! "$tag" =~ ^studio-v(.+)$ ]] || [[ ! "${BASH_REMATCH[1]}" =~ $semver ]]; then
echo "::error::studio release tag must be studio-vMAJOR.MINOR.PATCH[-PRERELEASE], got: $tag"
exit 1
fi
sha="$(git rev-parse --verify "refs/tags/$tag^{commit}")"
if ! git merge-base --is-ancestor "$sha" origin/studio; then
echo "::error::$tag is not on studio history"
exit 1
fi
version="${tag#studio-}"
candidate=false
if [[ "${version#v}" =~ $semver ]] && [ -n "${BASH_REMATCH[4]}" ]; then
candidate=true
fi
{ echo "tag=$tag"; echo "version=$version"; echo "sha=$sha"
echo "candidate=$candidate"; echo "publish=true"; } >> "$GITHUB_OUTPUT"
echo "resolved $tag -> $version at $sha (candidate=$candidate)"
build:
name: build the APK
needs: resolve
runs-on: ubuntu-latest
permissions:
contents: read
# `publish` decides whether to attach anything, and the only honest answer to
# "will this APK install" is whether a key was restored.
outputs:
signed: ${{ steps.key.outputs.signed }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ needs.resolve.outputs.sha }}
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: go.mod
cache: true
cache-dependency-path: go.sum
# pnpm first: setup-node's pnpm cache needs the executable to exist.
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v4
with:
version: 10
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
with:
node-version: "24"
cache: pnpm
cache-dependency-path: desktop/frontend-next/pnpm-lock.yaml
- uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
with:
distribution: temurin
java-version: "21"
cache: gradle
# The SDK is the one thing nothing sets for you, per android/README.md. The
# runner image carries one; this asserts the two pieces the build names
# rather than re-installing them, so a missing package fails here by name
# instead of surfacing later as an unresolved compileSdk.
- name: Provide the Android SDK
run: |
if [ -z "${ANDROID_HOME:-}" ]; then
echo "::error::ANDROID_HOME is unset; the runner image is expected to provide the Android SDK"
exit 1
fi
for piece in "platforms/android-35" "build-tools/35.0.0"; do
if [ ! -d "$ANDROID_HOME/$piece" ]; then
echo "::error::the Android SDK at $ANDROID_HOME has no $piece; android/README.md names it as a requirement"
exit 1
fi
done
echo "ANDROID_SDK_ROOT=$ANDROID_HOME" >> "$GITHUB_ENV"
# A key that exists signs; a key that does not leaves the artifact
# unsigned, which is what a release with no key has always produced. The
# decision is the repository's, not this workflow's: it reads the four
# secrets android/README.md already names.
- name: Restore the signing key
id: key
env:
KEYSTORE_BASE64: ${{ secrets.REASONIX_KEYSTORE_BASE64 }}
STORE_PASSWORD: ${{ secrets.REASONIX_KEYSTORE_PASSWORD }}
KEY_PASSWORD: ${{ secrets.REASONIX_KEY_PASSWORD }}
KEY_ALIAS: ${{ secrets.REASONIX_KEY_ALIAS }}
run: |
if [ -z "$KEYSTORE_BASE64" ] || [ -z "$STORE_PASSWORD" ] || [ -z "$KEY_ALIAS" ]; then
echo "signed=false" >> "$GITHUB_OUTPUT"
echo "no signing key configured: the APK will be unsigned"
exit 0
fi
mkdir -p android/keystore
printf '%s' "$KEYSTORE_BASE64" | base64 --decode > android/keystore/release.jks
{
echo "storeFile=keystore/release.jks"
echo "storePassword=${STORE_PASSWORD}"
echo "keyAlias=${KEY_ALIAS}"
# PKCS#12 keystores use one password for both; only write the second
# when it is genuinely a different fact.
[ -n "$KEY_PASSWORD" ] && echo "keyPassword=${KEY_PASSWORD}"
} > android/keystore/keystore.properties
chmod 600 android/keystore/*
echo "signed=true" >> "$GITHUB_OUTPUT"
echo "signing key restored"
- name: Build the kernel and the page
working-directory: android
env:
VERSION: ${{ needs.resolve.outputs.version }}
run: |
export REASONIX_APP_VERSION="${VERSION#v}"
bash ./scripts/build-kernel.sh
bash ./scripts/sync-page.sh
# The one end-to-end check that needs no phone: the kernel the shell starts,
# serving the page the APK carries. Running it here means an APK that would
# open blank fails the build instead of the install.
- name: Verify the packaged page
working-directory: android
run: |
(cd .. && go build -o reasonix-verify ./cmd/reasonix)
KERNEL_BIN="$PWD/../reasonix-verify" bash ./scripts/verify-page.sh
- name: Assemble
working-directory: android
env:
VERSION: ${{ needs.resolve.outputs.version }}
run: |
# The version comes from the tag, so the About screen and the update
# check name this release. versionCode keeps its existing shape, which
# is why the build digit stays 1: it is a single decimal place, and a
# number that grows past it would collide with the next patch.
./gradlew assembleRelease --no-daemon "-PreasonixVersion=${VERSION#v}"
- name: Name the artifact
id: apk
working-directory: android/app/build/outputs/apk/release
run: |
src="$(ls ./*.apk | head -n 1)"
if [ -z "$src" ]; then
echo "::error::assembleRelease produced no APK"
exit 1
fi
case "$src" in
*-unsigned.apk) name="ReasonixMobile-${{ needs.resolve.outputs.version }}-unsigned.apk" ;;
*) name="ReasonixMobile-${{ needs.resolve.outputs.version }}.apk" ;;
esac
mv "$src" "$name"
sha256sum "$name" > "$name.sha256"
{
echo "path=$PWD/$name"
echo "name=$name"
} >> "$GITHUB_OUTPUT"
ls -l "$name" "$name.sha256"
# An unsigned package is not a lesser APK, it is an unusable one: Android's
# package manager rejects it outright rather than reporting a signature
# mismatch. Say so where the run summary can be read, instead of letting a
# green build imply a shippable artifact.
- name: Say what an unsigned build means
if: steps.key.outputs.signed != 'true'
run: |
echo "::warning::no signing key configured, so this APK is unsigned. Android refuses to install an unsigned package, so the release will carry no APK; the file stays a workflow artifact for whoever signs it."
{
echo "### No APK on this release"
echo ""
echo "The build produced an unsigned APK, which Android refuses to install."
echo "It is attached to this run as a workflow artifact only. Add the four"
echo "signing secrets to publish one."
} >> "$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v8
with:
name: android-apk
path: |
android/app/build/outputs/apk/release/*.apk
android/app/build/outputs/apk/release/*.sha256
if-no-files-found: error
publish:
name: attach the APK
needs: [resolve, build]
# Nothing to attach unless the build was signed. Skipping is the honest
# outcome: a release page with an APK nobody can install is worse than a
# release page without one.
if: needs.build.outputs.signed == 'true' && needs.resolve.outputs.publish == 'true'
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: android-apk
path: dist
# The release itself belongs to release-studio.yml, which creates it and
# owns its notes. This waits for it rather than creating a second one: two
# jobs racing to define a release is how a page ends up with the wrong body.
- name: Attach to the release
env:
GH_TOKEN: ${{ github.token }}
TAG: ${{ needs.resolve.outputs.tag }}
REPO: ${{ github.repository }}
run: |
# 90 minutes: on the last Studio release the page appeared about 40
# minutes after the tag run started, and the APK build spends part of
# that before this wait begins.
for _ in $(seq 1 360); do
if gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1; then
gh release upload "$TAG" dist/*.apk dist/*.sha256 --repo "$REPO" --clobber
echo "attached to $TAG"
exit 0
fi
sleep 15
done
echo "::error::release $TAG did not appear within 90 minutes; the APK was built but not published"
exit 1
+304 -177
View File
@@ -15,7 +15,7 @@ permissions:
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.ref == 'refs/heads/studio' }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
# Cheap path gate for pull requests. PRs confined to docs/release-notes
@@ -170,6 +170,23 @@ jobs:
shell: pwsh
run: Get-CimInstance Win32_Processor | Select-Object -ExpandProperty Name
- name: test (Windows concurrency contracts)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
node --test scripts/windows-concurrency-tests.test.mjs
node scripts/windows-concurrency-tests.mjs "$RUNNER_TEMP/concurrency-windows.jsonl"
- name: upload Windows concurrency inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: concurrency-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/concurrency-windows.jsonl
if-no-files-found: error
retention-days: 7
- name: test
if: env.RUN_STEPS == 'true' && runner.os != 'Windows'
env:
@@ -179,6 +196,35 @@ jobs:
REASONIX_RELEASE_CACHE_GUARD: "1"
run: go test ./...
- name: test (Windows subagent status admission)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^TestSubagent(PersistedStatusAdmission|UnknownStatusStillReadableForDiagnosis)$' ./internal/runtime/delegation | tee "$RUNNER_TEMP/subagent-status-admission.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
["future-paused", "missing", "empty", "COMPLETED", "completed", "cancelled", "failed", "interrupted", "running"] as $states |
["continue", "copy", "ancestor", "legacy_ancestor", "current_copy"] as $routes |
(["TestSubagentPersistedStatusAdmission", "TestSubagentUnknownStatusStillReadableForDiagnosis"] +
[$states[] | "TestSubagentPersistedStatusAdmission/" + .] +
[$states[] as $state | $routes[] | "TestSubagentPersistedStatusAdmission/" + $state + "/" + .]) as $required |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/runtime/delegation"] - $packages | length) == 0 and
($required - $tests | length) == 0
' "$RUNNER_TEMP/subagent-status-admission.jsonl"
- name: upload Windows subagent status admission inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: subagent-status-admission-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/subagent-status-admission.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows composition links)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
@@ -255,30 +301,23 @@ jobs:
shell: bash
run: |
set -o pipefail
go test -count=1 -p 4 -timeout=18m -json ./internal/platform/worktree/... ./internal/state/mutation/... | tee "$RUNNER_TEMP/isolation-owners.jsonl" |
go test -count=1 -p 4 -timeout=18m -json ./internal/runtime/isolation/... ./internal/platform/worktree/... ./internal/state/mutation/... | tee "$RUNNER_TEMP/isolation-owners.jsonl" |
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $passed |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
(["reasonix/internal/platform/worktree", "reasonix/internal/state/mutation"] - $passed | length) == 0 and
(["TestHeldPathsCoverOnlyTheSameFileAcrossCaseSpellings",
(["reasonix/internal/runtime/isolation", "reasonix/internal/platform/worktree", "reasonix/internal/state/mutation"] - $passed | length) == 0 and
(["TestResultOperationUsesCrossProcessOwnership",
"TestActualResultOperationHoldsOwnershipThroughWriteAndCleanup/restore",
"TestActualResultOperationHoldsOwnershipThroughWriteAndCleanup/remove",
"TestApplyPathsWithUntrustedModesPreservesTheIndex/content",
"TestApplyPathsWithUntrustedModesPreservesTheIndex/source_index_mode",
"TestApplyPathsWithUntrustedModesPreservesTheIndex/candidate_mode",
"TestApplyPathsWithUntrustedModesPreservesTheIndex/candidate_type",
"TestApplyPathsEmulatedLinksUseSnapshotFilters",
"TestHeldPathsCoverOnlyTheSameFileAcrossCaseSpellings",
"TestLegacyWaitDoesNotConsumeExecutionCapacity"] - $tests | length) == 0
' "$RUNNER_TEMP/isolation-owners.jsonl"
jq -se '
. as $events |
{"reasonix/internal/platform/worktree": [
"TestWriteChangesRejectsRedirectedDeletion",
"TestWriteChangesUnlinksLeafDirectoryLink"
]} |
to_entries | all(.[]; . as $package |
([ $events[] | select(.Package == $package.key and .Test == null and
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
length == 1 and .[0].Action == "pass") and
all(.value[]; . as $test |
[ $events[] | select(.Package == $package.key and .Test == $test and
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
length == 1 and .[0].Action == "pass"))
' "$RUNNER_TEMP/isolation-owners.jsonl"
- name: upload Windows isolation test inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
@@ -289,15 +328,60 @@ jobs:
if-no-files-found: ignore
retention-days: 7
- name: test (Windows background resource owners)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 25
shell: bash
run: |
set -o pipefail
go test -count=1 -p 2 -timeout=18m -json ./internal/tools/jobs/... ./internal/runtime/delegation/... | tee "$RUNNER_TEMP/background-resources.jsonl" |
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
(["reasonix/internal/tools/jobs", "reasonix/internal/runtime/delegation"] - $packages | length) == 0 and
(["TestJobFinalizationCoversInvokedAndUninvoked/success",
"TestJobFinalizationCoversInvokedAndUninvoked/error",
"TestJobFinalizationCoversInvokedAndUninvoked/panic",
"TestJobFinalizationCoversInvokedAndUninvoked/closed",
"TestJobFinalizationCoversInvokedAndUninvoked/invalid_session",
"TestJobFinalizationCoversInvokedAndUninvoked/invalid_kind",
"TestJobFinalizationFailuresAttemptAllCallbacksAndKeepCauses/invoked",
"TestJobFinalizationFailuresAttemptAllCallbacksAndKeepCauses/uninvoked",
"TestJobFinalizationPrecedesCompletionPublication",
"TestJobFinalizationCloseBetweenPublicationAndLaunch",
"TestJobFinalizationWaitsForClosureAfterLogicalCancellation",
"TestBackgroundNoLaunchFinalizesOwnedResources/task/closed_manager",
"TestBackgroundNoLaunchFinalizesOwnedResources/task/invalid_session",
"TestBackgroundNoLaunchFinalizesOwnedResources/fleet/closed_manager",
"TestBackgroundNoLaunchFinalizesOwnedResources/fleet/invalid_session"] - $tests | length) == 0
' "$RUNNER_TEMP/background-resources.jsonl"
- name: upload Windows background resource inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: background-resources-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/background-resources.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows delegated Goal usage scope)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(UsageScope.*)$' ./internal/contract/event ./internal/session/control ./internal/assembly/boot | tee "$RUNNER_TEMP/goal-usage-scope.jsonl"
go test -count=1 -timeout=3m -json -run '^Test(UsageScope.*|DelegatedUsageKeepsGoalOwner|EffectDelegatedUsageKeepsGoalOwner)$' ./internal/contract/event ./internal/session/control ./internal/assembly/boot | tee "$RUNNER_TEMP/goal-usage-scope.jsonl"
jq -se '
["TestUsageScopeKeepsEarlierEventOwner",
"TestUsageScopeWithoutCapability" ] as $required |
["TestUsageScopePreservesCapturedOwnerAndWire", "TestUsageScopeKeepsEarlierEventOwner",
"TestUsageScopeWithoutCapability", "TestDelegatedUsageKeepsGoalOwner",
"TestDelegatedUsageKeepsGoalOwner/chat_to_goal", "TestDelegatedUsageKeepsGoalOwner/replaced_goal",
"TestDelegatedUsageKeepsGoalOwner/cleared_goal", "TestDelegatedUsageKeepsGoalOwner/resumed_goal",
"TestDelegatedUsageKeepsGoalOwner/same_goal_next_turn", "TestDelegatedUsageKeepsGoalOwner/between_turns",
"TestDelegatedUsageKeepsGoalOwner/nested_after_replacement", "TestEffectDelegatedUsageKeepsGoalOwner",
"TestEffectDelegatedUsageKeepsGoalOwner/chat_to_goal", "TestEffectDelegatedUsageKeepsGoalOwner/replaced_goal",
"TestEffectDelegatedUsageKeepsGoalOwner/nested_chat_to_goal", "TestEffectDelegatedUsageKeepsGoalOwner/same_goal",
"TestEffectDelegatedUsageKeepsGoalOwner/completed_before_goal"] as $required |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
all(.[]; .Action != "fail" and .Action != "skip") and
@@ -314,19 +398,60 @@ jobs:
if-no-files-found: error
retention-days: 7
- name: test (Windows planner local budget)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^TestEffectPlannerConfiguredTaskBudget$' ./internal/assembly/boot | tee "$RUNNER_TEMP/planner-local-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/assembly/boot"] - $packages | length) == 0 and
(["TestEffectPlannerConfiguredTaskBudget", "TestEffectPlannerConfiguredTaskBudget/tokens",
"TestEffectPlannerConfiguredTaskBudget/cost", "TestEffectPlannerConfiguredTaskBudget/wall",
"TestEffectPlannerConfiguredTaskBudget/unconfigured", "TestEffectPlannerConfiguredTaskBudget/explicit_zero",
"TestEffectPlannerConfiguredTaskBudget/unpriced_cost", "TestEffectPlannerConfiguredTaskBudget/missing_usage_cost",
"TestEffectPlannerConfiguredTaskBudget/context_bound", "TestEffectPlannerConfiguredTaskBudget/context_disables",
"TestEffectPlannerConfiguredTaskBudget/executor_only"] - $tests | length) == 0
' "$RUNNER_TEMP/planner-local-budget.jsonl"
- name: upload Windows planner local budget inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: planner-local-budget-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/planner-local-budget.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows Controller routing root budget)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(SemanticRootUsage)$' ./internal/runtime/capability | tee "$RUNNER_TEMP/controller-routing-budget.jsonl"
go test -count=1 -timeout=3m -json -run '^Test(SemanticRootUsage|EffectRoutingRootBudget)$' ./internal/runtime/capability ./internal/assembly/boot | tee "$RUNNER_TEMP/controller-routing-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/runtime/capability"] - $packages | length) == 0 and
(["reasonix/internal/runtime/capability", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
([
"TestEffectRoutingRootBudget",
"TestEffectRoutingRootBudget/balanced",
"TestEffectRoutingRootBudget/cached_continuation",
"TestEffectRoutingRootBudget/cached_new_scope",
"TestEffectRoutingRootBudget/cached_new_task",
"TestEffectRoutingRootBudget/cached_reset",
"TestEffectRoutingRootBudget/context_zero",
"TestEffectRoutingRootBudget/dual",
"TestEffectRoutingRootBudget/headless",
"TestEffectRoutingRootBudget/interactive",
"TestEffectRoutingRootBudget/missing",
"TestEffectRoutingRootBudget/unconfigured",
"TestSemanticRootUsage",
"TestSemanticRootUsage/billed_error",
"TestSemanticRootUsage/cache",
@@ -354,17 +479,22 @@ jobs:
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(PreparedRootBudget|CoordinatorRootBudget|EffectCoordinatorRoot)' ./internal/runtime/agent ./internal/runtime/coordinator | tee "$RUNNER_TEMP/coordinator-root-budget.jsonl"
go test -count=1 -timeout=3m -json -run '^Test(PreparedRootBudget|CoordinatorRootBudget|EffectCoordinatorRoot)' ./internal/runtime/agent ./internal/runtime/coordinator ./internal/assembly/boot | tee "$RUNNER_TEMP/coordinator-root-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/runtime/agent", "reasonix/internal/runtime/coordinator"] - $packages | length) == 0 and
(["TestCoordinatorRootBudgetRoutes",
(["reasonix/internal/runtime/agent", "reasonix/internal/runtime/coordinator", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
(["TestPreparedRootBudgetLifetime", "TestPreparedRootBudgetLifetime/continuation",
"TestPreparedRootBudgetLifetime/new_scope", "TestPreparedRootBudgetLifetime/new_task",
"TestPreparedRootBudgetLifetime/reset", "TestPreparedRootBudgetLifetime/new_session",
"TestPreparedRootBudgetProjectionAndPreservedTurn", "TestCoordinatorRootBudgetRoutes",
"TestCoordinatorRootBudgetRoutes/raw", "TestCoordinatorRootBudgetRoutes/tool_planner",
"TestCoordinatorRootBudgetRoutes/failure_fallback", "TestCoordinatorRootBudgetRoutes/plan_only_continuation",
"TestCoordinatorRootBudgetRoutes/approval_declined_continuation", "TestCoordinatorRootBudgetRoutes/approval_allowed"
] - $tests | length) == 0
"TestCoordinatorRootBudgetRoutes/approval_declined_continuation", "TestCoordinatorRootBudgetRoutes/approval_allowed",
"TestEffectCoordinatorRootTokenBudget", "TestEffectCoordinatorRootTokenBudget/shared",
"TestEffectCoordinatorRootTokenBudget/unconfigured", "TestEffectCoordinatorRootTokenBudget/context_disables",
"TestEffectCoordinatorRootTokenBudget/missing_planner", "TestEffectCoordinatorRootTokenBudget/executor_only"] - $tests | length) == 0
' "$RUNNER_TEMP/coordinator-root-budget.jsonl"
- name: upload Windows Coordinator root budget inventory
@@ -422,8 +552,10 @@ jobs:
"TestRootWallBudgetReachesDelegatedAgents/context", "TestRootWallBudgetReachesDelegatedAgents/unconfigured",
"TestRootWallBudgetReachesDelegatedAgents/explicit_zero", "TestRootWallBudgetContinuationAndReset",
"TestRootWallBudgetContinuationAndReset/continuation", "TestRootWallBudgetContinuationAndReset/new_task",
"TestRootWallBudgetContinuationAndReset/reset", "TestRootWallBudgetContinuationAndReset/override"
] - $tests | length) == 0
"TestRootWallBudgetContinuationAndReset/reset", "TestRootWallBudgetContinuationAndReset/override",
"TestRootWallBudgetProjectionKeepsJobDeadline", "TestEffectRootWallBudgetReachesManagedJobs",
"TestEffectRootWallBudgetReachesManagedJobs/task", "TestEffectRootWallBudgetReachesManagedJobs/capability",
"TestEffectRootWallBudgetReachesManagedJobs/fleet"] - $tests | length) == 0
' "$RUNNER_TEMP/root-wall-budget.jsonl"
- name: upload Windows root wall budget inventory
@@ -441,19 +573,22 @@ jobs:
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(RootToken|EffectRootToken|BudgetCountsKnownReportedTokens|ReportedToken)' ./internal/runtime/agent | tee "$RUNNER_TEMP/root-token-budget.jsonl"
go test -count=1 -timeout=3m -json -run '^Test(RootToken|EffectRootToken|BudgetCountsKnownReportedTokens|ReportedToken|EffectReportedTotalTokenBudget)' ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/root-token-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/runtime/agent"] - $packages | length) == 0 and
(["reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
(["TestRootTokenBudgetIncludesChildProviderUsage", "TestRootTokenBudgetIncludesChildProviderUsage/configured",
"TestRootTokenBudgetIncludesChildProviderUsage/context", "TestRootTokenBudgetIncludesChildProviderUsage/unconfigured",
"TestRootTokenBudgetIncludesChildProviderUsage/explicit_zero", "TestRootTokenBudgetSurvivesNestedAndConcurrentChildren",
"TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/nested", "TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/siblings",
"TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/child_override", "TestRootTokenBudgetContinuationAndReset",
"TestRootTokenBudgetContinuationAndReset/continuation", "TestRootTokenBudgetContinuationAndReset/new_task",
"TestRootTokenBudgetContinuationAndReset/reset" ] - $tests | length) == 0
"TestRootTokenBudgetContinuationAndReset/reset", "TestRootTokenScopeAtomicAccountingAndJobCancellation",
"TestEffectRootTokenBudgetReachesDelegatedJobs", "TestEffectRootTokenBudgetReachesDelegatedJobs/foreground",
"TestEffectRootTokenBudgetReachesDelegatedJobs/foreground_capability", "TestEffectRootTokenBudgetReachesDelegatedJobs/background",
"TestEffectRootTokenBudgetReachesDelegatedJobs/background_capability", "TestEffectRootTokenBudgetReachesDelegatedJobs/background_fleet"] - $tests | length) == 0
' "$RUNNER_TEMP/root-token-budget.jsonl"
jq -se '
. as $events |
@@ -467,7 +602,17 @@ jobs:
"TestBudgetCountsKnownReportedTokens/negative_split",
"TestBudgetCountsKnownReportedTokens/zero",
"TestBudgetCountsKnownReportedTokens/negative",
"TestReportedTokenCountersSaturate"]} |
"TestReportedTokensEnforceLocalCapOnInheritedRoot",
"TestReportedTokensFollowOriginalAuxiliaryRoot",
"TestReportedTokenCountersSaturate"],
"reasonix/internal/assembly/boot": [
"TestEffectReportedTotalTokenBudget",
"TestEffectReportedTotalTokenBudget/split",
"TestEffectReportedTotalTokenBudget/total_only",
"TestEffectReportedTotalTokenBudget/partial_split",
"TestEffectReportedTotalTokenBudget/lower_total",
"TestEffectReportedTotalTokenBudget/missing_total",
"TestEffectReportedTotalTokenBudget/disabled"]} |
to_entries | all(.[]; . as $package |
([ $events[] | select(.Package == $package.key and .Test == null and
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
@@ -526,7 +671,7 @@ jobs:
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(HostStorageEnvironment|HostObjectStorage|HostExplicitStorageControls|DetachedStorageEnvironment)$' ./internal/platform/gitcmd ./internal/platform/worktree ./internal/assembly/boot | tee "$RUNNER_TEMP/git-storage-environment.jsonl"
go test -count=1 -timeout=3m -json -run '^Test(HostStorageEnvironment|HostObjectStorage|HostExplicitStorageControls|DetachedStorageEnvironment|WorktreePreparationOwnsItsIndex|EffectIsolatedTaskOwnsGitStorage)$' ./internal/platform/gitcmd ./internal/platform/worktree ./internal/assembly/boot | tee "$RUNNER_TEMP/git-storage-environment.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
@@ -536,6 +681,10 @@ jobs:
"TestDetachedStorageEnvironment",
"TestDetachedStorageEnvironment/explicit_env",
"TestDetachedStorageEnvironment/inherited_env",
"TestEffectIsolatedTaskOwnsGitStorage",
"TestEffectIsolatedTaskOwnsGitStorage/ambient_existing",
"TestEffectIsolatedTaskOwnsGitStorage/ambient_missing",
"TestEffectIsolatedTaskOwnsGitStorage/control",
"TestHostExplicitStorageControls",
"TestHostExplicitStorageControls/explicit_object_directory",
"TestHostExplicitStorageControls/private_index",
@@ -546,8 +695,15 @@ jobs:
"TestHostStorageEnvironment",
"TestHostStorageEnvironment/GIT_ALTERNATE_OBJECT_DIRECTORIES",
"TestHostStorageEnvironment/GIT_INDEX_FILE",
"TestHostStorageEnvironment/GIT_OBJECT_DIRECTORY"
] - $tests | length) == 0
"TestHostStorageEnvironment/GIT_OBJECT_DIRECTORY",
"TestWorktreePreparationOwnsItsIndex",
"TestWorktreePreparationOwnsItsIndex/candidate_ambient_existing",
"TestWorktreePreparationOwnsItsIndex/candidate_ambient_missing",
"TestWorktreePreparationOwnsItsIndex/candidate_control",
"TestWorktreePreparationOwnsItsIndex/delivery_ambient_existing",
"TestWorktreePreparationOwnsItsIndex/delivery_ambient_missing",
"TestWorktreePreparationOwnsItsIndex/delivery_control"
] - $tests | length) == 0
' "$RUNNER_TEMP/git-storage-environment.jsonl"
- name: upload Windows Git storage environment inventory
@@ -597,19 +753,20 @@ jobs:
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(RootWebSearch|EffectRootWebSearch)' ./internal/tools/websearch | tee "$RUNNER_TEMP/root-websearch-budget.jsonl"
go test -count=1 -timeout=3m -json -run '^Test(RootWebSearch|EffectRootWebSearch)' ./internal/tools/websearch ./internal/assembly/boot | tee "$RUNNER_TEMP/root-websearch-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/tools/websearch"] - $packages | length) == 0 and
(["reasonix/internal/tools/websearch", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
(["TestRootWebSearchUsageTerminal", "TestRootWebSearchUsageTerminal/reported",
"TestRootWebSearchUsageTerminal/no_reporter", "TestRootWebSearchUsageTerminal/missing",
"TestRootWebSearchUsageTerminal/no_observer", "TestRootWebSearchUsageTerminal/input_refused",
"TestRootWebSearchUsageTerminal/no_factory", "TestRootWebSearchUsageTerminal/factory_error",
"TestRootWebSearchUsageTerminal/billed_failure", "TestRootWebSearchUsageTerminal/cancel_after_usage",
"TestRootWebSearchUsageTerminal/cancel_before_start", "TestRootWebSearchUsageTerminal/stream_error"
] - $tests | length) == 0
"TestRootWebSearchUsageTerminal/cancel_before_start", "TestRootWebSearchUsageTerminal/stream_error",
"TestEffectRootWebSearchTokenBudget", "TestEffectRootWebSearchTokenBudget/reported",
"TestEffectRootWebSearchTokenBudget/disabled", "TestEffectRootWebSearchTokenBudget/missing"] - $tests | length) == 0
' "$RUNNER_TEMP/root-websearch-budget.jsonl"
- name: upload Windows root websearch budget inventory
@@ -627,13 +784,21 @@ jobs:
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^(TestProfileRootBudgetLifetime|TestProfileBudgetPreparationUsesTurnAdmission|TestEffectExplicitSkillRootBudget)$' ./internal/session/control | tee "$RUNNER_TEMP/explicit-skill-root-budget.jsonl"
go test -count=1 -timeout=3m -json -run '^(TestProfileRootBudgetLifetime|TestProfileBudgetPreparationUsesTurnAdmission|TestEffectExplicitSkillRootBudget)$' ./internal/session/control ./internal/assembly/boot | tee "$RUNNER_TEMP/explicit-skill-root-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/session/control"] - $packages | length) == 0 and
(["TestProfileBudgetPreparationUsesTurnAdmission" ] - $tests | length) == 0
(["reasonix/internal/session/control", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
(["TestProfileRootBudgetLifetime", "TestProfileRootBudgetLifetime/new_task",
"TestProfileRootBudgetLifetime/same_scope", "TestProfileRootBudgetLifetime/new_scope",
"TestProfileRootBudgetLifetime/reset", "TestProfileRootBudgetLifetime/new_session",
"TestProfileRootBudgetLifetime/inherited", "TestProfileRootBudgetLifetime/context_zero",
"TestProfileBudgetPreparationUsesTurnAdmission", "TestEffectExplicitSkillRootBudget",
"TestEffectExplicitSkillRootBudget/profile", "TestEffectExplicitSkillRootBudget/preview",
"TestEffectExplicitSkillRootBudget/slash", "TestEffectExplicitSkillRootBudget/chips",
"TestEffectExplicitSkillRootBudget/goal", "TestEffectExplicitSkillRootBudget/context_zero",
"TestEffectExplicitSkillRootBudget/unconfigured"] - $tests | length) == 0
' "$RUNNER_TEMP/explicit-skill-root-budget.jsonl"
- name: upload Windows explicit skill root budget inventory
@@ -651,16 +816,19 @@ jobs:
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(RootTriage|EffectRootTriage)' ./internal/runtime/agent | tee "$RUNNER_TEMP/root-triage-budget.jsonl"
go test -count=1 -timeout=3m -json -run '^Test(RootTriage|EffectRootTriage)' ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/root-triage-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/runtime/agent"] - $packages | length) == 0 and
(["reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
(["TestRootTriageTokenBudget", "TestRootTriageTokenBudget/classifier", "TestRootTriageTokenBudget/screen",
"TestRootTriageTokenBudget/billed_failure", "TestRootTriageTokenBudget/missing", "TestRootTriageTokenBudget/disabled",
"TestRootTriageTokenBudget/no_sink", "TestRootTriageTokenBudget/counts_without_total",
"TestRootTriageOutsideRunDoesNotCreateScope" ] - $tests | length) == 0
"TestRootTriageOutsideRunDoesNotCreateScope", "TestEffectRootTriageScreenBudget",
"TestEffectRootTriageScreenBudget/reported", "TestEffectRootTriageScreenBudget/flagged",
"TestEffectRootTriageScreenBudget/disabled", "TestEffectRootTriageScreenBudget/missing",
"TestEffectRootTriageScreenBudget/screen_disabled"] - $tests | length) == 0
' "$RUNNER_TEMP/root-triage-budget.jsonl"
- name: upload Windows root triage budget inventory
@@ -672,22 +840,54 @@ jobs:
if-no-files-found: ignore
retention-days: 7
- name: test (Windows root auxiliary budget)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
timeout-minutes: 5
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(RootAuxiliary|EffectRootAuxiliary)' ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/root-auxiliary-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
(["TestRootAuxiliaryAutomaticCompaction", "TestRootAuxiliaryTokenBudget",
"TestRootAuxiliaryTokenBudget/summary", "TestRootAuxiliaryTokenBudget/summary_failed",
"TestRootAuxiliaryTokenBudget/summary_missing", "TestRootAuxiliaryTokenBudget/summary_disabled",
"TestRootAuxiliaryTokenBudget/diagnosis", "TestRootAuxiliaryTokenBudget/diagnosis_failed",
"TestRootAuxiliaryTokenBudget/diagnosis_missing", "TestRootAuxiliaryTokenBudget/diagnosis_disabled",
"TestRootAuxiliaryManualCompactionOutsideRun", "TestRootAuxiliaryBudgetProjectionKeepsOldTask",
"TestEffectRootAuxiliarySummaryBudget", "TestEffectRootAuxiliarySummaryBudget/configured",
"TestEffectRootAuxiliarySummaryBudget/disabled", "TestEffectRootAuxiliarySummaryBudget/missing_usage"] - $tests | length) == 0
' "$RUNNER_TEMP/root-auxiliary-budget.jsonl"
- name: upload Windows root auxiliary budget inventory
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: root-auxiliary-budget-windows-${{ github.run_attempt }}
path: ${{ runner.temp }}/root-auxiliary-budget.jsonl
if-no-files-found: ignore
retention-days: 7
- name: test (Windows local cost quote)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(LocalBudgetUsesOccurrenceQuote|LocalCost.*|UsageQuote.*)$' ./internal/contract/event ./internal/runtime/agent | tee "$RUNNER_TEMP/local-cost-quote.jsonl"
go test -count=1 -timeout=3m -json -run '^Test(LocalBudgetUsesOccurrenceQuote|LocalCost.*|UsageQuote.*|EffectLocalCostUsesOccurrenceQuote)$' ./internal/contract/event ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/local-cost-quote.jsonl"
jq -se '
["TestLocalBudgetUsesOccurrenceQuote", "TestLocalBudgetUsesOccurrenceQuote/peak", "TestLocalBudgetUsesOccurrenceQuote/offpeak",
"TestLocalCostKeepsOriginalCurrencyBuckets", "TestLocalCostFixedPointThresholdAndUnknown",
"TestUsageQuoteUsesOneOwnerAndOccurrence", "TestUsageQuoteLegacyAndNilSinks"
] as $required |
"TestUsageQuoteUsesOneOwnerAndOccurrence", "TestUsageQuoteLegacyAndNilSinks",
"TestEffectLocalCostUsesOccurrenceQuote", "TestEffectLocalCostUsesOccurrenceQuote/peak",
"TestEffectLocalCostUsesOccurrenceQuote/offpeak", "TestEffectLocalCostUsesOccurrenceQuote/relay"] as $required |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
all(.[]; .Action != "fail" and .Action != "skip") and
(($required - $passed) | length == 0) and
((["reasonix/internal/contract/event", "reasonix/internal/runtime/agent"] - $packages) | length == 0)
((["reasonix/internal/contract/event", "reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages) | length == 0)
' "$RUNNER_TEMP/local-cost-quote.jsonl"
- name: upload Windows local cost quote inventory
@@ -737,18 +937,24 @@ jobs:
shell: bash
run: |
set -o pipefail
go test -count=1 -timeout=3m -json -run '^Test(AuxiliaryUsageObserver|BoundedUsageObserver|RootBounded|EffectRootBounded)' ./internal/contract/provider ./internal/model/boundedllm ./internal/runtime/agent | tee "$RUNNER_TEMP/bounded-root-budget.jsonl"
go test -count=1 -timeout=3m -json -run '^Test(AuxiliaryUsageObserver|BoundedUsageObserver|RootBounded|EffectRootBounded)' ./internal/contract/provider ./internal/model/boundedllm ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/bounded-root-budget.jsonl"
jq -se '
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
all(.[]; .Action != "skip" and .Action != "fail") and
(["reasonix/internal/contract/provider", "reasonix/internal/model/boundedllm",
"reasonix/internal/runtime/agent"] - $packages | length) == 0 and
"reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
(["TestAuxiliaryUsageObserverReplacesAndCopies",
"TestBoundedUsageObserverOwnsOneTerminal", "TestBoundedUsageObserverOwnsOneTerminal/reported",
"TestBoundedUsageObserverOwnsOneTerminal/billed_failure", "TestBoundedUsageObserverOwnsOneTerminal/without_sink",
"TestBoundedUsageObserverOwnsOneTerminal/missing", "TestBoundedUsageObserverOwnsOneTerminal/input_refused"
] - $tests | length) == 0
"TestBoundedUsageObserverOwnsOneTerminal/missing", "TestBoundedUsageObserverOwnsOneTerminal/input_refused",
"TestRootBoundedReviewerBudget", "TestRootBoundedReviewerBudget/reported",
"TestRootBoundedReviewerBudget/billed_failure", "TestRootBoundedReviewerBudget/missing_usage",
"TestRootBoundedReviewerBudget/disabled", "TestRootBoundedReviewerBudget/no_emission",
"TestRootBoundedReviewerBudget/projected_job_context", "TestRootBoundedNestedBudgetChargesOnce",
"TestRootBoundedLateProjectionStaysWithOldTask", "TestEffectRootBoundedAdvisorBudget",
"TestEffectRootBoundedAdvisorBudget/reported", "TestEffectRootBoundedAdvisorBudget/disabled",
"TestEffectRootBoundedAdvisorBudget/missing_usage"] - $tests | length) == 0
' "$RUNNER_TEMP/bounded-root-budget.jsonl"
- name: upload Windows bounded root budget inventory
@@ -777,6 +983,42 @@ jobs:
timeout-minutes: 5
run: go test -count=1 -timeout=3m -run '^TestBashPowerShellExecuteDetailedContract$|^TestBashPowerShell51PreflightRejectsAndAndDetailed$|^TestBashPowerShellOutputIsUTF8$|^TestBashPowerShellSurfacesNonZeroExit$|^TestBashPowerShellRejectsChaining$' ./internal/tools/builtin
# Full push suite uses the same 18m package timeout as Windows smoke and a
# 35m step budget above it. -count=1 as at the smoke step. A failure prints
# the failed tests' output and the 15 slowest boot tests.
- name: test (full)
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name != 'pull_request'
timeout-minutes: 35
env:
# Run the prompt-cache prefix-stability guard (TestCacheHit*) in CI: a
# regression there silently tanks the cache hit rate the project is
# built around.
REASONIX_RELEASE_CACHE_GUARD: "1"
# Bound sandbox helper children in Windows tests so a failed OS-level
# launch cannot pin the Actions step after Go's package timeout fires.
WINDOWS_SANDBOX_WAIT_MS: "20000"
shell: bash
run: |
# internal/assembly/boot alone takes 14-15 minutes on a Windows runner;
# the package limit leaves headroom inside the step limit above.
go test -count=1 -p 4 -timeout=28m -json ./... | tee "$RUNNER_TEMP/full.jsonl" |
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
- name: test (full) failure detail
if: failure() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name != 'pull_request'
shell: bash
run: |
f="$RUNNER_TEMP/full.jsonl"
[ -s "$f" ] || exit 0
echo "::group::output of failed tests"
jq -rn '[inputs | select(.Test != null)] | group_by(.Package + "/" + .Test)
| map(select(any(.[]; .Action == "fail")))[]
| map(select(.Action == "output") | .Output) | join("")' "$f" || true
echo "::endgroup::"
echo "slowest internal/assembly/boot tests (seconds, name):"
jq -rn '[inputs | select((.Package | endswith("/internal/assembly/boot")) and .Test != null and .Action == "pass")]
| sort_by(-.Elapsed)[:15][] | "\(.Elapsed)\t\(.Test)"' "$f" || true
race:
needs: changes
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.code != 'false')
@@ -811,130 +1053,15 @@ jobs:
if: github.event_name == 'pull_request'
env:
REASONIX_RELEASE_CACHE_GUARD: "1"
run: go test -race ./internal/runtime/agent/... ./internal/runtime/delegation/... ./internal/platform/worktree/... ./internal/state/mutation/... ./internal/ext/plugin/... ./internal/tools/jobs/... ./internal/base/proc/... ./internal/safety/sandbox/... ./internal/base/filelock/... ./internal/contract/eventwire/... ./internal/platform/remote/... ./internal/ext/extension/... ./internal/assembly/boot/... ./internal/session/control/... ./internal/contract/tool/...
run: go test -race ./internal/runtime/agent/... ./internal/runtime/delegation/... ./internal/runtime/isolation/... ./internal/platform/worktree/... ./internal/state/mutation/... ./internal/ext/plugin/... ./internal/tools/jobs/... ./internal/base/proc/... ./internal/safety/sandbox/... ./internal/base/filelock/... ./internal/contract/eventwire/... ./internal/platform/remote/... ./internal/ext/extension/... ./internal/assembly/boot/... ./internal/session/control/... ./internal/contract/tool/...
# Push-only full suites. internal/frontend/serve and internal/assembly/boot run
# hundreds of serial tests and set the wall time of one whole-tree `go test`,
# so each is cut by test name into shards that run on their own runners. The
# -run / -skip patterns partition the names exactly: every test lands in one
# shard. `rest` is every other package. Pull requests keep the focused steps
# in the `test` and `race` jobs above.
test-windows-shards:
if: github.event_name != 'pull_request'
name: test (windows-latest, ${{ matrix.shard }})
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
include:
- { shard: rest }
- { shard: serve-1, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[A-E]' }
- { shard: serve-2, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[F-R]' }
- { shard: serve-3, pkg: ./internal/frontend/serve, flag: -skip, filter: '^Test[A-R]' }
- { shard: boot-1, pkg: ./internal/assembly/boot, flag: -skip, filter: '^TestEffect([^0-9A-C]|$)' }
- { shard: boot-2, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect[D-O]' }
- { shard: boot-3, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect([^0-9A-O]|$)' }
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: go.mod
cache: true
- name: test (shard)
timeout-minutes: 40
env:
# Run the prompt-cache prefix-stability guard (TestCacheHit*) in CI: a
# regression there silently tanks the cache hit rate the project is
# built around.
REASONIX_RELEASE_CACHE_GUARD: "1"
# Bound sandbox helper children in Windows tests so a failed OS-level
# launch cannot pin the Actions step after Go's package timeout fires.
WINDOWS_SANDBOX_WAIT_MS: "20000"
SHARD: ${{ matrix.shard }}
PKG: ${{ matrix.pkg }}
FLAG: ${{ matrix.flag }}
FILTER: ${{ matrix.filter }}
shell: bash
run: |
if [ "$SHARD" = rest ]; then
mapfile -t pkgs < <(go list ./... | tr -d '\r' | grep -v -E '/internal/(frontend/serve|assembly/boot)$')
else
pkgs=("$PKG")
fi
filter=()
if [ -n "$FILTER" ]; then filter=("$FLAG" "$FILTER"); fi
go test -count=1 -p 4 -timeout=36m -json "${filter[@]}" "${pkgs[@]}" | tee "$RUNNER_TEMP/shard.jsonl" |
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
- name: test (shard) failure detail
if: failure()
shell: bash
run: |
f="$RUNNER_TEMP/shard.jsonl"
[ -s "$f" ] || exit 0
echo "::group::output of failed tests"
jq -rn '[inputs | select(.Test != null)] | group_by(.Package + "/" + .Test)
| map(select(any(.[]; .Action == "fail")))[]
| map(select(.Action == "output") | .Output) | join("")' "$f" || true
echo "::endgroup::"
echo "slowest tests (seconds, package, name):"
jq -rn '[inputs | select(.Test != null and .Action == "pass")]
| sort_by(-.Elapsed)[:15][] | "\(.Elapsed) \(.Package) \(.Test)"' "$f" || true
race-shards:
if: github.event_name != 'pull_request'
name: race (${{ matrix.shard }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- { shard: rest }
- { shard: serve-1, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[A-E]' }
- { shard: serve-2, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[F-R]' }
- { shard: serve-3, pkg: ./internal/frontend/serve, flag: -skip, filter: '^Test[A-R]' }
- { shard: boot-1, pkg: ./internal/assembly/boot, flag: -skip, filter: '^TestEffect([^0-9A-C]|$)' }
- { shard: boot-2, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect[D-O]' }
- { shard: boot-3, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect([^0-9A-O]|$)' }
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: go.mod
cache: true
- name: Install and verify Linux sandbox backend
run: |
sudo apt-get update
sudo apt-get install -y bubblewrap
if sysctl -n kernel.unprivileged_userns_clone >/dev/null 2>&1; then
sudo sysctl -w kernel.unprivileged_userns_clone=1
fi
if sysctl -n kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
fi
bwrap --ro-bind / / --dev /dev --proc /proc -- true || echo "::warning::bubblewrap is installed but this runner denies user namespaces; unavailable-backend tests will run fail-closed"
- name: test -race (shard)
timeout-minutes: 30
- name: test -race (full)
if: github.event_name != 'pull_request'
env:
REASONIX_RELEASE_CACHE_GUARD: "1"
SHARD: ${{ matrix.shard }}
PKG: ${{ matrix.pkg }}
FLAG: ${{ matrix.flag }}
FILTER: ${{ matrix.filter }}
run: |
if [ "$SHARD" = rest ]; then
mapfile -t pkgs < <(go list ./... | grep -v -E '/internal/(frontend/serve|assembly/boot)$')
else
pkgs=("$PKG")
fi
filter=()
if [ -n "$FILTER" ]; then filter=("$FLAG" "$FILTER"); fi
go test -race -timeout=25m "${filter[@]}" "${pkgs[@]}"
# internal/assembly/boot under -race runs past ten minutes on a shared
# runner; the package limit leaves headroom over the default.
run: go test -race -timeout=25m ./...
# sdk/go is a nested stdlib-only module invisible to root `go test ./...`.
# Its DTOs are generated from internal/ext/extension/protocol, and the
-27
View File
@@ -1,27 +0,0 @@
name: PR author gate
# Pull requests are limited to maintainers and the logins in .github/pr-authors.txt.
# This runs in the base repository's context, checks out the base branch only, and
# never executes pull-request code; the allowlist it reads is the base branch's.
on:
pull_request_target:
types: [opened]
permissions:
contents: read
pull-requests: write
concurrency:
group: pr-author-gate-${{ github.event.pull_request.number }}
cancel-in-progress: false
jobs:
gate:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- run: node scripts/pr-author-gate.mjs
env:
GH_TOKEN: ${{ github.token }}
+1 -2
View File
@@ -16,7 +16,7 @@ permissions:
concurrency:
group: studio-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.ref == 'refs/heads/studio' }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
frontend:
@@ -188,7 +188,6 @@ jobs:
node perf/web-access.mjs
node perf/peers-settings.mjs
node perf/output-cap.mjs
node perf/tool-output-clip.mjs
node perf/agent-run-tab.mjs
node perf/changes-cap.mjs
node perf/session-menu-fit.mjs
-13
View File
@@ -169,19 +169,6 @@ Every package ships with a `.minisig` signature, and the release carries
The same release also carries the 2.x `reasonix` CLI as archives for
`darwin|linux|windows × amd64|arm64`.
The Android build ships on that release as `ReasonixMobile-<version>.apk`
(`arm64`, Android 8.0 or newer). The kernel and the page run on the phone, and
the agent reaches your servers over SSH; pairing with a computer is done by
scanning the code it shows.
The APK is signed with the Android release key rather than minisign, so it
carries a `.sha256` and no `.minisig`.
When that key is absent the build is named
`ReasonixMobile-<version>-unsigned.apk`. Android refuses to install an unsigned
package, so those builds stay a workflow artifact for whoever wants to sign one,
and the release carries no APK at all.
### Reasonix 1.x (stable)
The 1.x CLI installs through npm on any supported platform, or Homebrew on
-9
View File
@@ -156,15 +156,6 @@ Reasonix 分为两条版本线,调整的原因见[版本路线公告](https://
同一个发布页也提供 2.x 的 `reasonix` CLI 归档(`darwin|linux|windows × amd64|arm64`)。
安卓版在同一个发布页上,文件名是 `ReasonixMobile-<版本>.apk`(`arm64`,Android 8.0 及以上)。
这个 APK 用 Android 发布密钥签名,不走 minisign,因此只有一个 `.sha256`,没有 `.minisig`。
该密钥缺失时产物名为 `ReasonixMobile-<版本>-unsigned.apk`。Android 不接受未签名的包,
这类构建只留作 workflow 产物供自行签名,发布页上不会挂 APK。
内核与界面都跑在手机上,Agent 通过 SSH 连接你自己的服务器;与电脑配对时扫它显示的二维码。
### Reasonix 1.x(稳定版)
1.x CLI 在任意支持的平台上都可以通过 npm 安装,macOS 也可以使用 Homebrew:
+2 -2
View File
@@ -16,7 +16,7 @@ local.properties
keystore/
# Build inputs, produced by scripts/ before Gradle runs.
# The page sync also copies the repository license into the APK assets.
# jniLibs/ holds the cross-compiled kernel, assets/page/ the built
# frontend. assets/LICENSE is hand-placed and stays tracked.
app/src/main/jniLibs/
app/src/main/assets/page/
app/src/main/assets/LICENSE
+12 -45
View File
@@ -15,26 +15,6 @@ This file is for people building it. What a person installing it needs — what
does, how to pair with a computer, what it cannot do — belongs in the release
notes.
## Local host and mobile verification
The APK packages `cmd/reasonix-studio-host`, the same authenticated local host
used by Studio. Android reads its versioned launch handshake from stdout and
installs its HttpOnly session cookie before loading the WebView. Stderr remains
the diagnostic stream; launch credentials never enter a URL or a log.
The native container applies the maximum of system-bar, cutout and keyboard
insets to every page and overlay. The shared onboarding page has a dedicated layout
for narrow viewports. Bundled pages are refreshed using the content-hashed Vite
entry, including when installing another APK with the same version number.
To check the packaged page and real host without a phone, build
`go build -o reasonix-verify.exe ./cmd/reasonix-studio-host`, run
`android/scripts/sync-page.sh`, then `python android/scripts/verify-host.py`.
`KERNEL_BIN` can name another host executable. This checks authentication, origin
rejection, model discovery, credential saving and first-provider activation in
an isolated temporary home with a local test provider.
## One command
```bash
@@ -82,17 +62,19 @@ export PATH="$PATH:$(go env GOPATH)/bin"
| `build-kernel.sh` | Cross-compiles the kernel only (`GOOS=android GOARCH=arm64 CGO_ENABLED=0`) |
| `sync-page.sh` | Builds the frontend, copies `dist/` into assets, copies `LICENSE` |
| `verify-page.sh` | **No phone needed**: starts the real kernel against the packaged page |
| `verify-keyboard.mjs <adb-serial> [package]` | Checks keyboard resizing and composer visibility in an unlocked phone's foreground Debug WebView |
| `run.sh [debug\|release]` | `adb install`, launch, follow the log |
`verify-page.sh` runs the isolated local-host smoke check described above.
`verify-page.sh` is the one to run after a change: it starts the kernel with the
same arguments `Kernel.kt` uses (`--auth token`, `--token-file`, `--port-file`,
`-page`) and checks that it binds a port and answers 200. Without a device, that
is as deep as the chain can be verified.
## Where the code is
| File | Owns |
| --- | --- |
| `MainActivity.kt` | The WebView, the back gesture, downloads, external links, renderer restart |
| `Kernel.kt` | Unpacking the page, starting and stopping the kernel, reading the launch handshake, health probes, logs, the workspace directory |
| `Kernel.kt` | Unpacking the page, starting and stopping the kernel, waiting for the port, health probes, logs, the workspace directory |
| `KernelService.kt` | The foreground service that keeps a long turn alive |
| `NativeBridge.kt` | `@JavascriptInterface`: what the page may ask the phone for |
| `BrowserShellServer.kt` | Loopback HTTP server the kernel's browser tools drive the overlay WebView through |
@@ -101,29 +83,14 @@ export PATH="$PATH:$(go env GOPATH)/bin"
The page reaches the shell through `window.reasonixHost` and `ReasonixNative`,
defined in `desktop/frontend-next/src/port/host.ts` (`AndroidHost`).
**Phone and desktop share one page build.** The host port detects Android's
`ReasonixNative` bridge before the page loads. The browser-view bridge installed
at page completion is read when needed.
**Phone and desktop share one page build.** What differs only on a phone is
decided at runtime, not by a second set of components. `phoneShell()` in
`src/port/host.ts` and the `[data-shell="android"]` selector in `mobile.css` both
read `<html data-shell="android">`, which `index.html` sets before the bundle
loads.
Touch onboarding offers the current
workspace and a native folder picker; desktop keeps its drag-and-drop choice.
`FolderPicker.kt` owns the storage permission and directory-selection round trip.
Granting access resumes the picker. Cancelling or refusing it leaves the current
workspace available. Only readable, writable local storage folders become paths
for the kernel; document-provider and cloud URIs are not filesystem paths.
Folder selection and scanning use the existing asynchronous host-call channel.
Activity results settle the page's Promise without blocking the WebView thread.
Cancelling the camera keeps the connection card usable for another scan.
Android defaults to `/system/bin/sh` for local commands. This is the system
shell, not bundled Bash; Git, language runtimes and build tools are not included.
An explicit user shell configuration still takes precedence.
When command confinement is unavailable, enforced mode uses the existing
sandbox-escape confirmation. Refusing it or having no approver keeps the command
blocked; the sandbox setting is unchanged.
One attribute feeding both is what keeps style and components from disagreeing
about what a phone is.
## What the page may ask the phone for
-21
View File
@@ -20,25 +20,10 @@ plugins {
}
// One version, read by the About screen and the update check.
//
// A published build takes its version from the tag it was cut for, so the About
// screen and the update check name that release rather than the number this file
// was last edited to. Without the property — a local build — the numbers below
// are the answer.
def publishedVersion = findProperty("reasonixVersion") as String
def versionMajor = 2
def versionMinor = 0
def versionPatch = 0
def versionBuild = 1
if (publishedVersion) {
def parsed = publishedVersion =~ /^(\d+)\.(\d+)\.(\d+)/
if (!parsed.find()) {
throw new GradleException("reasonixVersion must start with MAJOR.MINOR.PATCH, got: ${publishedVersion}")
}
versionMajor = parsed.group(1) as int
versionMinor = parsed.group(2) as int
versionPatch = parsed.group(3) as int
}
// Where the two build inputs are expected. Both are produced from the checkout
// beside this module, by the scripts named in each message.
@@ -68,12 +53,6 @@ android {
// what `MainActivity` feeds real inset values for, since a WebView does
// not implement `env(safe-area-inset-*)`.
targetSdk 35
// versionCode must strictly increase, because Android refuses an update
// whose versionCode is not higher. The build digit is a single decimal
// place, so the scheme collides once a patch reaches 10: 2.55.10 and
// 2.56.0 both come out as 25601. Patch releases above 9 are not expected
// on Studio, and widening the digit now would renumber every build, so
// the constraint is recorded rather than fixed.
versionCode versionMajor * 10000 + versionMinor * 100 + versionPatch * 10 + versionBuild
versionName "${versionMajor}.${versionMinor}.${versionPatch}"
-1
View File
@@ -70,7 +70,6 @@
android:label="@string/app_name"
android:networkSecurityConfig="@xml/network_security_config"
android:supportsRtl="true"
android:requestLegacyExternalStorage="true"
android:theme="@style/Theme.Reasonix"
android:usesCleartextTraffic="false"
tools:targetApi="35">
@@ -1,107 +0,0 @@
package io.reasonix.mobile.app
import android.Manifest
import android.content.Intent
import android.content.pm.PackageManager
import android.net.Uri
import android.os.Build
import android.os.Environment
import android.provider.DocumentsContract
import android.provider.Settings
import android.widget.Toast
import androidx.activity.result.contract.ActivityResultContracts
import androidx.appcompat.app.AlertDialog
import androidx.appcompat.app.AppCompatActivity
import androidx.core.content.ContextCompat
import java.io.File
internal class FolderPicker(private val activity: AppCompatActivity, private val deliver: (String?) -> Unit) {
private var startIn = ""
private val tree = activity.registerForActivityResult(ActivityResultContracts.OpenDocumentTree()) { uri ->
val path = uri?.let(::realPath)
if (uri != null && path == null) message(R.string.workspace_unavailable)
deliver(path)
}
private val settings = activity.registerForActivityResult(ActivityResultContracts.StartActivityForResult()) {
afterPermission()
}
private val legacy = activity.registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) {
afterPermission()
}
fun hasAccess(): Boolean = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
Environment.isExternalStorageManager()
} else {
ContextCompat.checkSelfPermission(activity, Manifest.permission.WRITE_EXTERNAL_STORAGE) == PackageManager.PERMISSION_GRANTED
}
fun pick(path: String): Boolean {
startIn = path
if (hasAccess()) {
launchTree()
} else {
AlertDialog.Builder(activity)
.setTitle(R.string.wsperm_title)
.setMessage(R.string.wsperm_body)
.setPositiveButton(R.string.wsperm_go) { _, _ -> requestAccess() }
.setNegativeButton(R.string.wsperm_later) { _, _ -> deliver(null) }
.setOnCancelListener { deliver(null) }
.show()
}
return true
}
fun requestAccess() {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
legacy.launch(arrayOf(Manifest.permission.READ_EXTERNAL_STORAGE, Manifest.permission.WRITE_EXTERNAL_STORAGE))
return
}
runCatching {
settings.launch(Intent(Settings.ACTION_MANAGE_APP_ALL_FILES_ACCESS_PERMISSION, Uri.parse("package:${activity.packageName}")))
}.recoverCatching {
settings.launch(Intent(Settings.ACTION_MANAGE_ALL_FILES_ACCESS_PERMISSION))
}.onFailure {
message(R.string.workspace_permission_denied)
deliver(null)
}
}
private fun afterPermission() {
if (hasAccess()) launchTree() else {
message(R.string.workspace_permission_denied)
deliver(null)
}
}
private fun launchTree() {
val root = Environment.getExternalStorageDirectory().absolutePath
val relative = if (startIn.startsWith("$root/")) startIn.removePrefix("$root/") else "Documents"
val initial = DocumentsContract.buildDocumentUri(AUTHORITY, "primary:$relative")
runCatching { tree.launch(initial) }.onFailure {
message(R.string.workspace_unavailable)
deliver(null)
}
}
private fun message(resource: Int) = Toast.makeText(activity, resource, Toast.LENGTH_LONG).show()
private fun realPath(uri: Uri): String? = runCatching {
if (uri.authority != AUTHORITY) return@runCatching null
val document = DocumentsContract.getTreeDocumentId(uri)
val volume = document.substringBefore(':', "")
val relative = document.substringAfter(':', "")
if (!safeSegment(volume) || (relative.isNotEmpty() && !relative.split('/').all(::safeSegment))) return@runCatching null
val root = if (volume == "primary") Environment.getExternalStorageDirectory() else File("/storage", volume)
val target = File(root, relative).canonicalFile
val base = root.canonicalPath
if (target.path != base && !target.path.startsWith(base + File.separator)) return@runCatching null
target.path.takeIf { target.isDirectory && target.canRead() && target.canWrite() }
}.getOrNull()
private fun safeSegment(part: String): Boolean = part.isNotEmpty() && part != "." && part != ".." &&
part.none { it == '/' || it == '\\' || it == '\u0000' }
private companion object {
const val AUTHORITY = "com.android.externalstorage.documents"
}
}
@@ -1,34 +0,0 @@
package io.reasonix.mobile.app
import org.json.JSONObject
import java.net.URI
import java.util.concurrent.CompletableFuture
internal class HostConnection(val origin: String, val token: String) {
companion object {
fun read(process: Process): CompletableFuture<HostConnection> {
val ready = CompletableFuture<HostConnection>()
Thread({
try {
process.inputStream.bufferedReader().use { reader ->
val line = reader.readLine() ?: error("应用内核未返回启动信息")
val handshake = JSONObject(line)
val origin = handshake.getString("origin")
val token = handshake.getString("token")
val uri = URI(origin)
require(handshake.getInt("version") == 1 &&
uri.scheme == "http" && uri.host == "127.0.0.1" &&
uri.port in 1..65535 && uri.rawPath.orEmpty().isEmpty() &&
uri.rawUserInfo == null && uri.rawQuery == null && uri.rawFragment == null &&
token.matches(Regex("[0-9a-f]{64}"))) { "应用内核启动信息无效" }
ready.complete(HostConnection(origin, token))
while (reader.readLine() != null) { }
}
} catch (_: Exception) {
ready.completeExceptionally(IllegalStateException("应用内核未返回有效的本地连接"))
}
}, "kernel-handshake").apply { isDaemon = true }.start()
return ready
}
}
}
@@ -12,11 +12,27 @@ import java.io.File
import java.io.FileOutputStream
import java.net.HttpURLConnection
import java.net.URL
import java.security.MessageDigest
import java.security.SecureRandom
import java.util.Locale
import java.util.concurrent.TimeUnit
/** Owns the local Studio host, its private launch handshake, and its bundled page. */
/**
* The Reasonix kernel, as this app runs it: one child process serving loopback,
* plus the two things it needs on disk to start — the page it serves and a
* credential for this launch.
*
* The shape is the desktop host's (`cmd/reasonix-studio-host`), narrowed to what
* a phone does not have. That host binds a socket, mints a credential, speaks a
* handshake down a pipe and holds a lease; here the credential goes through
* `--token-file` and the bound address through `--port-file`, because a phone's
* app process has no pipe to hand a child and no reason to care which port was
* free. Everything after that — the kernel's own gate, its auth middleware, its
* CSRF check — is untouched, so the security of the surface is the one upstream
* tested, not one this shell invented.
*
* One instance lives in the app process. The activity asks it for a URL; the
* foreground service asks it to stay alive. Both see the same process, so there
* is nothing to synchronise beyond a lock on start/stop.
*/
class Kernel(private val context: Context) {
/** Where everything the kernel owns lives. Nothing here is backed up. */
@@ -53,6 +69,10 @@ class Kernel(private val context: Context) {
/** The built page, unpacked once per version. */
private val pageDir = File(context.filesDir, "page")
private val tokenFile = File(home, "remote/launch.token")
private val portFile = File(home, "remote/serve.port")
private val pidFile = File(home, "remote/serve.pid")
private val lock = Any()
@Volatile
@@ -77,12 +97,25 @@ class Kernel(private val context: Context) {
var origin: String? = null
private set
/** The host's per-launch credential, held only in memory. */
/**
* The launch credential, as the URL fragment the page trades for a cookie.
* Kept in memory only: the file on disk belongs to the kernel's own
* bookkeeping, and a phone's app data is not a place to leave a second copy.
*/
@Volatile
private var token: String? = null
/** Installed by the native shell before the WebView makes its first request. */
fun sessionCookie(): String? = token?.let { "reasonix_token=$it; Path=/; HttpOnly; SameSite=Strict" }
/**
* The URL the WebView loads first. It carries the credential in the
* fragment, which the page exchanges for an HttpOnly cookie before it makes
* any request — the same path a shared `reasonix web` link takes, so the
* token never enters a request line, a history entry or a log.
*/
fun entryUrl(): String? {
val origin = origin ?: return null
val token = token ?: return null
return "$origin/#token=$token"
}
/** The plain origin, for reloads once the cookie is set. */
fun plainUrl(): String? = origin
@@ -98,26 +131,31 @@ class Kernel(private val context: Context) {
*/
suspend fun start(): Result<String> = withContext(Dispatchers.IO) {
synchronized(lock) {
origin?.takeIf { process?.isAlive == true }?.let { return@withContext Result.success(it) }
var child: Process? = null
origin?.let { return@withContext Result.success(it) }
runCatching {
val binary = kernelBinary()
publishPage()
prepareHome()
val started = launch(binary)
child = started
val connection = HostConnection.read(started).get(START_TIMEOUT_MS, TimeUnit.MILLISECONDS)
awaitServing(started, connection)
token = connection.token
origin = connection.origin
process = started
connection.origin
}.onFailure {
child?.destroyForcibly()
origin = null
token = null
process = null
}
val launchToken = mintToken()
token = launchToken
val child = launch(binary, launchToken)
// The port is the kernel's answer, not this shell's guess. It
// writes the file after binding, so waiting on the file is
// waiting on the bind.
val bound = awaitPort(child)
val url = "http://$bound"
// A kernel that bound a port and then died would look healthy
// here and broken in the WebView; the two facts are checked
// together, and only a kernel that is answering is adopted.
awaitServing(child, url)
origin = url
process = child
url
}.onFailure { origin = null; process = null }
}
}
@@ -142,6 +180,8 @@ class Kernel(private val context: Context) {
p.destroyForcibly()
}
}
portFile.delete()
pidFile.delete()
}
}
@@ -165,13 +205,23 @@ class Kernel(private val context: Context) {
return binary
}
/** The entry HTML identifies Vite's content-hashed asset graph, including same-version APK rebuilds. */
/**
* Write the bundled page where the kernel can serve it.
*
* It goes to app storage rather than being served out of the APK because the
* kernel's `--page` is a directory on a real filesystem; there is no
* archive-aware variant, and adding one would be a change to the kernel for
* this shell's convenience alone.
*
* Unpacking is versioned: the marker holds the app's version name, so an
* update replaces the page and a relaunch does not. A partially written
* directory — the app killed mid-copy — is caught by the same marker never
* having been written, so the next launch rewrites it.
*/
private fun publishPage() {
val marker = File(pageDir, ".unpacked")
val entry = context.assets.open("page/index.html").use { it.readBytes() }
val digest = MessageDigest.getInstance("SHA-256").digest(entry).joinToString("") { "%02x".format(it) }
val stamp = "${BuildConfig.VERSION_NAME}:${BuildConfig.VERSION_CODE}:$digest"
if (marker.isFile && marker.readText().trim() == stamp && File(pageDir, "index.html").isFile) return
val stamp = "${BuildConfig.VERSION_NAME}:${BuildConfig.VERSION_CODE}"
if (marker.isFile && marker.readText().trim() == stamp) return
pageDir.deleteRecursively()
pageDir.mkdirs()
@@ -193,8 +243,12 @@ class Kernel(private val context: Context) {
private fun copyAssetTree(assets: AssetManager, from: String, to: File) {
val children = assets.list(from).orEmpty()
if (children.isEmpty()) {
assets.open(from).use { input ->
FileOutputStream(to).use { output -> input.copyTo(output) }
// A leaf: open it as a stream. A directory that happens to be empty
// is indistinguishable and harmless to skip.
runCatching {
assets.open(from).use { input ->
FileOutputStream(to).use { output -> input.copyTo(output) }
}
}
return
}
@@ -378,18 +432,69 @@ class Kernel(private val context: Context) {
/** The kernel's `config.WorkspaceSlug`, for the two paths that matter here. */
private fun slugOf(path: String): String = path.replace('/', '-').replace('\\', '-').replace(':', '-')
private fun launch(binary: File): Process {
/**
* The launch credential.
*
* Minted here rather than read from configuration: a credential a person can
* set is one anything that can read their files can present. 32 bytes, hex,
* the same shape the desktop host mints.
*/
private fun mintToken(): String {
val bytes = ByteArray(32)
SecureRandom().nextBytes(bytes)
val token = bytes.joinToString("") { "%02x".format(it) }
tokenFile.parentFile?.mkdirs()
tokenFile.writeText("$token\n")
// 0600 by intent, and the app's own private directory means it is
// already unreachable to every other app on the device.
runCatching {
tokenFile.setReadable(false, false)
tokenFile.setReadable(true, true)
tokenFile.setWritable(false, false)
tokenFile.setWritable(true, true)
}
return token
}
/**
* Start the child.
*
* `--addr 127.0.0.1:0` lets the kernel pick a free port; `--port-file`
* makes it announce which. `--auth token` is the mode a phone needs: the
* page authenticates, and a state-changing request without the credential
* is refused. `--no-open` is belt and braces — a serve never opens a
* browser, and a phone has no browser to open.
*/
private fun launch(binary: File, token: String): Process {
portFile.delete()
pidFile.delete()
// The folder the agent works in, resolved before the command line is
// built so it can be named on it. See the note under the environment
// block for why naming it is not the same as setting the directory.
val workspace = launchWorkspaceDir()
val command = listOf(
binary.absolutePath,
"serve",
"--addr", "127.0.0.1:0",
"--auth", "token",
"--token-file", tokenFile.absolutePath,
"--port-file", portFile.absolutePath,
"--pid-file", pidFile.absolutePath,
"--no-open",
"-page", pageDir.absolutePath,
"-studio-version", BuildConfig.VERSION_NAME,
// The project root, said outright. Without this the kernel derives
// it from its working directory, and on this phone the two did not
// agree: the sidebar named the shared folder, `write_roots` confined
// the file tools to the app's own directory, and every write into the
// folder the person was looking at came back
// `workspace.write_outside_scope`. A flag is a fact the kernel was
// told rather than one it inferred from a mount it sees differently.
"--dir", workspace.absolutePath,
)
val builder = ProcessBuilder(command)
builder.environment().apply {
put("REASONIX_HOME", home.absolutePath)
put("HOME", home.absolutePath)
put("SHELL", "/system/bin/sh")
// The kernel's own language, for its tool failures, approval
// prompts and error sentences. Without this it detects from the
// POSIX locale variables, and Android sets none of them for an
@@ -468,6 +573,7 @@ class Kernel(private val context: Context) {
// derives `write_roots` from that root, so the two spellings have to be
// the same one.
builder.environment().put("PWD", workspace.absolutePath)
builder.redirectErrorStream(true)
val process = builder.start()
// The kernel's own log is the only place a startup failure says why, and
@@ -564,7 +670,7 @@ class Kernel(private val context: Context) {
private fun drain(process: Process) {
runCatching {
process.errorStream.bufferedReader().useLines { lines ->
process.inputStream.bufferedReader().useLines { lines ->
lines.forEach { line ->
Log.i(TAG, line)
synchronized(logLines) {
@@ -579,6 +685,21 @@ class Kernel(private val context: Context) {
/** The kernel's tail, newest last, for a bug report or the About screen. */
fun recentLog(): String = synchronized(logLines) { logLines.joinToString("\n") }
private fun awaitPort(process: Process): String {
val deadline = System.currentTimeMillis() + START_TIMEOUT_MS
while (System.currentTimeMillis() < deadline) {
if (!process.isAlive) {
throw IllegalStateException("内核启动即退出。\n${recentLog().takeLast(2000)}")
}
val text = if (portFile.isFile) runCatching { portFile.readText().trim() }.getOrNull() else null
// The file is written by the kernel on the way to serving, and the
// kernel rewrites it with the real port when it was given :0.
if (!text.isNullOrBlank() && text.contains(":")) return text
Thread.sleep(POLL_MS)
}
throw IllegalStateException("内核在 ${START_TIMEOUT_MS / 1000}s 内没有报告监听地址。\n${recentLog().takeLast(2000)}")
}
/**
* Wait until the bound port answers.
*
@@ -586,24 +707,23 @@ class Kernel(private val context: Context) {
* kernel's life; a WebView pointed at it would show a connection error and
* cache the failure, so the wait belongs here rather than in the page.
*/
private fun awaitServing(process: Process, connection: HostConnection) {
private fun awaitServing(process: Process, origin: String) {
val deadline = System.currentTimeMillis() + START_TIMEOUT_MS
while (System.currentTimeMillis() < deadline) {
if (!process.isAlive) {
throw IllegalStateException("内核在开始服务前退出。\n${recentLog().takeLast(2000)}")
}
if (probe("${connection.origin}/", connection.token)) return
if (probe("$origin/")) return
Thread.sleep(POLL_MS)
}
throw IllegalStateException("内核已监听 ${connection.origin},但页面没有响应。\n${recentLog().takeLast(2000)}")
throw IllegalStateException("内核已监听 $origin,但页面没有响应。\n${recentLog().takeLast(2000)}")
}
private fun probe(url: String, launchToken: String): Boolean = runCatching {
private fun probe(url: String): Boolean = runCatching {
(URL(url).openConnection() as HttpURLConnection).run {
connectTimeout = 1000
readTimeout = 1000
requestMethod = "GET"
setRequestProperty("Cookie", "reasonix_token=$launchToken")
try {
responseCode in 200..399
} finally {
@@ -9,7 +9,9 @@ import android.graphics.Canvas
import android.net.Uri
import android.os.Build
import android.os.Bundle
import android.os.Environment
import android.os.PowerManager
import android.provider.DocumentsContract
import android.provider.Settings
import android.util.Base64
import android.util.Log
@@ -32,7 +34,6 @@ import androidx.activity.OnBackPressedCallback
import androidx.activity.result.contract.ActivityResultContracts
import androidx.appcompat.app.AppCompatActivity
import androidx.core.view.ViewCompat
import androidx.core.view.WindowCompat
import androidx.core.view.WindowInsetsCompat
import androidx.lifecycle.lifecycleScope
import kotlinx.coroutines.launch
@@ -192,7 +193,6 @@ class MainActivity : AppCompatActivity() {
@SuppressLint("SetJavaScriptEnabled")
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
WindowCompat.setDecorFitsSystemWindows(window, false)
rootLayout = FrameLayout(this)
web = WebView(this)
@@ -206,7 +206,6 @@ class MainActivity : AppCompatActivity() {
FrameLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT),
)
setContentView(rootLayout)
syncWindowTheme("light")
configureWebView()
// Keep the renderer alive while the app is off screen.
@@ -259,7 +258,7 @@ class MainActivity : AppCompatActivity() {
// `env(safe-area-inset-*)` is an iOS Safari feature and a WebView answers
// it with 0, so without this the interface starts under the status bar.
installInsetsListener()
ViewCompat.requestApplyInsets(rootLayout)
ViewCompat.requestApplyInsets(web)
// The kernel runs under the service, so it survives the screen going off
// and a task that takes minutes finishes. Starting it here as well is
@@ -387,11 +386,80 @@ class MainActivity : AppCompatActivity() {
return picked.takeIf { it.isNotEmpty() }?.toTypedArray()
}
private val nativeResults = mutableMapOf<String, String>()
private val folders = FolderPicker(this) { path -> finishNativePicker("pickFolder", path) }
/**
* A folder on this phone, as a workspace.
*
* `OpenDocumentTree` is the only system picker for a directory, and it
* answers with a tree URI — not the path the kernel opens. The translation
* below is the one that makes it usable: the tree's document id names the
* volume and the relative folder, and with every-file access granted those
* map onto a real directory the kernel can `stat`.
*
* That mapping is why the permission is asked for first. Without it the path
* is real but unreadable, and the kernel reports an empty or missing folder
* — a failure a long way from its cause.
*/
private val pickFolderTree = registerForActivityResult(ActivityResultContracts.OpenDocumentTree()) { uri ->
bridge.deliverFolder(uri?.let(::realPathFromTree))
}
private fun finishNativePicker(method: String, value: String?) {
nativeResults.remove(method)?.let { replyHost(it, JSONObject.quote(value.orEmpty())) }
/**
* A tree URI as the filesystem path a child process can open, or null when
* the volume is one this mapping does not cover (a cloud or MTP provider).
*
* The path is checked before it is handed over, because the mapping is a
* guess about a naming convention rather than a lookup: the document id is
* the volume and the folder *below* it, and this turns that into a string.
* A tree whose id carries `..` would compose into a path outside the folder
* the person picked — and the kernel, once given the all-files permission,
* would open it.
*
* Every part of the id is checked, and the volume is the part that is easy
* to miss: a document id of `..:data` composes to `/storage/../data`, which
* `canonicalFile` collapses to `/data` and `isDirectory` then accepts, since
* it really is a directory. Rejecting only the folder half left exactly the
* traversal this function exists to refuse.
*
* The prefix check at the end is the belt to that braces: whatever the parts
* spelled, the answer has to still be the folder that was asked for.
*/
private fun realPathFromTree(uri: Uri): String? = runCatching {
val docId = DocumentsContract.getTreeDocumentId(uri)
val type = docId.substringBefore(':', "")
val rel = docId.substringAfter(':', "")
if (!isSafeSegment(type) || !isSafeRelPath(rel)) return@runCatching null
val base = when (type) {
"primary" -> Environment.getExternalStorageDirectory().absolutePath
// A removable volume: its document id is the volume's own name
// (`1234-ABCD`), which is the directory under `/storage`.
else -> File("/storage", type).path
}
val target = if (rel.isEmpty()) base else "$base/$rel"
val dir = File(target).canonicalFile
if (!dir.isDirectory) return@runCatching null
// Still under the volume it claimed to be on. A path that resolved
// somewhere else is not the folder that was picked, whatever it is.
val root = File(base).canonicalFile
val canonical = dir.canonicalPath
if (canonical != root.canonicalPath && !canonical.startsWith(root.canonicalPath + File.separator)) {
return@runCatching null
}
canonical
}.getOrNull()
/** One path component from a document id: a name, never a direction. `.`
* and `..` are refused because they are the two spellings of "not a name"
* that a filesystem still resolves. */
private fun isSafeSegment(part: String): Boolean {
if (part.isEmpty() || part == "." || part == "..") return false
return !part.contains('/') && !part.contains('\\') && !part.contains('\u0000')
}
/** The folder part of a document id: a chain of safe components, each of
* which is checked, so a `..` cannot hide behind a later segment. */
private fun isSafeRelPath(rel: String): Boolean {
if (rel.isEmpty()) return true
return rel.split('/').all { isSafeSegment(it) }
}
/**
@@ -412,7 +480,7 @@ class MainActivity : AppCompatActivity() {
// the person's confirmation that that origin may become the window, and
// it is spent by the one `openInApp` that follows.
confirmedPairOrigin = text?.let { originOf(Uri.parse(it.trim())) }
finishNativePicker("scanPairingCode", text)
bridge.deliverScan(text)
}
/**
@@ -443,7 +511,7 @@ class MainActivity : AppCompatActivity() {
pickIdentity.launch(arrayOf("*/*"))
true
}
bridge.onInsets = { Insets(0, 0, 0, 0) }
bridge.onInsets = { currentInsets }
bridge.askForSave = { name, content ->
pendingSave = name to content
runCatching { saveTextFile.launch(name) }
@@ -452,33 +520,110 @@ class MainActivity : AppCompatActivity() {
.isSuccess
}
bridge.askForShare = { name, content -> shareText(name, content) }
bridge.askForFolder = { startIn ->
// The picker opens at its own last location; `startIn` is a hint it
// may ignore, so a failure to use it is not a failure to ask.
pickFolderTree.launch(null)
true
}
bridge.onRunning = { running -> runOnUiThread { applyRunning(running) } }
bridge.filesAccess = folders::hasAccess
bridge.askForFilesAccess = { runOnUiThread { folders.requestAccess() } }
bridge.filesAccess = { hasAllFilesAccess() }
bridge.askForFilesAccess = { openAllFilesSettings() }
bridge.batteryFree = { batteryUnrestricted() }
bridge.askForBattery = { requestBatteryExemption() }
bridge.askForScan = {
// The camera is live from the moment this opens, so the ask is the
// launch itself; there is no second step to report.
scanCode.launch(Intent(this, ScanActivity::class.java))
true
}
bridge.onOpenInApp = { url -> openInApp(url) }
bridge.onHostCall = { id, method, args -> handleHostCall(id, method, args) }
target.addJavascriptInterface(bridge, "ReasonixNative")
}
/**
* The window's real insets, in CSS pixels, recomputed on every change.
*
* Read from the WebView's own window insets rather than from the decor view,
* and every value is the max of the system bars and the display cutout: a
* phone with a transparent status bar and a hole-punch camera reports a small
* or zero bar inset and a real cutout inset, and padding by only the former
* leaves the interface under the camera.
*/
private var currentInsets = Insets(0, 0, 0, 0)
private fun installInsetsListener() {
ViewCompat.setOnApplyWindowInsetsListener(rootLayout) { view, insets ->
val safe = insets.getInsets(
WindowInsetsCompat.Type.systemBars() or WindowInsetsCompat.Type.displayCutout() or WindowInsetsCompat.Type.ime(),
ViewCompat.setOnApplyWindowInsetsListener(web) { _, insets ->
val density = resources.displayMetrics.density.takeIf { it > 0f } ?: 1f
fun px(value: Int) = Math.round(value / density)
val system = insets.getInsets(WindowInsetsCompat.Type.systemBars())
val cutout = insets.getInsets(WindowInsetsCompat.Type.displayCutout())
val values = Insets(
left = px(maxOf(system.left, cutout.left)),
top = px(maxOf(system.top, cutout.top)),
right = px(maxOf(system.right, cutout.right)),
bottom = px(maxOf(system.bottom, cutout.bottom)),
)
view.setPadding(safe.left, safe.top, safe.right, safe.bottom)
WindowInsetsCompat.CONSUMED
currentInsets = values
pushInsets(values)
// The keyboard rides its own channel. It is not a system bar: it
// comes and goes with focus, and only the composer rises above it.
//
// `isVisible` gates the number. An IME inset of zero is not the
// same as "no keyboard" — a device can report the navigation bar's
// height on this type while the keyboard is down, and feeding that
// to the page would lift the composer by a keyboard nobody opened.
val ime = if (insets.isVisible(WindowInsetsCompat.Type.ime())) {
px(insets.getInsets(WindowInsetsCompat.Type.ime()).bottom)
} else {
0
}
pushIme(ime)
insets
}
}
private fun syncWindowTheme(scheme: String) {
val light = scheme != "dark"
WindowCompat.getInsetsController(window, web).apply {
isAppearanceLightStatusBars = light
isAppearanceLightNavigationBars = light
}
rootLayout.setBackgroundColor(getColor(if (light) R.color.page else R.color.page_dark))
/**
* Hand the page the insets it cannot compute for itself.
*
* A WebView answers `env(safe-area-inset-*)` with `0` on every Android
* version — those are an iOS Safari feature — so `mobile.css`'s padding,
* which is written against them, does nothing here and the first row of the
* interface sits behind the status bar. Android 15 makes that unconditional
* by forcing the window edge-to-edge whatever the app asks for, so the
* numbers have to be pushed in rather than computed in CSS.
*
* Pushed on every inset change, not once: the values move when the phone is
* rotated, when a foldable is unfolded, and when the keyboard opens.
*/
private fun pushInsets(values: Insets) {
web.evaluateJavascript(
"window.__reasonixInsets && window.__reasonixInsets(${values.left},${values.top},${values.right},${values.bottom});",
null,
)
}
/**
* Hand the page the keyboard's height, which it cannot reliably measure.
*
* The page reads `visualViewport` for this, and that works only while the
* window stays edge to edge: Android 15 forces that from targetSdk 35, but a
* WebView told to `adjustResize` shrinks the window instead, and then the
* visual and layout viewports are equal and the difference the page
* computes is zero while the keyboard is up. The window insets answer in
* exactly that case, so the two sources are both sent and the page takes
* the larger — whichever arrangement the platform picked, the number is the
* keyboard's real height.
*
* Pushed on every inset change, including the one that reports the keyboard
* going away, so the composer comes back down with it.
*/
private fun pushIme(bottom: Int) {
web.evaluateJavascript(
"window.__reasonixIme && window.__reasonixIme($bottom);",
null,
)
}
private fun installClients() {
@@ -643,8 +788,6 @@ class MainActivity : AppCompatActivity() {
// the old object was reachable only from the destroyed window.
installBridge(fresh)
installClients()
installInsetsListener()
ViewCompat.requestApplyInsets(rootLayout)
splash.alpha = 1f
splash.visibility = View.VISIBLE
@@ -655,11 +798,10 @@ class MainActivity : AppCompatActivity() {
private fun load(origin: String) {
installClients()
val cookie = (application as App).kernel.sessionCookie() ?: return
android.webkit.CookieManager.getInstance().setCookie(origin, cookie) { accepted ->
if (accepted) web.loadUrl(origin)
else showFailure(IllegalStateException("无法建立应用内核的本地会话"))
}
// The fragment path: the page exchanges the token for an HttpOnly cookie
// before its first request, exactly as a shared `reasonix web` link does.
val entry = (application as App).kernel.entryUrl() ?: origin
web.loadUrl(entry)
}
/**
@@ -768,6 +910,31 @@ class MainActivity : AppCompatActivity() {
web.keepScreenOn = running
}
/** Whether this app may read every file. Below API 30 the concept does not
* exist and the answer is yes. */
private fun hasAllFilesAccess(): Boolean =
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) Environment.isExternalStorageManager() else true
/**
* The one place Android offers the every-file permission.
*
* There is no runtime dialog for it — it is a special access, granted by
* hand per app — so the only honest implementation is to open the page that
* holds the switch. A person who does not find it there loses nothing: the
* app keeps working on the folders it already has.
*/
private fun openAllFilesSettings() {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return
val intent = Intent(
Settings.ACTION_MANAGE_APP_ALL_FILES_ACCESS_PERMISSION,
Uri.parse("package:$packageName"),
)
runCatching { startActivity(intent) }.recoverCatching {
// Some ROMs do not route the per-app page; the list is the fallback.
startActivity(Intent(Settings.ACTION_MANAGE_ALL_FILES_ACCESS_PERMISSION))
}
}
/** Whether this app is exempt from the system's battery optimisation. */
private fun batteryUnrestricted(): Boolean {
val power = getSystemService(POWER_SERVICE) as? PowerManager ?: return true
@@ -998,24 +1165,6 @@ class MainActivity : AppCompatActivity() {
runOnUiThread {
try {
when (method) {
"syncTheme" -> {
syncWindowTheme(JSONArray(args).optString(0, "light"))
replyHost(id, "null")
}
"pickFolder", "scanPairingCode" -> {
if (nativeResults.isNotEmpty()) {
replyHost(id, JSONObject.quote(""))
} else {
nativeResults[method] = id
runCatching {
if (method == "pickFolder") folders.pick(JSONArray(args).optString(0, ""))
else {
confirmedPairOrigin = null
scanCode.launch(Intent(this, ScanActivity::class.java))
}
}.onFailure { finishNativePicker(method, null) }
}
}
"showBrowserView" -> {
val arr = JSONArray(args)
val rect = if (arr.length() > 1) arr.optJSONObject(1) else null
@@ -1234,7 +1383,7 @@ class MainActivity : AppCompatActivity() {
};
function call(method, args) {
return new Promise(function(resolve, reject) {
var id = Date.now() + ':' + String(nextId++);
var id = String(nextId++);
pending[id] = { resolve: resolve, reject: reject };
try {
ReasonixNative.hostCall(id, method, JSON.stringify(args || []));
@@ -1248,9 +1397,6 @@ class MainActivity : AppCompatActivity() {
shell: 'electron',
platform: 'android',
titleBar: false,
syncTheme: function(mode, scheme) { return call('syncTheme', [scheme]); },
pickFolder: function(startIn) { return call('pickFolder', [startIn]); },
scanPairingCode: function() { return call('scanPairingCode', []); },
showBrowserView: function(target, rect) { return call('showBrowserView', [target, rect]); },
hideBrowserView: function() { return call('hideBrowserView', []); },
freezeBrowserView: function() { return call('freezeBrowserView', []); },
@@ -1273,7 +1419,6 @@ class MainActivity : AppCompatActivity() {
answerBrowserLogin: function(id, username, password) { return call('answerBrowserLogin', [id, username, password]); },
trustBrowserCertificate: function(target) { return call('trustBrowserCertificate', [target]); }
};
window.dispatchEvent(new Event('reasonix-host-ready'));
})();
""".trimIndent(), null)
}
@@ -261,6 +261,21 @@ class NativeBridge(
return "${insets.left},${insets.top},${insets.right},${insets.bottom}"
}
/**
* Read the pairing code a computer is showing.
*
* A phone cannot type what a computer renders as a QR code — it is a
* one-time code inside a URL, and the whole reason it is a picture is that
* nobody reads one aloud. So the page asks for the camera and gets the text
* back, which is the same direction the computer's own dialog describes.
*
* The answer is the code's own text, not a parsed result: what a pairing
* fragment means is the page's to decide, and it already knows how — the
* `#pair=` handling ships with the interface.
*/
@JavascriptInterface
fun scanPairingCode(): String = if (!trusted()) "" else scanWithCamera()
/**
* Load a URL in this window instead of handing it to the browser.
*
@@ -340,6 +355,11 @@ class NativeBridge(
private val pending = AtomicReference<Uri?>(null)
private val latch = AtomicReference<CountDownLatch?>(null)
/** The code the camera read, held between the activity's answer and the
* wait below — the same round trip the file picker takes. */
private val scanned = AtomicReference<String?>(null)
private val scanLatch = AtomicReference<CountDownLatch?>(null)
/** The main thread, because starting an Activity result must happen there. */
private val main = Handler(Looper.getMainLooper())
@@ -358,6 +378,13 @@ class NativeBridge(
*/
var onInsets: (() -> Insets?)? = null
/**
* How to open the camera. Set by the activity, which owns the Activity
* Result API; like `askForFile` it answers whether the ask was dispatched,
* and the code itself arrives through `deliverScan`.
*/
var askForScan: () -> Boolean = { false }
/**
* How to put a file where the person picks. Set by the activity, which owns
* the Activity Result API; it answers whether the ask was dispatched, and
@@ -415,6 +442,51 @@ class NativeBridge(
saveLatch.get()?.countDown()
}
/**
* Read the pairing code a computer is showing, and return it.
*
* The same shape as `pickFile` and for the same reason: a
* `@JavascriptInterface` method is called on the WebView's thread and must
* return a value, so the camera is opened on the main thread and this one
* waits. The block lasts as long as the person is aiming, which is why the
* timeout is generous — a scan that gave up after a few seconds would be
* one nobody could complete.
*
* An empty string is "no code", which is what a refusal, a cancellation and
* a timeout all are. The caller acts on the text, so a distinct error would
* only be a fourth way to say the same thing.
*/
private fun scanWithCamera(): String {
val gate = CountDownLatch(1)
scanLatch.set(gate)
scanned.set(null)
val posted = CountDownLatch(1)
var dispatched = false
main.post {
dispatched = runCatching { askForScan() }.getOrDefault(false)
posted.countDown()
}
val ready = runCatching { posted.await(SCAN_TIMEOUT_SECONDS, TimeUnit.SECONDS) }.getOrDefault(false)
if (!ready || !dispatched) {
scanLatch.set(null)
return ""
}
return try {
if (!gate.await(SCAN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) "" else scanned.get().orEmpty()
} catch (_: InterruptedException) {
""
} finally {
scanLatch.set(null)
}
}
/** Called by the activity when the scanner answers. Null is a dismissal. */
fun deliverScan(text: String?) {
scanned.set(text)
scanLatch.get()?.countDown()
}
/**
* Ask for one file and wait for the answer.
*
@@ -598,10 +670,63 @@ class NativeBridge(
var askForBattery: (() -> Unit)? = null
/**
* Ask for a folder on this phone and answer with its real filesystem path.
*
* "" is a dismissal, the same answer `importIntoHome` gives — and, unlike
* that one, an empty string is a normal outcome here rather than a failure.
* The `startIn` hint is a path the kernel knows; the picker may ignore it,
* which is the picker's business.
*
* The round trip is the same latch as `pickFile`, for the same reason: a
* document-tree picker is another activity, and this call cannot suspend.
*/
@JavascriptInterface
fun pickFolder(startIn: String): String {
if (!trusted()) return ""
val gate = CountDownLatch(1)
folderLatch.set(gate)
folderPending.set(null)
val posted = CountDownLatch(1)
var dispatched = false
main.post {
dispatched = runCatching { askForFolder(startIn) }.getOrDefault(false)
posted.countDown()
}
val ready = runCatching { posted.await(PICK_TIMEOUT_SECONDS, TimeUnit.SECONDS) }.getOrDefault(false)
if (!ready || !dispatched) {
folderLatch.set(null)
return ""
}
return try {
if (!gate.await(PICK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) "" else folderPending.get().orEmpty()
} catch (_: InterruptedException) {
""
} finally {
folderLatch.set(null)
}
}
var askForFolder: (String) -> Boolean = { false }
private val folderPending = AtomicReference<String?>(null)
private val folderLatch = AtomicReference<CountDownLatch?>(null)
/** Called by the activity with the folder's real path, or null on dismissal. */
fun deliverFolder(path: String?) {
folderPending.set(path ?: "")
folderLatch.get()?.countDown()
}
companion object {
const val BRIDGE_VERSION = "5"
const val BRIDGE_VERSION = "4"
private const val PICK_TIMEOUT_SECONDS = 5L * 60
/** Long, because this is however long someone takes to aim. A shorter
* one would turn "still lining it up" into a failed scan. */
private const val SCAN_TIMEOUT_SECONDS = 5L * 60
/** Only as long as the sheet takes to appear: the answer is "something
* took it", which is known the moment the Intent resolves. */
private const val SHARE_TIMEOUT_SECONDS = 15L
@@ -8,8 +8,6 @@ import android.view.Gravity
import android.view.ViewGroup
import android.widget.FrameLayout
import android.widget.TextView
import android.widget.Button
import android.widget.Toast
import androidx.activity.result.contract.ActivityResultContracts
import androidx.appcompat.app.AppCompatActivity
import androidx.camera.core.CameraSelector
@@ -19,8 +17,6 @@ import androidx.camera.core.Preview
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.view.PreviewView
import androidx.core.content.ContextCompat
import androidx.core.view.ViewCompat
import androidx.core.view.WindowInsetsCompat
import java.util.concurrent.Executors
/**
@@ -86,16 +82,6 @@ class ScanActivity : AppCompatActivity() {
}
root.addView(preview)
root.addView(hint)
root.addView(Button(this).apply {
text = getString(R.string.scan_close)
setOnClickListener { setResult(RESULT_CANCELED); finish() }
layoutParams = FrameLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT, Gravity.TOP or Gravity.START)
})
ViewCompat.setOnApplyWindowInsetsListener(root) { view, insets ->
val bars = insets.getInsets(WindowInsetsCompat.Type.systemBars() or WindowInsetsCompat.Type.displayCutout())
view.setPadding(bars.left, bars.top, bars.right, bars.bottom)
insets
}
setContentView(root)
if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) ==
@@ -107,8 +93,10 @@ class ScanActivity : AppCompatActivity() {
}
}
private fun refuse(message: Int = R.string.scan_denied) {
Toast.makeText(this, message, Toast.LENGTH_LONG).show()
private fun refuse() {
// No dialog of our own: the permission prompt already said no, and a
// second window repeating it is one more tap to dismiss. The caller
// gets the same answer as a cancelled scan.
setResult(RESULT_CANCELED)
finish()
}
@@ -116,7 +104,7 @@ class ScanActivity : AppCompatActivity() {
private fun start() {
val providerFuture = ProcessCameraProvider.getInstance(this)
providerFuture.addListener({
val provider = runCatching { providerFuture.get() }.getOrNull() ?: return@addListener refuse(R.string.scan_unavailable)
val provider = runCatching { providerFuture.get() }.getOrNull() ?: return@addListener refuse()
val previewUseCase = Preview.Builder().build().also { it.setSurfaceProvider(preview.surfaceProvider) }
val analysis = ImageAnalysis.Builder()
// Only the newest frame matters: a stale one describes where the
@@ -128,7 +116,7 @@ class ScanActivity : AppCompatActivity() {
runCatching {
provider.unbindAll()
provider.bindToLifecycle(this, CameraSelector.DEFAULT_BACK_CAMERA, previewUseCase, analysis)
}.onFailure { refuse(R.string.scan_unavailable) }
}.onFailure { refuse() }
}, ContextCompat.getMainExecutor(this))
}
@@ -138,10 +126,10 @@ class ScanActivity : AppCompatActivity() {
val text = Qr.decode(image) ?: return
if (settled) return
settled = true
runOnUiThread {
setResult(RESULT_OK, Intent().putExtra(EXTRA_TEXT, text))
finish()
}
setResult(RESULT_OK, Intent().putExtra(EXTRA_TEXT, text))
// Finishing from the analysis thread is the whole reason `settled`
// exists; the activity teardown has to happen on the main one.
runOnUiThread { finish() }
} finally {
image.close()
}
@@ -5,6 +5,5 @@
rather than the platform default white. -->
<resources>
<color name="page">#F4F1EA</color>
<color name="page_dark">#171A18</color>
<color name="boot">#07080C</color>
</resources>
+1 -5
View File
@@ -23,17 +23,13 @@
<string name="splash_retry">重试</string>
<string name="wsperm_title">需要「所有文件访问」权限</string>
<string name="wsperm_body">要直接读写手机公共文件夹,需要开启「所有文件访问」权限。点「去设置」授权后返回,会继续选择文件夹。使用应用默认工作区不需要此权限。</string>
<string name="wsperm_body">要打开手机上的文件夹作为工作区,需要先允许本应用访问所有文件。点「去设置」,在列表里打开 Reasonix 的开关,然后返回这里。</string>
<string name="wsperm_go">去设置</string>
<string name="wsperm_later">以后再说</string>
<string name="workspace_permission_denied">未开启文件访问权限,仍可使用应用默认工作区。</string>
<string name="workspace_unavailable">无法读写这个位置,请选择手机本地存储中的文件夹。</string>
<string name="scan_title">扫描电脑上的二维码</string>
<string name="scan_hint">对准电脑上「设备访问」里的二维码,识别后即可连接这台电脑。</string>
<string name="scan_button">扫码连接电脑</string>
<string name="scan_denied">没有相机权限,无法扫码。请在系统设置里允许。</string>
<string name="scan_close">返回</string>
<string name="scan_unavailable">相机暂时不可用,请关闭其他占用相机的应用后重试。</string>
<string name="share_title">分享文件</string>
</resources>
File diff suppressed because it is too large. Load diff
+4 -2
View File
@@ -47,16 +47,18 @@ app_gradle="$android/app/build.gradle"
version_field() {
sed -n "s/^def $1 *= *\([0-9]*\).*/\1/p" "$app_gradle" | head -n1
}
app_version="${REASONIX_APP_VERSION:-$(version_field versionMajor).$(version_field versionMinor).$(version_field versionPatch)}"
app_version="$(version_field versionMajor).$(version_field versionMinor).$(version_field versionPatch)"
build_time_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
ldflags="-s -w"
ldflags="$ldflags -X main.version=v$app_version"
ldflags="$ldflags -X main.buildTimeUTC=$build_time_utc"
ldflags="$ldflags -X reasonix/internal/tools/productdocs.linkedVersion=v$app_version"
ldflags="$ldflags -X reasonix/internal/tools/productdocs.linkedRevision=apk-v$app_version"
echo "building the kernel (android/arm64, v$app_version) ..."
mkdir -p "$(dirname "$out")"
cd "$upstream"
go build -trimpath -ldflags="$ldflags" -o "$out" ./cmd/reasonix-studio-host
go build -trimpath -ldflags="$ldflags" -o "$out" ./cmd/reasonix
size=$(wc -c < "$out")
echo "kernel ready: $out ($((size / 1024 / 1024)) MiB)"
-95
View File
@@ -1,95 +0,0 @@
import http.server
import json
import os
from pathlib import Path
import queue
import subprocess
import tempfile
import threading
import urllib.error
import urllib.parse
import urllib.request
class Provider(http.server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(b'{"data":[{"id":"smoke-model","object":"model"}]}')
def log_message(self, *_args):
pass
android = Path(__file__).resolve().parents[1]
kernel = Path(os.environ.get("KERNEL_BIN", android.parent / "reasonix-verify.exe")).resolve()
page = android / "app/src/main/assets/page"
if not kernel.is_file() or not (page / "index.html").is_file():
raise SystemExit("Build ./cmd/reasonix-studio-host as KERNEL_BIN and run sync-page.sh first.")
with tempfile.TemporaryDirectory(prefix="reasonix-android-host-") as temporary:
home = Path(temporary) / "home"
workspace = Path(temporary) / "workspace"
home.mkdir()
workspace.mkdir()
(home / "config.toml").write_text("providers = []\n", encoding="utf-8")
env = {**os.environ, "REASONIX_HOME": str(home), "REASONIX_STATE_HOME": str(home),
"REASONIX_CREDENTIALS_STORE": "file", "HOME": str(home)}
logs = open(Path(temporary) / "host.log", "w", encoding="utf-8")
child = subprocess.Popen([str(kernel), "-page", str(page)], cwd=workspace, env=env,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=logs, text=True)
provider = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Provider)
threading.Thread(target=provider.serve_forever, daemon=True).start()
try:
announced = queue.Queue()
threading.Thread(target=lambda: announced.put(child.stdout.readline()), daemon=True).start()
connection = json.loads(announced.get(timeout=30))
origin = connection["origin"]
address = urllib.parse.urlsplit(origin)
assert connection["version"] == 1 and address.hostname == "127.0.0.1" and address.scheme == "http"
def request(path, body=None, authenticated=True, request_origin=None):
headers = {"Content-Type": "application/json", "Origin": request_origin or origin}
if authenticated:
headers["Cookie"] = "reasonix_token=" + connection["token"]
payload = None if body is None else json.dumps(body).encode()
req = urllib.request.Request(origin + path, data=payload, headers=headers)
try:
response = urllib.request.urlopen(req, timeout=30)
except urllib.error.HTTPError as error:
response = error
with response:
raw = response.read()
return response.status, json.loads(raw) if "json" in response.headers.get("Content-Type", "") else raw
assert request("/runtimes", authenticated=False)[0] == 403
assert request("/runtimes", request_origin="https://untrusted.invalid")[0] == 403
assert request("/")[0] == 200
status, runtimes = request("/runtimes")
assert status == 200
entries = runtimes if isinstance(runtimes, list) else runtimes["runtimes"]
route = "/rt/" + entries[0]["id"]
assert request(route + "/provider-setup")[1]["required"]
endpoint = "http://127.0.0.1:" + str(provider.server_port)
status, probe = request(route + "/providers/probe", {"baseUrl": endpoint, "apiKey": "sk-smoke"})
assert status == 200 and "smoke-model" in probe["models"], (status, probe)
status, saved = request(route + "/providers", {
"name": "smoke", "kind": "openai", "baseUrl": endpoint, "apiKey": "sk-smoke",
"models": ["smoke-model"], "default": "smoke-model",
})
assert status == 200, (status, saved)
status, setup = request(route + "/provider-setup")
assert status == 404 or not setup.get("required"), (status, setup)
print("Android host: page, authentication, origin guard, provider probe, save and activation passed.")
finally:
child.stdin.close()
try:
child.wait(timeout=10)
except subprocess.TimeoutExpired:
child.kill()
child.wait()
child.stdout.close()
logs.close()
provider.shutdown()
provider.server_close()
-72
View File
@@ -1,72 +0,0 @@
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import path from "node:path";
const serial = process.argv[2];
assert(serial, "Usage: node android/scripts/verify-keyboard.mjs <adb-serial> [package]");
const pkg = process.argv[3] || "io.reasonix.mobile.app.debug";
const sdk = process.env.ANDROID_HOME || process.env.ANDROID_SDK_ROOT;
const binary = process.platform === "win32" ? "adb.exe" : "adb";
const adb = sdk ? path.join(sdk, "platform-tools", binary) : binary;
const run = (...args) => execFileSync(adb, ["-s", serial, ...args], { encoding: "utf8", timeout: 15000 });
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const activity = run("shell", "dumpsys", "activity", "activities");
assert(activity.split("\n").some((line) => line.includes("topResumedActivity") && line.includes(pkg) && line.includes("MainActivity")), "Unlock the phone and open Reasonix first");
const pid = run("shell", "pidof", pkg).trim();
const port = run("forward", "tcp:0", `localabstract:webview_devtools_remote_${pid}`).trim();
let ws;
try {
const targets = await (await fetch(`http://127.0.0.1:${port}/json/list`, { signal: AbortSignal.timeout(8000) })).json();
const target = targets.find((entry) => entry.title === "Reasonix Studio");
assert(target, "Reasonix Studio WebView is unavailable");
ws = new WebSocket(target.webSocketDebuggerUrl);
await new Promise((resolve, reject) => { ws.addEventListener("open", resolve, { once: true }); ws.addEventListener("error", reject, { once: true }); });
const pending = new Map();
let id = 0;
ws.addEventListener("message", (event) => {
const reply = JSON.parse(event.data);
if (reply.id) pending.get(reply.id)?.(reply);
});
const call = (method, params = {}) => new Promise((resolve, reject) => {
const request = ++id;
const timer = setTimeout(() => { pending.delete(request); reject(new Error(`${method} timed out`)); }, 8000);
pending.set(request, (reply) => {
clearTimeout(timer); pending.delete(request);
if (reply.error || reply.result?.exceptionDetails) reject(new Error(`${method} failed`));
else resolve(reply.result);
});
ws.send(JSON.stringify({ id: request, method, params }));
});
const evaluate = async (expression) => (await call("Runtime.evaluate", { expression, returnByValue: true })).result.value;
const shown = () => /mInputShown=true/.test(run("shell", "dumpsys", "input_method"));
if (shown()) { run("shell", "input", "keyevent", "4"); await pause(500); }
const measure = () => evaluate(`(() => {
const input = document.querySelector('.compose textarea');
const send = document.querySelector('[data-action="session.send"]');
return { height: innerHeight, overflow: document.documentElement.scrollWidth - innerWidth,
input: input?.getBoundingClientRect().toJSON(), send: send?.getBoundingClientRect().toJSON() };
})()`);
const before = await measure();
assert(before.input?.width > 0, "Open a conversation with its composer visible");
try {
const point = { x: before.input.x + before.input.width / 2, y: before.input.y + before.input.height / 2 };
await call("Input.dispatchTouchEvent", { type: "touchStart", touchPoints: [point] });
await call("Input.dispatchTouchEvent", { type: "touchEnd", touchPoints: [] });
let after;
for (let attempt = 0; attempt < 12; attempt++) {
await pause(250); after = await measure();
if (before.height - after.height > 100) break;
}
assert(shown(), "The software keyboard did not open");
assert(before.height - after.height > 100, `Keyboard covers the page: viewport stayed ${before.height} -> ${after.height}`);
assert(after.input.top >= 0 && after.input.bottom <= after.height, "Input is outside the keyboard viewport");
assert(after.send.top >= 0 && after.send.bottom <= after.height, "Send button is outside the keyboard viewport");
assert(after.overflow <= 1, "Keyboard causes horizontal overflow");
console.log(JSON.stringify({ result: "passed", before: before.height, keyboard: after.height, inputBottom: after.input.bottom, sendBottom: after.send.bottom }));
} finally {
if (shown()) run("shell", "input", "keyevent", "4");
}
} finally {
ws?.close();
run("forward", "--remove", `tcp:${port}`);
}
+91 -1
View File
@@ -1,4 +1,94 @@
#!/usr/bin/env bash
#
# Reproduce what `Kernel.kt` does, on this machine, against the page that is
# actually packaged into the APK.
#
# This is the closest thing to an end-to-end test that runs without a phone: the
# launch flags, the token file, the port file and the `--page` directory are the
# same ones the Kotlin shell passes, and the page is read out of
# `app/src/main/assets/page` rather than the frontend's own `dist/`. If this
# serves a working interface, the only untested part is Android itself.
set -euo pipefail
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
exec python "$here/verify-host.py"
android="$here/.."
page="$android/app/src/main/assets/page"
kernel="${KERNEL_BIN:-$android/../reasonix-verify.exe}"
work="${VERIFY_DIR:-/tmp/reasonix-verify}"
rm -rf "$work"
mkdir -p "$work/home/remote"
if [ ! -f "$kernel" ]; then
echo "no runnable kernel; build one first:" >&2
echo " (cd .. && go build -o reasonix-verify.exe ./cmd/reasonix)" >&2
exit 1
fi
if [ ! -f "$page/index.html" ]; then
echo "no packaged page: $page/index.html" >&2
exit 1
fi
# 32 random bytes as hex — the same shape `Kernel.mintToken` writes.
token=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n')
printf '%s\n' "$token" > "$work/home/remote/launch.token"
export REASONIX_HOME="$work/home"
export HOME="$work/home"
export TMPDIR="$work"
echo "kernel: $kernel"
echo "page: $page"
echo "token: ${token:0:8}... (${#token} chars)"
"$kernel" serve \
--addr 127.0.0.1:0 \
--auth token \
--token-file "$work/home/remote/launch.token" \
--port-file "$work/home/remote/serve.port" \
--pid-file "$work/home/remote/serve.pid" \
--no-open \
-page "$page" \
> "$work/kernel.log" 2>&1 &
echo $! > "$work/kernel.pid"
# Wait for the port file the kernel writes after binding, which is exactly the
# wait `Kernel.awaitPort` performs.
for _ in $(seq 1 150); do
if [ -s "$work/home/remote/serve.port" ]; then break; fi
if ! kill -0 "$(cat "$work/kernel.pid")" 2>/dev/null; then
echo "the kernel exited; log:" >&2
cat "$work/kernel.log" >&2
exit 1
fi
sleep 0.2
done
if [ ! -s "$work/home/remote/serve.port" ]; then
echo "the kernel reported no listening address; log:" >&2
cat "$work/kernel.log" >&2
exit 1
fi
port=$(cat "$work/home/remote/serve.port")
echo "listening on: $port"
# The two things the WebView will do: fetch the page, and exist on the origin.
probe=$(curl -s -o /dev/null -w '%{http_code}' "http://$port/")
echo "GET / -> $probe"
if [ "$probe" != "200" ]; then
echo "the page did not answer; log:" >&2
cat "$work/kernel.log" >&2
exit 1
fi
cat > "$work/entry.txt" <<EOF
origin=http://$port
entry=http://$port/#token=$token
token=$token
EOF
echo "entry written: $work/entry.txt"
echo "kernel log (first 10 lines):"
head -10 "$work/kernel.log"
+4
View File
@@ -103,6 +103,10 @@ func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorde
ContextWindow: window,
ArchiveDir: filepath.Join(t.dir, "archive"),
SessionPath: path,
RecentKeep: 4,
// boot's default when cfg.Agent.Keep is unset; without it the bench
// would measure a configuration no real session runs.
KeepPolicy: agent.KeepErrors,
}, rec.sink())
return &harness{sess: sess, agentA: a, path: path, calls: rec}
}
@@ -108,6 +108,7 @@ func newAgent(p provider.Provider, sess *sessionstore.Session, path string, sink
return agent.New(p, tool.NewRegistry(), sess, agent.Options{
ContextWindow: windowTokens,
CompactRatio: compactRatio,
RecentKeep: 2,
SessionPath: path,
WorkspaceID: "cm-e2e",
ModelRef: "deepseek/" + model,
+180
View File
@@ -0,0 +1,180 @@
# Context retrieval bench
Two questions about long-horizon memory, and the harness that makes their
answers trustworthy.
-mode=preflight build all 18 tasks under every arm and check the contract
-mode=snippet does a rank-1 hit hand over the whole answer?
-mode=adversarial send a model after the answer through every host surface
-mode=run-search Experiment R: is searchable canonical a real capability?
-mode=run-boundary Experiment I, paired: the 6 efficiency tasks across their boundary
-mode=run-index Experiment I, full: the 6 efficiency tasks x 4 budgets
-mode=calibrate find each index task's PlantAfterGen for its cue tier
`-dry` drives any run mode with a scripted provider, for nothing. Real runs need
`DEEPSEEK_API_KEY`.
## Status
**Searchable canonical recall: proven.** Six of six recovered with search on,
three of six with it off, and the successes without it came from enumerating
addresses by hand at roughly nine times the page-in cost (139 vs 1292
tokens/task). Measured after three separate leaks were closed, so the numbers
are from a clean environment.
**Fold index marginal utility: direction reproduced, size still conditional.** On the v2
substrate the direction holds in six of six tasks and in the pooled paired batch: with the
cue visible the index addresses the target, so the same task is searched less and pages in
less (0.67 searches and 70 recall tokens per task, against 1.50 and 120 with the cue
absent).
A size is visible but is not yet quotable. On the four tasks whose absent-cue arm stayed
out of a loop, the index cost 64,952 prompt tokens against 115,551 without, 44% less
(`i106-budgetseal` -35,147, `i70-ingestfloor` -8,222, `i37-dispatchlag` -7,599, and
`i112-lagquota` +369, the prefix rent a task that never searches pays). That is a
**post-hoc subset**: `i118-fencelag` and `i47-tenantlease` are excluded because their
absent-cue arm looped in this batch (five and three retrieval rounds, three escapes each;
together 80% of the absent-cue total), and the qualification above does not screen for
stopping behaviour — holding on both boundary arms across three passes shows a stable cue
boundary, not a stopping arm. Quoting 44% as the index's effect would mean choosing the
sample after seeing the result; a batch screened on stopping stability is what would make
a size quotable, and one arm also reverses either way (`i70-ingestfloor` default, +11,135
prompt tokens over three escapes).
Batch facts, because the counts differ: 36 runs were executed (12 paired, 24 index), and
the token totals here cover the 24-run index batch — the paired batch ran before the
per-run usage line existed.
**Things that turned out not to be problems**, each after being measured rather
than argued about:
| Suspected | Measured |
| --- | --- |
| Model writes poor queries (2.17 searches/task) | 22/23 found the target on query 1; the extra searches are one round's parallel fan-out |
| Model reaches for the workspace before memory (7/11) | Linear event order misread; by model round it is MemoryFirst 4/6 |
| Snippets too short for multi-value answers | 240 runes covers 12/12 tasks at 100%; records are 30-179 runes |
| A general stopping failure | 37/44 runs issued no search after the answer was in hand; all 7 that did belong to 2 tasks |
## Measurement distinctions this bench had to learn
Each of these reversed a conclusion once. They are enforced in the schema now,
not left to whoever reads the numbers next.
- **TargetHit is not EvidenceSufficient.** A rank-1 hit can return a window too
short to answer with. `FirstHitCoverage` records how much of the scored answer
the first hit actually handed over.
- **Linear event order is not model-round order.** Two calls in one round are a
parallel fan-out, not a preference. Routing compares rounds.
- **An isolated workdir is not an isolated host.** The sandbox mounts the host
read-only by design. Three leaks were found and closed: the corpus source, the
fixture transcript, and its event log sidecar.
- **A count is not its evidence.** Query text, snippet contents and trajectories
are all persisted now, because three analyses in a row needed a paid re-run to
ask a question of data already collected.
## How a run stays honest
The corpus holds templates, never answers: every scored literal is a `{{var}}`
instantiated per run, so grepping this directory reveals the question and not
the answer. `TestNoAnswerLiteralExistsInTheRepository` asserts it with
`git grep`. Preflight rebuilds every arm and checks the answer is unreadable in
the real `provider.Request`, the probe query ranks the target within five, and
an index task's cue sits at exactly the scales its tier names. The fixture is
removed from disk once the agent holds it in memory, and the directory is
scanned for answer literals before the first provider request and again after
the turn. Any answer reaching the model through a tool that is not `recall`
marks the run contaminated and takes it out of every statistic.
## Index corpus v2
The efficiency substrate is six tasks that hold on both arms: `i37-dispatchlag`
and `i118-fencelag` (quarter), `i106-budgetseal` and `i70-ingestfloor` (half),
`i47-tenantlease` and `i112-lagquota` (default).
They came out of five batches of candidates, each screened the same way: one
`-mode=run-index` pass over the batch, then two more only for the candidates that
were clean on both boundary arms, and only the candidates that held on both arms
in all three passes stayed. A candidate that does not hold is dropped, not
promoted. The numbers are candidates, per tier:
| batch | quarter | half | default |
| --- | --- | --- | --- |
| first pool, `i31`-`i54` | 1 of 8 | 0 of 8 | 2 of 8 |
| second pool, `i61`-`i84` | 1 of 4 screened | 4 of 8 | 0 of 8 |
| third pool, `i85`-`i108` | 2 of 8 | 8 of 16 | none authored |
| replacement pool, `i109`-`i116` | - | - | 4 of 8 |
| replacement pool, `i92`, `i117`-`i124` | 3 of 9 | - | - |
That table counts candidates that held on both arms in all three passes, which is
not the size of the substrate. 89 candidates were authored, 77 were screened (the
second pool's quarter tier stopped at 4 of 8 and its default tier was never run),
and 25 held: 7 quarter, 12 half, 6 default. The substrate keeps two per tier, so
the other 19 survivors are spares rather than corpus members. Candidates from the
earlier round are not counted here.
| task | tier | cue present | cue absent |
| --- | --- | ---: | ---: |
| `i37-dispatchlag` | quarter | 3/3 | 3/3 |
| `i118-fencelag` | quarter | 3/3 | 3/3 |
| `i106-budgetseal` | half | 3/3 | 3/3 |
| `i70-ingestfloor` | half | 3/3 | 3/3 |
| `i47-tenantlease` | default | 3/3 | 3/3 |
| `i112-lagquota` | default | 3/3 | 3/3 |
Across their 36 judged cells: `SnippetStop` 23, `SnippetThenRead` 10 and
`DefensiveRead` 3 - every cell clean, nothing post-sufficient and nothing
`NeverSufficient`.
Four of the first six were flagged in review for sharing answer values, probe
queries and prompt wording within a tier. Separating them and re-screening those
four changed two of them: `i37` and `i47` held, `i62` and `i51` did not, so they
were replaced - `i90-recoverylag` was tried first and did not hold either, and
`i118-fencelag` and `i112-lagquota` are what did. The separation was not
cosmetic: the shared wording had been carrying two of the six.
Nothing in the task shape separates the survivors from the rest: within the half
tier, plain-number answers survived 0 of 8 in the first pool and 4 of 8 in the
second, and codename answers survived 8 of 16 in the third. An earlier claim here
that the answer kind decides whether a candidate holds does not survive this
table and is withdrawn. The dirty class was `PostSufficientRetrieval` in most
failures - the model re-checking a value it already had - but which tasks hit it
was not predictable from their shape at this sample.
`i01`-`i06` are the frozen study's six tasks, kept as `roleStopping`. Measured
once each with `-mode=run-index -task <id>`, per task and per arm:
| task | tier | cue present | cue absent |
| --- | --- | --- | --- |
| `i01-transport-fallback` | quarter | SnippetStop | DefensiveRead |
| `i02-scheduler-handoff` | quarter | PostSufficientRetrieval | PostSufficientRetrieval |
| `i03-coalescing` | half | PostSufficientRetrieval | PostSufficientRunaway |
| `i04-recovery-fence` | half | DefensiveRead | PostSufficientRunaway |
| `i05-cache-probe` | default | SnippetThenRead | PostSufficientRetrieval |
| `i06-dispatch-handoff` | default | SnippetStop | PostSufficientRetrieval |
Five of the six show post-sufficient behaviour in at least one arm (`i02`-`i06`),
and `i01` is clean on both. The cue-present arm is clean for four of them
(`i01`, `i04`, `i05`, `i06`), which is why they stay in the corpus as the
stopping side rather than being retitled. All six are excluded from `run-index`
and `run-boundary` batches and stay measurable one at a time with `-task <id>`. A
task added to the corpus declares a role, so it cannot arrive without one.
A task qualifies only if it is clean on both of its boundary arms - the
cue-present run and the cue-absent run - in every run of a screening batch. Clean
means a `stopping_class` other than `PostSufficientRetrieval` or
`PostSufficientRunaway`: `SnippetStop`, `SnippetThenRead` and `DefensiveRead`
all qualify, and a defensive read is deliberately not counted as waste. The
class comes from the three counts in that report — `searches_after_sufficient`,
`reads_after_sufficient` and `escapes_after_sufficient` — together with how
sufficiency was reached and how many rounds followed it. The remaining
requirements still apply — successful tool call, answer in the output rather
than the cue, rank 1, `FirstHitCoverage` complete, one or two simple values,
nothing in the workspace.
`-mode=calibrate` is not a screening tool: it builds and reads back the
fixtures, and reads nothing else, so its cue-visibility profile is the same for
every task.
Then repeats rather than levels: cue-present against cue-absent on each task's
own boundary, three runs per cell. What is unknown is the variance, and four
budgets sampled once each cannot show it.
@@ -0,0 +1,86 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
// runBlockedBatch runs one dry index batch where every arm's runOne fails
// before any model turn: the task's own directory is a regular file, so the
// fixture cannot be built. In production the same early return happens when the
// fixture build, the session load, or the answer-on-disk guard fails.
func runBlockedBatch(t *testing.T) string {
t.Helper()
tk := indexTasks()[0]
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, tk.ID), []byte("not a directory\n"), 0o644); err != nil {
t.Fatal(err)
}
if code := runExperiment("run-index", experimentIndex, root, true, []contextTask{tk}); code != 0 {
t.Fatalf("runExperiment exit = %d, want 0 (every run failed, but the batch finished)", code)
}
return root
}
func readResultLines(t *testing.T, root string) ([]contextMetrics, []string) {
t.Helper()
raw, err := os.ReadFile(filepath.Join(root, "results.jsonl"))
if err != nil {
t.Fatal(err)
}
var runs []contextMetrics
var lines []string
for text := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") {
var m contextMetrics
if err := json.Unmarshal([]byte(text), &m); err != nil {
t.Fatalf("results.jsonl: %v", err)
}
runs = append(runs, m)
lines = append(lines, text)
}
return runs, lines
}
// A run that failed before the model turn produced no measurement. Recording it
// as a zero-valued contextMetrics puts it in the funnel's Scored denominator,
// in "Where it broke", and in the dose curve as a real zero.
func TestBatchDoesNotRecordAFailedRunAsAResult(t *testing.T) {
root := runBlockedBatch(t)
runs, lines := readResultLines(t, root)
if len(runs) == 0 {
t.Fatal("no runs recorded at all")
}
for i, m := range runs {
if m.Task == "" || m.Arm == "" || m.FailureStage == "" {
t.Fatalf("results.jsonl line %d records a run that never ran as a result "+
"(task=%q arm=%q failure_stage=%q):\n %s", i+1, m.Task, m.Arm, m.FailureStage, lines[i])
}
}
}
// The manifest is the artefact that says which runs counted, so a failed run
// must not be added to CompletedRuns or to the task id list.
func TestBatchManifestDoesNotReportFailedRunsAsCompleted(t *testing.T) {
root := runBlockedBatch(t)
var manifest batchManifest
data, err := os.ReadFile(filepath.Join(root, "batch.json"))
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(data, &manifest); err != nil {
t.Fatal(err)
}
for _, id := range manifest.TaskIDs {
if id == "" {
t.Fatalf("manifest lists an empty task id: task_ids=%q (completed=%d planned=%d)",
manifest.TaskIDs, manifest.CompletedRuns, manifest.PlannedRuns)
}
}
if manifest.CompletedRuns == manifest.PlannedRuns {
t.Fatalf("manifest reports every planned run completed although none produced evidence: "+
"completed=%d planned=%d task_ids=%q", manifest.CompletedRuns, manifest.PlannedRuns, manifest.TaskIDs)
}
}
+155
View File
@@ -0,0 +1,155 @@
package main
import (
"crypto/sha256"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"time"
)
// batchInfo is what the mode itself knows before any run happens. Everything
// else in the manifest is derived from the runs that were written, so a figure
// in it cannot drift from the artifact beside it.
type batchInfo struct {
Mode string
Planned int
Started time.Time
Dry bool
// Repo is the checkout the batch ran from, captured before the harness
// isolates its workspace: git cannot describe a directory that is gone.
Repo string
}
// batchManifest is written beside results.jsonl so a later reader does not have
// to reconstruct what the batch was from a terminal scrollback.
type batchManifest struct {
SchemaVersion int `json:"schema_version"`
Mode string `json:"mode"`
StartedAt string `json:"started_at"`
FinishedAt string `json:"finished_at"`
GitCommit string `json:"git_commit"`
GitDirty *bool `json:"git_dirty,omitempty"`
Model string `json:"model"`
Window int `json:"window"`
Generations int `json:"generations"`
TaskIDs []string `json:"task_ids"`
Arms []string `json:"arms"`
PlannedRuns int `json:"planned_runs"`
CompletedRuns int `json:"completed_runs"`
// FailedRuns counts the planned runs that ended before a transcript existed.
// They are rows in results.jsonl, never part of CompletedRuns, and the field
// is absent when there were none so an intact batch is unchanged.
FailedRuns int `json:"failed_runs,omitempty"`
// Usage totals are absent when nothing reported any: an unmeasured cost must
// not read as a measured zero. UsageReportedRuns stays, because zero there
// is a fact about the batch.
UsageReportedRuns int `json:"usage_reported_runs"`
UsageReportedRequests int `json:"usage_reported_requests,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
CacheHitTokens int `json:"cache_hit_tokens,omitempty"`
CacheMissTokens int `json:"cache_miss_tokens,omitempty"`
UsageEstimated bool `json:"usage_estimated,omitempty"`
ResultsSHA256 string `json:"results_sha256"`
TrajectoriesSHA256 string `json:"trajectories_sha256"`
}
// gitCommitAndDirty reports the checkout the batch ran from. Unknown is written
// rather than omitted, so an absent field is never mistaken for a clean tree.
func gitCommitAndDirty(repo string) (commit string, dirty *bool) {
commit = "unknown"
if repo == "" {
return commit, nil
}
in := func(args ...string) ([]byte, error) {
cmd := exec.Command("git", args...)
cmd.Dir = repo
return cmd.Output()
}
if out, err := in("rev-parse", "--short", "HEAD"); err == nil {
commit = strings.TrimSpace(string(out))
}
if out, err := in("status", "--porcelain"); err == nil {
state := strings.TrimSpace(string(out)) != ""
dirty = &state
}
return commit, dirty
}
func fileSHA256(path string) (string, error) {
data, err := os.ReadFile(path)
if err != nil {
return "", err
}
return fmt.Sprintf("%x", sha256.Sum256(data)), nil
}
func sortedKeys(set map[string]bool) []string {
out := make([]string, 0, len(set))
for k := range set {
out = append(out, k)
}
sort.Strings(out)
return out
}
// writeBatchManifest records the batch beside the runs it describes. It runs
// last, so its digests are of what is already on disk.
func writeBatchManifest(root string, info batchInfo, all []contextMetrics) error {
ids, arms := map[string]bool{}, map[string]bool{}
var prompt, completion, cacheHit, cacheMiss, requests, reported, failed int
estimated := false
for _, m := range all {
// A failed run has no task, arm or evidence to contribute: it is counted
// as a failure below and is not part of what the batch measured.
if m.Failed {
failed++
continue
}
ids[m.Task] = true
arms[m.Arm] = true
if m.UsageReportedRequests > 0 {
reported++
requests += m.UsageReportedRequests
prompt += m.PromptTokens
completion += m.CompletionTokens
cacheHit += m.CacheHitTokens
cacheMiss += m.CacheMissTokens
estimated = estimated || m.UsageEstimated
}
}
commit, dirty := gitCommitAndDirty(info.Repo)
model := runModel
if info.Dry {
model = "scripted"
}
manifest := batchManifest{
SchemaVersion: 1, Mode: info.Mode,
StartedAt: info.Started.UTC().Format(time.RFC3339),
FinishedAt: time.Now().UTC().Format(time.RFC3339),
GitCommit: commit, GitDirty: dirty,
Model: model, Window: fixtureWindow, Generations: fixtureGenerations,
TaskIDs: sortedKeys(ids), Arms: sortedKeys(arms),
PlannedRuns: info.Planned, CompletedRuns: len(all) - failed, FailedRuns: failed,
UsageReportedRuns: reported, UsageReportedRequests: requests,
PromptTokens: prompt, CompletionTokens: completion,
CacheHitTokens: cacheHit, CacheMissTokens: cacheMiss, UsageEstimated: estimated,
}
var err error
if manifest.ResultsSHA256, err = fileSHA256(filepath.Join(root, "results.jsonl")); err != nil {
return err
}
if manifest.TrajectoriesSHA256, err = fileSHA256(filepath.Join(root, "trajectories.jsonl")); err != nil {
return err
}
data, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return err
}
return os.WriteFile(filepath.Join(root, "batch.json"), append(data, '\n'), 0o644)
}
@@ -0,0 +1,91 @@
package main
import (
"crypto/sha256"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func seedArtifacts(t *testing.T, root string) {
t.Helper()
for _, name := range []string{"results.jsonl", "trajectories.jsonl"} {
if err := os.WriteFile(filepath.Join(root, name), []byte("{}\n"), 0o644); err != nil {
t.Fatal(err)
}
}
}
func readManifest(t *testing.T, root string) (batchManifest, string) {
t.Helper()
data, err := os.ReadFile(filepath.Join(root, "batch.json"))
if err != nil {
t.Fatal(err)
}
var got batchManifest
if err := json.Unmarshal(data, &got); err != nil {
t.Fatal(err)
}
return got, string(data)
}
func TestBatchManifestRecordsWhatTheBatchWas(t *testing.T) {
root := t.TempDir()
seedArtifacts(t, root)
all := []contextMetrics{
{Task: "b", Arm: "index-off"},
{Task: "a", Arm: "index-off"},
{Task: "a", Arm: "index-half", UsageReportedRequests: 2, PromptTokens: 100,
CompletionTokens: 10, CacheHitTokens: 60, CacheMissTokens: 40},
}
if err := writeBatchManifest(root, batchInfo{Mode: "run-index", Planned: 4, Started: time.Now().Add(-time.Minute)}, all); err != nil {
t.Fatal(err)
}
got, _ := readManifest(t, root)
if got.SchemaVersion != 1 || got.Mode != "run-index" || got.PlannedRuns != 4 || got.CompletedRuns != 3 {
t.Fatalf("manifest header = %+v", got)
}
if strings.Join(got.TaskIDs, ",") != "a,b" || strings.Join(got.Arms, ",") != "index-half,index-off" {
t.Fatalf("ids/arms = %v / %v, want sorted and unique", got.TaskIDs, got.Arms)
}
if got.UsageReportedRuns != 1 || got.UsageReportedRequests != 2 || got.PromptTokens != 100 ||
got.CompletionTokens != 10 || got.CacheHitTokens != 60 || got.CacheMissTokens != 40 {
t.Fatalf("usage totals = %+v", got)
}
// No repo means git could not describe the checkout: unknown commit, and no
// dirty flag at all, because "clean" is a claim this batch cannot make.
if got.GitCommit != "unknown" || got.GitDirty != nil {
t.Fatalf("git state = %q / %v, want unknown and no dirty flag", got.GitCommit, got.GitDirty)
}
want := fmt.Sprintf("%x", sha256.Sum256([]byte("{}\n")))
if got.ResultsSHA256 != want || got.TrajectoriesSHA256 != want {
t.Fatalf("digests = %q / %q, want %q", got.ResultsSHA256, got.TrajectoriesSHA256, want)
}
}
// A dry batch pays nothing, so its cost fields must be absent rather than zero:
// an unmeasured spend must not read as a measured one.
func TestBatchManifestLeavesUnreportedUsageAbsent(t *testing.T) {
root := t.TempDir()
seedArtifacts(t, root)
all := []contextMetrics{{Task: "a", Arm: "index-off"}, {Task: "a", Arm: "index-half"}}
if err := writeBatchManifest(root, batchInfo{Mode: "run-index", Planned: 2, Started: time.Now(), Dry: true}, all); err != nil {
t.Fatal(err)
}
got, raw := readManifest(t, root)
if got.UsageReportedRuns != 0 {
t.Fatalf("usage_reported_runs = %d, want 0 reported", got.UsageReportedRuns)
}
if got.Model != "scripted" {
t.Fatalf("dry batch model = %q, want scripted", got.Model)
}
for _, absent := range []string{"usage_reported_requests", "prompt_tokens", "completion_tokens", "cache_hit_tokens"} {
if strings.Contains(raw, absent) {
t.Fatalf("a dry batch wrote %q:\n%s", absent, raw)
}
}
}
@@ -0,0 +1,69 @@
package main
import (
"strings"
"testing"
)
// boundaryRuns builds the input reportBoundaries reads: one scored run for an
// index task's cue side, and one failed run for another index task's cue side.
// The runs are constructed because the report is a pure function of the slice;
// the batch path that produces them is covered by the failed-batch tests.
func boundaryRuns(t *testing.T) (all []contextMetrics, scoredTask, failedTask string) {
t.Helper()
index := indexTasks()
if len(index) < 2 {
t.Fatalf("need at least two index tasks, have %d", len(index))
}
scored, failed := index[0], index[1]
cue, noCue := boundaryPair(scored.CueTier)
if cue == "" || noCue == "" {
t.Fatalf("task %s has an empty boundary pair (%q, %q)", scored.ID, cue, noCue)
}
failedCue, _ := boundaryPair(failed.CueTier)
if failedCue == "" {
t.Fatalf("task %s has no cue side", failed.ID)
}
all = []contextMetrics{
{Task: scored.ID, Arm: "index-" + cue, AnswerRecovered: true, SearchCalls: 1, RecallReturnedTokens: 318},
{Task: scored.ID, Arm: "index-" + noCue, AnswerRecovered: false},
{Task: failed.ID, Arm: "index-" + failedCue, Failed: true, FailureStage: stageNoEvidence},
}
return all, scored.ID, failed.ID
}
// TestBoundaryReportExcludesFailedRuns: a run that never produced a transcript
// must not appear in the cue-boundary table, which was the last reporting exit
// that still folded it in as a zero measurement.
func TestBoundaryReportExcludesFailedRuns(t *testing.T) {
all, scoredTask, failedTask := boundaryRuns(t)
out := captureStdout(t, func() { reportBoundaries(all) })
if strings.Contains(out, failedTask) {
t.Fatalf("the boundary table printed the failed task %q as a measurement:\n%s", failedTask, out)
}
if !strings.Contains(out, scoredTask) {
t.Fatalf("the boundary table dropped the scored task %q, so the exclusion is not specific to failures:\n%s", scoredTask, out)
}
if !strings.Contains(out, "cue-present") || !strings.Contains(out, "cue-absent") {
t.Fatalf("the pooled comparison disappeared:\n%s", out)
}
}
// TestBoundaryReportSaysNothingToCompareWhenEveryRunFailed: with no scored run
// the report must say so, rather than printing a comparison of zeros.
func TestBoundaryReportSaysNothingToCompareWhenEveryRunFailed(t *testing.T) {
all, _, _ := boundaryRuns(t)
failedOnly := make([]contextMetrics, 0, len(all))
for _, m := range all {
m.Failed = true
m.FailureStage = stageNoEvidence
failedOnly = append(failedOnly, m)
}
out := captureStdout(t, func() { reportBoundaries(failedOnly) })
if !strings.Contains(out, "scored=0") || !strings.Contains(out, "failed=") {
t.Fatalf("an all-failed batch did not report scored=0 with a failed count:\n%s", out)
}
if strings.Contains(out, "cue-present") || strings.Contains(out, "cue-absent") {
t.Fatalf("an all-failed batch still printed a pooled comparison of zeros:\n%s", out)
}
}
+171
View File
@@ -0,0 +1,171 @@
// capture.go — what the agent actually hands a provider.
package main
import (
"context"
"encoding/json"
"fmt"
"reasonix/internal/state/sessionstore"
"strings"
"reasonix/internal/assembly/boot"
"reasonix/internal/contract/ablation"
"reasonix/internal/contract/event"
"reasonix/internal/contract/provider"
"reasonix/internal/contract/tool"
"reasonix/internal/runtime/agent"
_ "reasonix/internal/tools/builtin"
)
// A projection is what the agent believes; a provider.Request is what the model
// is told. Rendering, sanitization and tool-schema construction sit between
// them, so the arm's contract is checked on the request.
type captureProvider struct {
requests []provider.Request
}
func (p *captureProvider) Name() string { return "contextbench-capture" }
// Stream records the request and answers with a clean terminal response. An
// empty answer would buy an empty-final retry and a second request, which is
// not the one round this is here to inspect.
func (p *captureProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
p.requests = append(p.requests, cloneRequest(req))
ch := make(chan provider.Chunk, 2)
ch <- provider.Chunk{Type: provider.ChunkText, Text: "contextbench preflight complete"}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
return ch, nil
}
func cloneRequest(req provider.Request) provider.Request {
out := req
out.Messages = append([]provider.Message(nil), req.Messages...)
out.Tools = append([]provider.ToolSchema(nil), req.Tools...)
return out
}
// captureRequest rebuilds the session the way a resume does — transcript from
// disk, projection from its sidecar, the shipped tool surface at this arm —
// and returns the request the first round hands the provider.
func captureRequest(f builtFixture, arm ablation.Set, prompt string) (provider.Request, error) {
sess, err := sessionstore.LoadSession(f.Path)
if err != nil {
return provider.Request{}, fmt.Errorf("load session: %w", err)
}
reg := tool.NewRegistry()
for _, t := range tool.Builtins() {
reg.Add(t)
}
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
cap := &captureProvider{}
a := agent.New(cap, reg, sess, agent.Options{
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
}, event.Discard)
a.LoadProjectionSidecar(f.Path)
if err := a.Run(context.Background(), prompt); err != nil {
return provider.Request{}, fmt.Errorf("run: %w", err)
}
// The turn's own request is the one carrying tools. A fold on the way in
// asks the same provider for a digest, and that request has none — taking
// the first request blindly would inspect the summarizer's prompt.
for _, req := range cap.requests {
if len(req.Tools) > 0 {
return req, nil
}
}
return provider.Request{}, fmt.Errorf("no request carried tools (%d captured)", len(cap.requests))
}
// surfaceField is one place a model could read a string, named so a leak
// report says where rather than only that.
type surfaceField struct {
Path string
Text string
}
// providerSurface enumerates every provider-visible string in a request.
// Enumerated rather than marshalled whole: a Message also carries local-only
// fields, and scanning those would fail on bytes no model ever sees.
func providerSurface(req provider.Request) []surfaceField {
var out []surfaceField
add := func(path, text string) {
if strings.TrimSpace(text) != "" {
out = append(out, surfaceField{Path: path, Text: text})
}
}
for i, m := range req.Messages {
add(fmt.Sprintf("messages[%d].content", i), m.Content)
add(fmt.Sprintf("messages[%d].reasoning_content", i), m.ReasoningContent)
add(fmt.Sprintf("messages[%d].name", i), m.Name)
for j, tc := range m.ToolCalls {
add(fmt.Sprintf("messages[%d].tool_calls[%d].name", i, j), tc.Name)
add(fmt.Sprintf("messages[%d].tool_calls[%d].arguments", i, j), tc.Arguments)
}
for j, item := range m.ResponsesItems {
add(fmt.Sprintf("messages[%d].responses_items[%d]", i, j), string(item))
}
}
for i, t := range req.Tools {
add(fmt.Sprintf("tools[%d].name", i), t.Name)
add(fmt.Sprintf("tools[%d].description", i), t.Description)
if len(t.Parameters) > 0 {
b, err := json.Marshal(t.Parameters)
if err == nil {
add(fmt.Sprintf("tools[%d].parameters", i), string(b))
}
}
}
return out
}
// findInSurface reports every place a marker is readable.
func findInSurface(req provider.Request, marker string) []string {
var hits []string
for _, f := range providerSurface(req) {
if strings.Contains(f.Text, marker) {
hits = append(hits, f.Path)
}
}
return hits
}
// recallToolSchema returns the recall tool as the provider sees it.
func recallToolSchema(req provider.Request) (provider.ToolSchema, bool) {
for _, t := range req.Tools {
if t.Name == "recall" {
return t, true
}
}
return provider.ToolSchema{}, false
}
// localOnlyLeaks reports host-side fields that should never survive the
// provider boundary. A hit here is a boundary regression, marker or no marker.
func localOnlyLeaks(req provider.Request) []string {
var out []string
for i, m := range req.Messages {
switch {
case m.RawContent != "":
out = append(out, fmt.Sprintf("messages[%d].raw_content", i))
case m.LocalOnly:
out = append(out, fmt.Sprintf("messages[%d].local_only", i))
case m.DecisionReceipt != nil || len(m.DecisionReceipts) > 0:
out = append(out, fmt.Sprintf("messages[%d].decision_receipt", i))
case m.ToolExecution != nil:
out = append(out, fmt.Sprintf("messages[%d].tool_execution", i))
case len(m.MemoryCitations) > 0:
out = append(out, fmt.Sprintf("messages[%d].memory_citations", i))
}
}
return out
}
// searchWording is what a search-off arm must never be told. The arm exists to
// reproduce the runtime before search; an arm offered a capability it does not
// have is a different experiment.
var searchWording = []string{"recall with a query", "search the whole folded region", "query still finds"}
@@ -0,0 +1,671 @@
// contextbench.go — what the model did with the affordance, read off structure.
package main
import (
"encoding/json"
"fmt"
"io"
"regexp"
"slices"
"strconv"
"strings"
"reasonix/internal/base/tokencount"
"reasonix/internal/contract/provider"
)
// Every judgement here reads a tool call's name and its JSON arguments, or a
// canonical position, and never the model's prose. "I will search the history"
// is not a search; a recall call with a query is.
// failure stages, in funnel order. Exactly one is reported per run, except for
// NoEvidence: that stage belongs to a run with no measurement at all.
const (
stageReadMissed = "NoSearch" // never asked, and never read the target either
stageSearchMiss = "SearchMiss" // searched, target never came back
stageHitNotRead = "HitNotRead" // target was returned and never opened
stageAnswerWrong = "ReadButAnswerWrong" // opened it and still answered wrong
stageRecovered = "Recovered"
stageContaminated = "Contaminated"
stageNoEvidence = "NoEvidence"
// Routing is judged by model round, not by the linear order of tool events:
// two calls in one round are one parallel fan-out, and calling that
// "workspace first" would report a choice the model never made.
routeMemoryFirst = "MemoryFirst"
routeWorkspaceFirst = "WorkspaceFirst"
routeParallel = "Parallel"
routeNeither = "Neither"
stageCueRead = "DirectRead" // read the target from an index address, no search needed
stageNoRetrieval = "NoRetrieval" // answered without touching recall at all
toolRecall = "recall"
)
// searchAttempt is one query the model wrote, and what it got back. Kept per
// attempt rather than aggregated: the question is why a retrieval that the host
// probe ranks first takes the model more than one try.
type searchAttempt struct {
Ordinal int `json:"ordinal"`
Round int `json:"round"`
Query string `json:"query"`
TargetRank int `json:"target_rank"` // 0 = the target did not come back
ResultCount int `json:"result_count"`
// Reformulation classifies this query against the one before it, and
// MissingProbeTerms are the host probe's terms this query left out. Both
// are derived from token sets: no judge, and nothing read out of prose.
Reformulation string `json:"reformulation,omitempty"`
MissingProbeTerms []string `json:"missing_probe_terms,omitempty"`
}
// contextMetrics is one run's behaviour.
type contextMetrics struct {
Task string `json:"task"`
Arm string `json:"arm"`
SearchCalls int `json:"search_calls"`
// SearchHits counts searches that returned anything; TargetSearchHits
// counts searches that returned the target. The gap is query quality.
SearchHits int `json:"search_hits"`
TargetSearchHits int `json:"target_search_hits"`
FirstTargetRank int `json:"first_target_rank"` // 0 = never returned
// FirstHitCoverage: how much of the answer the first hit handed over.
// rank=1 with coverage 1/3 is a window too short to answer with, not a
// model that would not stop.
FirstHitCoverage int `json:"first_hit_coverage"`
AnswerValues int `json:"answer_values"`
ReadCalls int `json:"read_calls"`
TargetRead bool `json:"target_read"`
ReadAfterHit bool `json:"read_after_hit"`
// RecallReadWithoutSearch: the index addressed the target, so no search was
// needed. Not "direct read" — read_file is a read too, and in an audit the
// two mean opposite things.
RecallReadWithoutSearch bool `json:"recall_read_without_search"`
SearchThenRecallRead bool `json:"search_then_recall_read"`
FirstSearchRound int `json:"first_search_round"`
FirstReadRound int `json:"first_read_round"`
RetrievalRounds int `json:"retrieval_rounds"`
// RecallReturnedTokens is what recall paged back in. The index saves
// resident prefix; this is what is paid dynamically instead.
RecallReturnedTokens int `json:"recall_returned_tokens"`
// Provider usage as reported on the stream, or empty for a dry run. The count
// is completions that reported usage, so it is not a count of attempts.
UsageReportedRequests int `json:"usage_reported_requests,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
CacheHitTokens int `json:"cache_hit_tokens,omitempty"`
CacheMissTokens int `json:"cache_miss_tokens,omitempty"`
UsageEstimated bool `json:"usage_estimated,omitempty"`
Searches []searchAttempt `json:"searches,omitempty"`
// PostHitSearches are queries issued after the target had already been
// returned: not a query problem, a result-to-read problem.
PostHitSearches int `json:"post_hit_searches"`
AnswerRecovered bool `json:"answer_recovered"`
// AnswerAmbiguous marks an answer that offered more than one candidate for
// a value. Markers matching proves the right fact appears; it does not
// prove it was submitted as the answer. Recorded, never acted on.
AnswerAmbiguous bool `json:"answer_ambiguous,omitempty"`
AmbiguousVar string `json:"ambiguous_var,omitempty"`
MissingMarkers []string `json:"missing_markers,omitempty"`
// FinalAnswer is kept bounded so a surprising run can be read back. A
// metric nobody can audit is a claim, not a measurement.
FinalAnswer string `json:"final_answer,omitempty"`
FailureStage string `json:"failure_stage"`
TimeoutReason string `json:"timeout_reason,omitempty"`
// Failed marks a run that ended before a transcript existed: it has no score
// and no place in the statistics. The row stays in results.jsonl with its
// task, arm and stage, so a failure is counted rather than dropped.
Failed bool `json:"failed,omitempty"`
Reason string `json:"reason,omitempty"`
// UnexpectedWorkTools are investigation tools used instead of recall.
// Reaching for the workspace is behaviour, not noise: how a model routes
// its uncertainty when memory is missing is part of what is measured.
UnexpectedWorkTools []string `json:"unexpected_work_tools,omitempty"`
EscapeCalls int `json:"escape_calls"`
EscapeBeforeFirstRecall int `json:"escape_before_first_recall"`
EscapeAfterFirstRecall int `json:"escape_after_first_recall"`
// Contaminated: the answer was reachable through something other than
// recall. Asserting this beats trusting the isolation, and a run that trips
// it leaves the statistics rather than inflating one.
Contaminated bool `json:"contaminated,omitempty"`
LeakedVia string `json:"leaked_via,omitempty"`
LeakedMarker string `json:"leaked_marker,omitempty"`
LeakedArgs string `json:"leaked_args,omitempty"`
LeakedOutput string `json:"leaked_output,omitempty"`
// EscapeFoundAnswer separates "looked at the workspace" from "found the
// answer there": different facts about the same call.
EscapeFoundAnswer bool `json:"escape_found_answer,omitempty"`
CueVisible bool `json:"cue_visible"`
// trajectory is kept out of the metric line and written beside it, so a
// future question can be asked of this batch instead of a new one.
trajectory []provider.Message
// CueDirectRead: the cue was addressed, the first memory action was a
// positions read, and it covered the target. Crediting any positions call
// would hand the index a search's work.
CueDirectRead bool `json:"cue_direct_read"`
// Sufficiency, not hit. A search that returns the target at rank 1 has not
// necessarily returned the answer: the snippet is bounded. The stopping
// boundary is the round where everything needed was readable.
FirstSufficientRound int `json:"first_sufficient_round"`
SufficientVia string `json:"sufficient_via"` // snippet | read | never
RoundsAfterSufficient int `json:"rounds_after_sufficient"`
SearchesAfterSufficient int `json:"searches_after_sufficient"`
ReadsAfterSufficient int `json:"reads_after_sufficient"`
EscapesAfterSufficient int `json:"escapes_after_sufficient"`
RecallTokensAfterSufficient int `json:"recall_tokens_after_sufficient"`
StoppingClass string `json:"stopping_class"`
// What happened after the answer was already in hand. A timeout that spent
// seven rounds past its target is a stopping problem, and extending the
// budget would only buy the loop more time.
TargetFirstSeenRound int `json:"target_first_seen_round"`
RoundsAfterTarget int `json:"rounds_after_target"`
SearchesAfterTarget int `json:"searches_after_target"`
RecallTokensAfterTarget int `json:"recall_tokens_after_target"`
TargetHeldAtEnd bool `json:"target_held_at_end"`
// Routing is where the model looked first when its memory was missing.
Routing string `json:"routing"`
FirstRecallRound int `json:"first_recall_round"`
FirstEscapeRound int `json:"first_escape_round"`
}
// hitPosition matches the address recall prints for a search hit. Reading our
// own rendered format, not guessing at the model's words.
var hitPosition = regexp.MustCompile(`(?m)^#(\d+) `)
// scoreRun reads the messages one turn appended and reports what happened.
// runScorer walks one turn's messages, holding the state a judgement needs:
// which calls are open, whether the target has come back, and which search
// each result belongs to.
type runScorer struct {
m contextMetrics
inst fixtureInstance
target int
calls map[string]provider.ToolCall
pending map[string]int
round int
sawTargetHit bool
// seen accumulates everything recall has shown the model, so sufficiency is
// judged on what it could read rather than on which call produced it.
seen strings.Builder
sufficient bool
}
// scoreRun reads the messages one turn appended and reports what happened.
func scoreRun(msgs []provider.Message, inst fixtureInstance, target int, arm string, cueVisible bool) contextMetrics {
s := &runScorer{
m: contextMetrics{Task: inst.Task.ID, Arm: arm, CueVisible: cueVisible},
inst: inst, target: target,
calls: map[string]provider.ToolCall{}, pending: map[string]int{},
}
for _, msg := range msgs {
if len(msg.ToolCalls) > 0 {
s.round++
}
for _, tc := range msg.ToolCalls {
s.calls[tc.ID] = tc
s.observeCall(tc)
}
if msg.Role == provider.RoleTool {
s.observeResult(msg)
}
}
return s.finish(cueVisible)
}
// observeCall records one tool call: a recall search, a recall read, or a
// reach for the workspace.
func (s *runScorer) observeCall(tc provider.ToolCall) {
m := &s.m
if tc.Name != toolRecall {
if !containsString(m.UnexpectedWorkTools, tc.Name) {
m.UnexpectedWorkTools = append(m.UnexpectedWorkTools, tc.Name)
}
m.EscapeCalls++
if s.sufficient {
m.EscapesAfterSufficient++
}
if m.FirstEscapeRound == 0 {
m.FirstEscapeRound = s.round
}
if m.SearchCalls+m.ReadCalls == 0 {
m.EscapeBeforeFirstRecall++
} else {
m.EscapeAfterFirstRecall++
}
return
}
if m.FirstRecallRound == 0 {
m.FirstRecallRound = s.round
}
query, positions := recallArgs(tc.Arguments)
switch {
case query != "":
m.SearchCalls++
if m.FirstSearchRound == 0 {
m.FirstSearchRound = s.round
}
if s.sawTargetHit {
m.PostHitSearches++
m.SearchesAfterTarget++
}
if s.sufficient {
m.SearchesAfterSufficient++
}
m.Searches = append(m.Searches, searchAttempt{Ordinal: m.SearchCalls, Round: s.round, Query: query})
s.pending[tc.ID] = len(m.Searches) - 1
case len(positions) > 0:
m.ReadCalls++
if s.sufficient {
m.ReadsAfterSufficient++
}
if m.FirstReadRound == 0 {
m.FirstReadRound = s.round
}
// A read covers the target when it names its position: recall returns
// the tool results under the caller's own address.
if containsInt(positions, s.target) {
m.TargetRead = true
switch {
case s.sawTargetHit:
m.ReadAfterHit, m.SearchThenRecallRead = true, true
case m.SearchCalls == 0:
m.RecallReadWithoutSearch = true
}
}
}
}
// observeResult reads what a call returned: contamination, page-in cost, and
// whether this search brought the target back.
func (s *runScorer) observeResult(msg provider.Message) {
m := &s.m
call, ok := s.calls[msg.ToolCallID]
if !ok {
return
}
if call.Name != toolRecall {
// Codenames only: a three-digit epoch appears in any long file by
// chance, and one did. A model holding the codename holds the numbers
// beside it anyway.
for _, marker := range s.inst.AnswerMarkers {
if len(marker) >= 6 && strings.Contains(msg.Content, marker) {
m.Contaminated, m.LeakedVia, m.LeakedMarker = true, call.Name, marker
m.EscapeFoundAnswer = true
m.LeakedArgs, m.LeakedOutput = truncate(call.Arguments, 200), truncate(msg.Content, 300)
}
}
return
}
cost := tokencount.Text(msg.Content)
m.RecallReturnedTokens += cost
if s.sawTargetHit {
m.RecallTokensAfterTarget += cost
}
if s.sufficient {
m.RecallTokensAfterSufficient += cost
}
// Everything recall showed the model, judged as a whole: the answer may
// arrive across a snippet and a read, and either way it is readable.
s.seen.WriteString(msg.Content)
s.seen.WriteString("\n")
if !s.sufficient && allMarkersIn(s.seen.String(), s.inst.AnswerMarkers) {
s.sufficient = true
m.FirstSufficientRound = s.round
query, _ := recallArgs(call.Arguments)
m.SufficientVia = "read"
if query != "" {
m.SufficientVia = "snippet"
}
}
if query, _ := recallArgs(call.Arguments); query == "" {
return
}
positions := searchHitPositions(msg.Content)
if len(positions) > 0 {
m.SearchHits++
}
rank := indexOfInt(positions, s.target)
if i, ok := s.pending[msg.ToolCallID]; ok {
m.Searches[i].TargetRank, m.Searches[i].ResultCount = rank, len(positions)
}
if rank == 0 {
return
}
m.TargetSearchHits++
if !s.sawTargetHit {
m.TargetFirstSeenRound = s.round
m.FirstHitCoverage = answerCoverage(msg.Content, s.inst.AnswerMarkers)
}
s.sawTargetHit = true
if m.FirstTargetRank == 0 {
m.FirstTargetRank = rank
}
}
// finish derives what only the whole walk can say.
func (s *runScorer) finish(cueVisible bool) contextMetrics {
m := s.m
m.RetrievalRounds = m.SearchCalls + m.ReadCalls
m.TargetHeldAtEnd = s.sawTargetHit || m.TargetRead
if m.TargetFirstSeenRound > 0 {
m.RoundsAfterTarget = s.round - m.TargetFirstSeenRound
}
m.Routing = routing(m.FirstRecallRound, m.FirstEscapeRound)
m.CueDirectRead = cueVisible && m.RecallReadWithoutSearch
if m.SufficientVia == "" {
m.SufficientVia = "never"
}
if m.FirstSufficientRound > 0 {
m.RoundsAfterSufficient = s.round - m.FirstSufficientRound
}
m.AnswerValues = len(s.inst.AnswerMarkers)
m.StoppingClass = stoppingClass(m)
classifySearches(&m, s.inst)
return m
}
// scoreAnswer grades the final text on the task's markers. Nonces, matched
// exactly: no judge model, and nothing a general answer could satisfy.
func (m *contextMetrics) scoreAnswer(final string, inst fixtureInstance) {
lower := strings.ToLower(final)
for _, marker := range inst.AnswerMarkers {
if !strings.Contains(lower, strings.ToLower(marker)) {
m.MissingMarkers = append(m.MissingMarkers, marker)
}
}
m.AnswerRecovered = len(m.MissingMarkers) == 0
m.AnswerAmbiguous, m.AmbiguousVar = ambiguousAnswer(final, inst)
m.FailureStage = m.stage()
if len(final) > 1200 {
final = final[:1200] + "…"
}
m.FinalAnswer = final
}
// stage places the run in the funnel. One label, chosen at the first step that
// did not happen, so a table of stages reads as where the retrieval broke.
func (m contextMetrics) stage() string {
switch {
case m.Contaminated:
return stageContaminated
case m.AnswerRecovered && m.RecallReadWithoutSearch:
return stageCueRead
case m.AnswerRecovered:
return stageRecovered
case m.TargetRead:
return stageAnswerWrong
case m.TargetSearchHits > 0:
return stageHitNotRead
case m.SearchCalls > 0:
return stageSearchMiss
case m.ReadCalls > 0:
return stageReadMissed
default:
return stageNoRetrieval
}
}
// recallArgs pulls the two fields that say which operation a recall call is.
func recallArgs(args string) (query string, positions []int) {
var parsed struct {
Query string `json:"query"`
Positions []int `json:"positions"`
}
if err := json.Unmarshal([]byte(args), &parsed); err != nil {
return "", nil
}
return strings.TrimSpace(parsed.Query), parsed.Positions
}
// searchHitPositions reads the addresses out of a search result, in rank order.
func searchHitPositions(text string) []int {
var out []int
for _, match := range hitPosition.FindAllStringSubmatch(text, -1) {
if n, err := strconv.Atoi(match[1]); err == nil {
out = append(out, n)
}
}
return out
}
func containsInt(list []int, want int) bool { return slices.Contains(list, want) }
// indexOfInt returns a 1-based rank, or 0 when absent.
func indexOfInt(list []int, want int) int {
for i, v := range list {
if v == want {
return i + 1
}
}
return 0
}
func containsString(list []string, want string) bool { return slices.Contains(list, want) }
// funnel summarizes one arm's runs the way the experiment is read.
type funnel struct {
Arm string
Runs int
Scored, Contaminated, Errored int
Searched, TargetHit, TargetRead, Answered int
CueReads int
SearchCalls, ReadCalls, RecallTokens int
Escapes, EscapeCalls int
Stages, Routes map[string]int
}
func summarize(arm string, runs []contextMetrics) funnel {
f := funnel{Arm: arm, Runs: len(runs), Stages: map[string]int{}, Routes: map[string]int{}}
for _, r := range runs {
// A run that never produced a transcript is not a scored run and has no
// stage to place: counting it would put an empty name in the funnel, the
// stage table and the routes.
if r.Failed {
continue
}
if r.Contaminated {
f.Contaminated++
f.Stages[r.FailureStage]++
continue
}
if r.FailureStage == "RunError" {
f.Errored++
f.Stages[r.FailureStage]++
continue
}
f.Scored++
if r.SearchCalls > 0 {
f.Searched++
}
if r.TargetSearchHits > 0 {
f.TargetHit++
}
if r.TargetRead {
f.TargetRead++
}
if r.AnswerRecovered {
f.Answered++
}
if r.RecallReadWithoutSearch {
f.CueReads++
}
if len(r.UnexpectedWorkTools) > 0 {
f.Escapes++
}
f.EscapeCalls += r.EscapeCalls
f.SearchCalls += r.SearchCalls
f.ReadCalls += r.ReadCalls
f.RecallTokens += r.RecallReturnedTokens
f.Stages[r.FailureStage]++
if r.FailureStage != "RunError" {
f.Routes[r.Routing]++
}
}
return f
}
func (f funnel) line() string {
per := func(n int) float64 {
if f.Scored == 0 {
return 0
}
return float64(n) / float64(f.Scored)
}
line := fmt.Sprintf("%-14s runs=%-3d answered=%d/%d searched=%d target-hit=%d cue-read=%d search/task=%.2f recall-tok/task=%.0f escape/task=%.2f",
f.Arm, f.Scored, f.Answered, f.Scored, f.Searched, f.TargetHit, f.CueReads,
per(f.SearchCalls), per(f.RecallTokens), per(f.EscapeCalls))
if f.Contaminated > 0 {
line += fmt.Sprintf(" CONTAMINATED=%d", f.Contaminated)
}
if f.Errored > 0 {
line += fmt.Sprintf(" ERRORED=%d", f.Errored)
}
return line
}
func writeJSONLine(w io.Writer, v any) error {
b, err := json.Marshal(v)
if err != nil {
return err
}
_, err = fmt.Fprintln(w, string(b))
return err
}
func truncate(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n] + "…"
}
// routing says where the model looked first, comparing rounds rather than
// event order. Same round is a parallel fan-out, not a preference.
func routing(recallRound, escapeRound int) string {
switch {
case recallRound == 0 && escapeRound == 0:
return routeNeither
case recallRound == 0:
return routeWorkspaceFirst
case escapeRound == 0:
return routeMemoryFirst
case recallRound < escapeRound:
return routeMemoryFirst
case escapeRound < recallRound:
return routeWorkspaceFirst
default:
return routeParallel
}
}
// ambiguousAnswer reports an answer naming more than one candidate of a
// codename's shape. Only codenames are checked: they carry a stem this run
// declared, so two distinct "fence-…" values in one answer is one answer too
// many. Numbers have no such shape and are left alone.
func ambiguousAnswer(final string, inst fixtureInstance) (bool, string) {
for _, spec := range inst.Task.Vars {
if spec.Kind != varCodename {
continue
}
seen := map[string]bool{}
for _, match := range regexp.MustCompile(regexp.QuoteMeta(spec.Word)+`-[a-z0-9]{3,}`).FindAllString(final, -1) {
seen[match] = true
}
if len(seen) > 1 {
return true, spec.Name
}
}
return false, ""
}
func allMarkersIn(text string, markers []string) bool {
if len(markers) == 0 {
return false
}
lower := strings.ToLower(text)
for _, marker := range markers {
if !strings.Contains(lower, strings.ToLower(marker)) {
return false
}
}
return true
}
// Stopping classes. The boundary is sufficiency, not the target coming back:
// a rank-1 hit whose snippet was too short to answer leaves a read entirely
// justified, and counting that as waste would blame the model for the bound.
const (
stopSnippet = "SnippetStop" // enough arrived, and the turn ended
stopSnippetRead = "SnippetThenRead" // the snippet was short; the read finished it
stopDefensiveRead = "DefensiveRead" // enough had arrived, one read confirmed it
stopPostRetrieval = "PostSufficientRetrieval" // enough had arrived, and retrieval continued
stopRunaway = "PostSufficientRunaway" // it continued for rounds, or ran out of budget
stopNeverEnough = "NeverSufficient"
)
// runawayRounds is where continuing stops looking like confirmation.
const runawayRounds = 3
func stoppingClass(m contextMetrics) string {
if m.FirstSufficientRound == 0 {
return stopNeverEnough
}
after := m.SearchesAfterSufficient + m.ReadsAfterSufficient + m.EscapesAfterSufficient
switch {
case after == 0 && m.SufficientVia == "snippet":
return stopSnippet
case after == 0:
return stopSnippetRead
case m.RoundsAfterSufficient >= runawayRounds || m.FailureStage == "RunError":
return stopRunaway
case after == 1 && m.ReadsAfterSufficient == 1:
return stopDefensiveRead
default:
return stopPostRetrieval
}
}
// reportStopping is the funnel that matters once retrieval succeeds: what the
// turn did after it already had everything it needed.
func reportStopping(all []contextMetrics) string {
classes := map[string]int{}
var extraRounds, extraTokens []float64
sufficient, answeredNext := 0, 0
for _, m := range all {
if m.Failed || m.Contaminated {
continue
}
classes[m.StoppingClass]++
if m.FirstSufficientRound == 0 {
continue
}
sufficient++
extraRounds = append(extraRounds, float64(m.RoundsAfterSufficient))
extraTokens = append(extraTokens, float64(m.RecallTokensAfterSufficient))
if m.RoundsAfterSufficient <= 1 {
answeredNext++
}
}
var b strings.Builder
b.WriteString("\n## After the evidence was already enough\n")
fmt.Fprintf(&b, " sufficient in %d runs; ended within one round in %d\n", sufficient, answeredNext)
fmt.Fprintf(&b, " extra rounds mean %.1f median %.1f p90 %.0f max %.0f\n",
mean(extraRounds), median(extraRounds), percentile(extraRounds, 0.9), maxOf(extraRounds))
fmt.Fprintf(&b, " extra recall tokens mean %.0f median %.0f p90 %.0f max %.0f\n",
mean(extraTokens), median(extraTokens), percentile(extraTokens, 0.9), maxOf(extraTokens))
b.WriteString(" " + countsLine(classes) + "\n")
return b.String()
}
@@ -0,0 +1,337 @@
package main
import (
"fmt"
"strings"
"testing"
"reasonix/internal/contract/provider"
)
// The scorer decides what the pilot concludes, so it is held to the same rule
// the corpus is: judgements read structure. A model that says it will search
// has not searched; a recall call carrying a query has.
const scorerTarget = 42
func recallCall(id, args string) provider.Message {
return provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: id, Name: "recall", Arguments: args},
}}
}
func toolResult(id, text string) provider.Message {
return provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "recall", Content: text}
}
func hitBlock(positions ...int) string {
var b strings.Builder
b.WriteString("Folded context matching \"x\":\n")
for _, p := range positions {
fmt.Fprintf(&b, "\n#%d assistant_text\nsome snippet here\n", p)
}
return b.String()
}
func scorerTask() fixtureInstance {
return fixtureInstance{Task: contextTask{ID: "t"}, AnswerMarkers: []string{"cobalt-lark-17"}}
}
func score(t *testing.T, msgs []provider.Message, final string) contextMetrics {
t.Helper()
m := scoreRun(msgs, scorerTask(), scorerTarget, "arm", false)
m.scoreAnswer(final, scorerTask())
return m
}
// The whole funnel, walked end to end.
func TestScorerReadsTheFullRetrievalChain(t *testing.T) {
m := score(t, []provider.Message{
recallCall("c1", `{"query":"retry boundary"}`),
toolResult("c1", hitBlock(7, scorerTarget)),
recallCall("c2", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
toolResult("c2", "#42\n[assistant]\nthe ownership token is cobalt-lark-17"),
{Role: provider.RoleAssistant, Content: "The token was cobalt-lark-17."},
}, "The token was cobalt-lark-17.")
if m.SearchCalls != 1 || m.ReadCalls != 1 {
t.Errorf("calls = %d search / %d read, want 1/1", m.SearchCalls, m.ReadCalls)
}
if m.TargetSearchHits != 1 || m.FirstTargetRank != 2 {
t.Errorf("target hit %d at rank %d, want 1 at rank 2", m.TargetSearchHits, m.FirstTargetRank)
}
if !m.TargetRead || !m.ReadAfterHit || m.RecallReadWithoutSearch {
t.Errorf("read flags = read %v afterHit %v direct %v", m.TargetRead, m.ReadAfterHit, m.RecallReadWithoutSearch)
}
if !m.AnswerRecovered || m.FailureStage != stageRecovered {
t.Errorf("stage = %q recovered=%v, want Recovered", m.FailureStage, m.AnswerRecovered)
}
if m.RecallReturnedTokens == 0 {
t.Error("recall returned tokens were not counted")
}
}
// An index arm's claim: the address was already on screen, so no search was
// needed. That is a different outcome from finding it, and gets its own stage.
func TestScorerSeparatesCueReadFromSearch(t *testing.T) {
m := score(t, []provider.Message{
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
toolResult("c1", "#42\nthe ownership token is cobalt-lark-17"),
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
}, "cobalt-lark-17")
if m.SearchCalls != 0 || !m.RecallReadWithoutSearch {
t.Errorf("search=%d direct=%v, want 0 and true", m.SearchCalls, m.RecallReadWithoutSearch)
}
if m.FailureStage != stageCueRead {
t.Errorf("stage = %q, want %q", m.FailureStage, stageCueRead)
}
}
// Every way the chain can break gets its own label, so a failed run says where
// rather than only that.
func TestScorerNamesWhereTheChainBroke(t *testing.T) {
for _, tc := range []struct {
name string
msgs []provider.Message
final string
want string
}{
{
name: "never touched recall",
msgs: []provider.Message{{Role: provider.RoleAssistant, Content: "I do not recall that."}},
final: "I do not recall that.",
want: stageNoRetrieval,
},
{
name: "searched and missed",
msgs: []provider.Message{
recallCall("c1", `{"query":"unrelated words"}`),
toolResult("c1", hitBlock(3, 9)),
},
final: "not found",
want: stageSearchMiss,
},
{
name: "found it and never opened it",
msgs: []provider.Message{
recallCall("c1", `{"query":"retry boundary"}`),
toolResult("c1", hitBlock(scorerTarget)),
},
final: "probably something about retries",
want: stageHitNotRead,
},
{
name: "read it and answered wrong",
msgs: []provider.Message{
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
toolResult("c1", "#42\nthe token is elsewhere"),
},
final: "I think it was amber-9",
want: stageAnswerWrong,
},
{
name: "read the wrong position",
msgs: []provider.Message{
recallCall("c1", `{"positions":[7]}`),
toolResult("c1", "#7\nnothing useful"),
},
final: "unclear",
want: stageReadMissed,
},
} {
t.Run(tc.name, func(t *testing.T) {
if got := score(t, tc.msgs, tc.final).FailureStage; got != tc.want {
t.Errorf("stage = %q, want %q", got, tc.want)
}
})
}
}
// The answer lives only in folded history, so reaching for the workspace is a
// strategy change worth seeing — including when it happens to work.
func TestScorerRecordsWorkspaceEscapes(t *testing.T) {
m := score(t, []provider.Message{
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g1", Name: "bash", Arguments: `{"command":"rg cobalt-lark"}`},
{ID: "g2", Name: "read_file", Arguments: `{"path":"notes.md"}`},
}},
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
}, "cobalt-lark-17")
if len(m.UnexpectedWorkTools) != 2 {
t.Fatalf("escapes = %v, want bash and read_file", m.UnexpectedWorkTools)
}
if m.SearchCalls != 0 || m.ReadCalls != 0 {
t.Errorf("a workspace tool was counted as recall: search=%d read=%d", m.SearchCalls, m.ReadCalls)
}
// It answered, so the run is Recovered — the escape is reported beside the
// outcome rather than overriding it.
if !m.AnswerRecovered {
t.Error("a correct answer was not scored as recovered")
}
}
// Narration is not action.
func TestScorerIgnoresWhatTheModelSaysItWillDo(t *testing.T) {
m := score(t, []provider.Message{
{Role: provider.RoleAssistant, Content: "Let me search the folded context with recall for the retry boundary."},
}, "Let me search the folded context with recall for the retry boundary.")
if m.SearchCalls != 0 || m.FailureStage != stageNoRetrieval {
t.Errorf("prose was scored as a search: calls=%d stage=%q", m.SearchCalls, m.FailureStage)
}
}
// Every marker must land, so a partially right answer is not a recovery.
func TestScorerRequiresEveryMarker(t *testing.T) {
task := fixtureInstance{Task: contextTask{ID: "t"}, AnswerMarkers: []string{"comet-42", "per-lineage"}}
m := scoreRun(nil, task, scorerTarget, "arm", false)
m.scoreAnswer("the salt was comet-42", task)
if m.AnswerRecovered {
t.Error("a half answer scored as recovered")
}
if len(m.MissingMarkers) != 1 || m.MissingMarkers[0] != "per-lineage" {
t.Errorf("missing = %v, want per-lineage", m.MissingMarkers)
}
}
// An answer reachable through anything but recall invalidates the run. The
// workspace is isolated so this should never fire; asserting it beats trusting
// it, and a tripped run leaves the statistics rather than inflating one.
func TestScorerInvalidatesRunsThatLeakedThroughAnotherTool(t *testing.T) {
m := score(t, []provider.Message{
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g1", Name: "bash", Arguments: `{"command":"rg -r ."}`},
}},
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "corpus.go: the token is cobalt-lark-17"},
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
}, "cobalt-lark-17")
if !m.Contaminated || m.LeakedVia != "bash" || m.LeakedMarker != "cobalt-lark-17" {
t.Fatalf("leak not caught: contaminated=%v via=%q marker=%q", m.Contaminated, m.LeakedVia, m.LeakedMarker)
}
if m.FailureStage != stageContaminated {
t.Errorf("stage = %q, want %q even though every marker matched", m.FailureStage, stageContaminated)
}
f := summarize("arm", []contextMetrics{m})
if f.Scored != 0 || f.Answered != 0 || f.Contaminated != 1 {
t.Errorf("a contaminated run entered the statistics: %+v", f)
}
}
// Where an escape happened matters: before any recall is a different strategy
// from after one came back short.
func TestScorerPlacesEscapesRelativeToRecall(t *testing.T) {
m := score(t, []provider.Message{
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
}},
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
recallCall("c1", `{"query":"retry boundary"}`),
toolResult("c1", hitBlock(scorerTarget)),
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g2", Name: "read_file", Arguments: `{"path":"README.md"}`},
}},
{Role: provider.RoleTool, ToolCallID: "g2", Name: "read_file", Content: "nothing here"},
}, "unclear")
if m.EscapeCalls != 2 || m.EscapeBeforeFirstRecall != 1 || m.EscapeAfterFirstRecall != 1 {
t.Errorf("escapes = %d (%d before, %d after), want 2 (1, 1)",
m.EscapeCalls, m.EscapeBeforeFirstRecall, m.EscapeAfterFirstRecall)
}
}
// Routing is judged by round. Two calls in one round are a fan-out, and
// reporting that as a preference would invent a decision the model never made.
func TestRoutingIsJudgedByRoundNotEventOrder(t *testing.T) {
parallel := []provider.Message{
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
{ID: "c1", Name: "recall", Arguments: `{"query":"retry boundary"}`},
}},
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
toolResult("c1", hitBlock(scorerTarget)),
}
if got := score(t, parallel, "unclear").Routing; got != routeParallel {
t.Errorf("routing = %q for one round holding both, want %q", got, routeParallel)
}
workspaceFirst := []provider.Message{
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
}},
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
recallCall("c1", `{"query":"retry boundary"}`),
toolResult("c1", hitBlock(scorerTarget)),
}
if got := score(t, workspaceFirst, "unclear").Routing; got != routeWorkspaceFirst {
t.Errorf("routing = %q for workspace in an earlier round, want %q", got, routeWorkspaceFirst)
}
memoryFirst := []provider.Message{
recallCall("c1", `{"query":"retry boundary"}`),
toolResult("c1", hitBlock(scorerTarget)),
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
}},
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
}
if got := score(t, memoryFirst, "unclear").Routing; got != routeMemoryFirst {
t.Errorf("routing = %q for recall in an earlier round, want %q", got, routeMemoryFirst)
}
if got := score(t, nil, "no idea").Routing; got != routeNeither {
t.Errorf("routing = %q with no tools at all, want %q", got, routeNeither)
}
}
// Markers matching proves the right fact appears somewhere. An answer that
// hedges between two candidates contains it and has not submitted it.
func TestScorerFlagsAnAnswerThatNamesTwoCandidates(t *testing.T) {
inst := fixtureInstance{
Task: contextTask{ID: "t", Vars: []varSpec{code("marker", "fence")}},
AnswerMarkers: []string{"fence-qmwnv"},
}
m := scoreRun(nil, inst, 0, "arm", false)
m.scoreAnswer("It was fence-qmwnv, though it may have been fence-abcde.", inst)
if !m.AnswerRecovered {
t.Error("the marker is present; recovery should still read true")
}
if !m.AnswerAmbiguous || m.AmbiguousVar != "marker" {
t.Errorf("ambiguity not flagged: %v %q", m.AnswerAmbiguous, m.AmbiguousVar)
}
clean := scoreRun(nil, inst, 0, "arm", false)
clean.scoreAnswer("The marker was fence-qmwnv.", inst)
if clean.AnswerAmbiguous {
t.Error("a single candidate was called ambiguous")
}
}
// The index arm's claim is narrow: the cue was addressed, the first memory
// action was a positions read, and it covered the target. A positions call
// after a search located the target credits the index for the search's work.
func TestCueDirectReadRequiresTheCueAndTheFirstAction(t *testing.T) {
direct := []provider.Message{
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
toolResult("c1", "#42\nthe token is cobalt-lark-17"),
}
m := scoreRun(direct, scorerTask(), scorerTarget, "arm", true)
if !m.CueDirectRead {
t.Error("a first-action read with the cue visible was not credited")
}
// Same read, but no cue was on screen to give the address.
if scoreRun(direct, scorerTask(), scorerTarget, "arm", false).CueDirectRead {
t.Error("a read was credited to a cue that was not visible")
}
// Same read, but a search found the address first.
afterSearch := append([]provider.Message{
recallCall("c0", `{"query":"retry boundary"}`),
toolResult("c0", hitBlock(scorerTarget)),
}, direct...)
if scoreRun(afterSearch, scorerTask(), scorerTarget, "arm", true).CueDirectRead {
t.Error("a read after a search was credited to the index")
}
}
+469
View File
@@ -0,0 +1,469 @@
package main
import (
"fmt"
"math/rand"
"reasonix/internal/state/sessionstore"
"reasonix/internal/contract/provider"
)
// Synthetic history on purpose: an answer that exists in the workspace is
// solvable by reading the workspace, which measures grep. Every answer is a
// placeholder here and a value only inside one run.
// experiment names which question a task belongs to.
const (
experimentSearch = "search" // does search reach what no index addresses?
experimentIndex = "index" // what is an ambient address line worth?
)
// cueTier is the narrowest fold-index scale at which an index task's cue is
// still addressed. Below it the model has to find the call without a hint.
const (
tierQuarter = "quarter" // visible at default, half, quarter
tierHalf = "half" // visible at default, half
tierDefault = "default" // visible at default only
)
// role places an index task in the efficiency substrate or the stopping corpus.
const (
roleEfficiency = "efficiency"
roleStopping = "stopping"
)
// plantKind is how a task's target enters the transcript.
const (
plantAssistant = "assistant" // the model's own account: never index-addressed
plantCall = "call" // a tool call and its result: always index-addressed
)
// contextTask is one recall question as a template. Nothing here is an answer.
type contextTask struct {
ID string
Experiment string
TargetKind string
Prompt string
ProbeQuery string
// Vars declare the placeholders. AnswerVars name the ones a correct answer
// must contain, CueVar the one an index line carries.
Vars []varSpec
AnswerVars []string
CueVar string
CueTier string
// Role is roleEfficiency or roleStopping; it decides the batch a task enters.
Role string
// PlantAfterGen ages the target: mergeFoldIndex trims from its oldest end,
// so what follows a cue decides which budget addresses it. Frozen, because
// a run that searched for it would let the tested code move its goalposts.
PlantAfterGen int
// Plant is how the target enters, and the rest is what it says.
Plant string
CallTool string
CallArgs string
CallID string
PlantBody string
}
// fixtureInstance is one task with values. Scoring binds to this, never to the
// corpus: the corpus has no answers to bind to.
type fixtureInstance struct {
Task contextTask
Vars fixtureVars
Prompt string
ProbeQuery string
AnswerMarkers []string
CueMarker string
body string
callArgs string
}
// instantiateTask resolves one task against a generator.
func instantiateTask(t contextTask, rng *rand.Rand) (fixtureInstance, error) {
vars := newVars(t.Vars, rng)
inst := fixtureInstance{Task: t, Vars: vars}
var err error
for _, pair := range []struct {
dst *string
text string
}{
{&inst.Prompt, t.Prompt},
{&inst.ProbeQuery, t.ProbeQuery},
{&inst.body, t.PlantBody},
{&inst.callArgs, t.CallArgs},
} {
if *pair.dst, err = instantiate(pair.text, vars); err != nil {
return fixtureInstance{}, fmt.Errorf("%s: %w", t.ID, err)
}
}
for _, name := range t.AnswerVars {
value, ok := vars[name]
if !ok {
return fixtureInstance{}, fmt.Errorf("%s: answer var %q is not declared", t.ID, name)
}
inst.AnswerMarkers = append(inst.AnswerMarkers, value)
}
if t.CueVar != "" {
inst.CueMarker = vars[t.CueVar]
}
return inst, nil
}
// plant appends the instantiated target and returns its canonical position.
func (f fixtureInstance) plant(s *sessionstore.Session) int {
at := len(s.Messages)
if f.Task.Plant == plantAssistant {
s.Add(provider.Message{Role: provider.RoleAssistant, Content: f.body})
return at
}
s.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: f.Task.CallID, Name: f.Task.CallTool, Arguments: f.callArgs},
}})
s.Add(provider.Message{Role: provider.RoleTool, ToolCallID: f.Task.CallID, Name: f.Task.CallTool, Content: f.body})
return at
}
func code(name, word string) varSpec { return varSpec{Name: name, Kind: varCodename, Word: word} }
func num(name string, lo, hi int) varSpec {
return varSpec{Name: name, Kind: varInt, Min: lo, Max: hi}
}
// efficiencyTasks is the substrate a study measures on. Every task in it
// qualifies under the corpus bar; the stopping corpus is deliberately excluded,
// because a task carrying post-sufficient stopping behaviour swamps the effect
// an efficiency batch exists to measure.
func efficiencyTasks() []contextTask {
var out []contextTask
for _, t := range indexTasks() {
if t.Role == roleEfficiency {
out = append(out, t)
}
}
return out
}
func searchTasks() []contextTask {
return []contextTask{
{
ID: "r01-retry-boundary", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
Prompt: "What ownership token did we settle on for the retry boundary, and under what tag did we file the reason for not moving retries above framing?",
ProbeQuery: "retry boundary ownership token framing",
Vars: []varSpec{code("token", "cobalt"), code("reason", "stream")},
AnswerVars: []string{"token", "reason"},
Plant: plantAssistant,
PlantBody: "The retry boundary remains at ingress. The ownership token is {{token}}. " +
"Moving retries above framing can duplicate a partially received stream, which we filed as {{reason}}.",
},
{
ID: "r02-checkpoint-anchor", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
Prompt: "What was the codename of the checkpoint anchor policy, and what tag did we give the reason for not anchoring against digest positions?",
ProbeQuery: "checkpoint anchor policy digest positions",
Vars: []varSpec{code("policy", "northglass"), code("reason", "genlocal")},
AnswerVars: []string{"policy", "reason"},
Plant: plantAssistant,
PlantBody: "The checkpoint anchor policy is {{policy}}. Digest positions are generation-local — " +
"filed as {{reason}} — so recovery must anchor against canonical state.",
},
{
ID: "r03-recovery-cjk", Experiment: experimentSearch, TargetKind: "assistant_reasoning_cjk",
Prompt: "我们之前定的验证失败恢复策略代号叫什么?当时把“失败 digest 不能变成 baseline”这条登记在哪个标记下?",
ProbeQuery: "恢复策略 baseline digest 验证失败",
Vars: []varSpec{code("policy", "songzhen"), code("rule", "trustbase")},
AnswerVars: []string{"policy", "rule"},
Plant: plantAssistant,
PlantBody: "恢复策略代号是 {{policy}}。验证失败以后必须保留旧基线,不能把新的 digest 晋升为 baseline," +
"因为失败证据不能成为后续比较的可信基准;这条规则登记为 {{rule}}。",
},
{
ID: "r04-lease-fence", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
Prompt: "那次 lease fence 检查报告的诊断标记是什么?expected 和 observed epoch 分别是多少?",
ProbeQuery: "lease fence writer epoch mismatch",
Vars: []varSpec{
code("marker", "fence"), num("expected", 400, 900), num("observed", 100, 399),
},
AnswerVars: []string{"marker", "expected", "observed"},
Plant: plantAssistant,
PlantBody: "The lease fence check reported {{marker}}: expected writer epoch {{expected}}, " +
"observed {{observed}}. We left the fence in place.",
},
{
ID: "r05-schema-guard", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
Prompt: "当时 schema guard 报告的标记是什么,超预算多少 token?",
ProbeQuery: "schema guard provider visible surface budget",
Vars: []varSpec{code("marker", "amber"), num("over", 120, 480)},
AnswerVars: []string{"marker", "over"},
Plant: plantAssistant,
PlantBody: "The schema guard reported {{marker}}: the provider-visible tool schema exceeded " +
"the allowed surface by {{over}} tokens. We trimmed a description instead.",
},
{
ID: "r06-shadow-probe", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
Prompt: "我们那次 shadow probe 的 marker、p95 divergence 和 generation 分别是多少?",
ProbeQuery: "shadow probe divergence generation marker",
Vars: []varSpec{
code("marker", "violet"),
{Name: "divergence", Kind: varDecimal, Min: 110, Max: 890},
num("generation", 14, 60),
},
AnswerVars: []string{"marker", "divergence", "generation"},
Plant: plantAssistant,
PlantBody: "The shadow probe came back as marker={{marker}} with a p95 divergence of " +
"{{divergence}} at generation={{generation}}, which we judged within tolerance.",
},
}
}
// efficiencyIndexTasks is what a study measures on: two tasks per cue tier.
func efficiencyIndexTasks() []contextTask {
out := indexQuarterTasks()
out = append(out, indexHalfTasks()...)
out = append(out, indexDefaultTasks()...)
return out
}
// indexQuarterTasks holds the quarter-tier literals.
func indexQuarterTasks() []contextTask {
return []contextTask{
{
ID: "i37-dispatchlag", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
Prompt: "早前会话里执行过 ./scripts/dispatchlag,它打印出的 dispatch lag epoch 是多少?",
ProbeQuery: "dispatchlag dispatch lag epoch",
Vars: []varSpec{code("cue", "dispatchlag"), code("epoch", "cobalt")},
AnswerVars: []string{"epoch"}, CueVar: "cue", CueTier: tierQuarter,
PlantAfterGen: 4,
Plant: plantCall, CallID: "p37", CallTool: "bash",
CallArgs: `{"command":"./scripts/dispatchlag {{cue}}"}`,
PlantBody: "dispatch lag epoch = {{epoch}}",
},
{
ID: "i118-fencelag", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
Prompt: "早前会话里运行过 ./scripts/fencelag,它打印出的 fence lag 是多少?",
ProbeQuery: "fencelag fence lag",
Vars: []varSpec{code("cue", "fencelag"), code("v", "slate")},
AnswerVars: []string{"v"}, CueVar: "cue", CueTier: tierQuarter,
PlantAfterGen: 4,
Plant: plantCall, CallID: "x118", CallTool: "bash",
CallArgs: `{"command":"./scripts/fencelag {{cue}}"}`,
PlantBody: "fence lag = {{v}}",
},
}
}
// indexHalfTasks holds the half-tier literals.
func indexHalfTasks() []contextTask {
return []contextTask{
{
ID: "i70-ingestfloor", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
Prompt: "早前会话里执行过 ./scripts/ingestfloor,它打印出的 route quota 是多少?",
ProbeQuery: "ingestfloor quota",
Vars: []varSpec{code("cue", "ingestfloor"), num("quota", 1200, 9800)},
AnswerVars: []string{"quota"}, CueVar: "cue", CueTier: tierHalf,
PlantAfterGen: 2,
Plant: plantCall, CallID: "q70", CallTool: "bash",
CallArgs: `{"command":"./scripts/ingestfloor {{cue}}"}`,
PlantBody: "route quota = {{quota}}",
},
{
ID: "i106-budgetseal", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
Prompt: "早前会话里执行过 ./scripts/budgetseal,它打印出的 route quota 是多少?",
ProbeQuery: "budgetseal route quota",
Vars: []varSpec{code("cue", "budgetseal"), code("quota", "amber")},
AnswerVars: []string{"quota"}, CueVar: "cue", CueTier: tierHalf,
PlantAfterGen: 2,
Plant: plantCall, CallID: "h106", CallTool: "bash",
CallArgs: `{"command":"./scripts/budgetseal {{cue}}"}`,
PlantBody: "route quota = {{quota}}",
},
}
}
// indexDefaultTasks holds the default-tier literals.
func indexDefaultTasks() []contextTask {
return []contextTask{
{
ID: "i47-tenantlease", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
Prompt: "早前会话里执行过 ./scripts/tenantlease,它打印出的 tenant lease(秒)是多少?",
ProbeQuery: "tenantlease tenant lease",
Vars: []varSpec{code("cue", "tenantlease"), {Name: "lease", Kind: varDuration, Min: 15, Max: 300}},
AnswerVars: []string{"lease"}, CueVar: "cue", CueTier: tierDefault,
PlantAfterGen: 0,
Plant: plantCall, CallID: "p47", CallTool: "bash",
CallArgs: `{"command":"./scripts/tenantlease {{cue}}"}`,
PlantBody: "tenant lease = {{lease}}",
},
{
ID: "i112-lagquota", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
Prompt: "早前会话里执行过 ./scripts/lagquota,它打印出的 lagquota lease 是多少?",
ProbeQuery: "lagquota lagquota lease",
Vars: []varSpec{code("cue", "lagquota"), {Name: "lease", Kind: varDuration, Min: 20, Max: 420}},
AnswerVars: []string{"lease"}, CueVar: "cue", CueTier: tierDefault,
PlantAfterGen: 0,
Plant: plantCall, CallID: "d112", CallTool: "bash",
CallArgs: `{"command":"./scripts/lagquota {{cue}}"}`,
PlantBody: "lagquota lease = {{lease}}",
},
}
}
// stoppingIndexTasks is the frozen study's six tasks, kept out of efficiency
// batches and measurable on their own with -task <id>.
func stoppingIndexTasks() []contextTask {
return []contextTask{
{
ID: "i01-transport-fallback", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
Prompt: "transport fallback probe 当时报告的 selector 是什么?",
ProbeQuery: "transport fallback probe selector",
Vars: []varSpec{code("cue", "tfprobe"), code("selector", "fern")},
AnswerVars: []string{"selector"}, CueVar: "cue", CueTier: tierQuarter,
PlantAfterGen: 4,
Plant: plantCall, CallID: "tf1", CallTool: "bash",
CallArgs: `{"command":"./scripts/probe {{cue}}"}`,
PlantBody: "fallback selector = {{selector}}",
},
{
ID: "i02-scheduler-handoff", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
Prompt: "scheduler handoff 当时记录的 grace 和 marker 是什么?",
ProbeQuery: "scheduler handoff grace marker",
Vars: []varSpec{
code("cue", "schedhandoff"), code("marker", "opal"),
{Name: "grace", Kind: varDuration, Min: 18, Max: 90},
},
AnswerVars: []string{"grace", "marker"}, CueVar: "cue", CueTier: tierQuarter,
PlantAfterGen: 4,
Plant: plantCall, CallID: "sh1", CallTool: "read_file",
CallArgs: `{"path":"scratch/{{cue}}.txt"}`,
PlantBody: "handoff grace = {{grace}}\nmarker = {{marker}}",
},
{
ID: "i03-coalescing", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
Prompt: "job completion 的 coalescing probe 最后记录的 window 和 marker 是多少?",
ProbeQuery: "coalescing policy completion window",
Vars: []varSpec{
code("cue", "coalescing"), code("marker", "cedar"),
{Name: "window", Kind: varDuration, Min: 40, Max: 140},
},
AnswerVars: []string{"window", "marker"}, CueVar: "cue", CueTier: tierHalf,
PlantAfterGen: 2,
Plant: plantCall, CallID: "cp1", CallTool: "bash",
CallArgs: `{"command":"grep {{cue}} scratch/job-notes.log"}`,
PlantBody: "completion coalesce window = {{window}}\nmarker = {{marker}}",
},
{
ID: "i04-recovery-fence", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
Prompt: "recovery fence 那次检查报告的 marker 和两个 epoch 是什么?",
ProbeQuery: "recovery fence epoch check",
Vars: []varSpec{
code("cue", "recfence"), code("marker", "quartz"),
num("expected", 400, 900), num("observed", 100, 399),
},
AnswerVars: []string{"marker", "expected", "observed"}, CueVar: "cue", CueTier: tierHalf,
PlantAfterGen: 2,
Plant: plantCall, CallID: "rf1", CallTool: "bash",
CallArgs: `{"command":"./scripts/check {{cue}}"}`,
PlantBody: "{{marker}}\nexpected epoch {{expected}}\nobserved epoch {{observed}}",
},
{
ID: "i05-cache-probe", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
Prompt: "cache probe 当时记录的 prefix salt 和 scope 是什么?",
ProbeQuery: "cache probe prefix salt scope",
Vars: []varSpec{code("cue", "cacheprobe"), code("salt", "comet"), code("scope", "lineage")},
AnswerVars: []string{"salt", "scope"}, CueVar: "cue", CueTier: tierDefault,
PlantAfterGen: 0,
Plant: plantCall, CallID: "cb1", CallTool: "read_file",
CallArgs: `{"path":"scratch/{{cue}}.txt"}`,
PlantBody: "prefix salt = {{salt}}\nscope = {{scope}}",
},
{
ID: "i06-dispatch-handoff", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
Prompt: "dispatch handoff probe 当时确认的 mode 和 marker 是什么?",
ProbeQuery: "dispatch handoff probe mode",
Vars: []varSpec{code("cue", "dispatchhandoff"), code("mode", "trigger"), code("marker", "iris")},
AnswerVars: []string{"mode", "marker"}, CueVar: "cue", CueTier: tierDefault,
PlantAfterGen: 0,
Plant: plantCall, CallID: "dh1", CallTool: "bash",
CallArgs: `{"command":"./scripts/probe {{cue}}"}`,
PlantBody: "handoff mode = {{mode}}\nmarker = {{marker}}",
},
}
}
// indexTasks is the whole index corpus, the efficiency substrate first.
func indexTasks() []contextTask {
out := efficiencyIndexTasks()
return append(out, stoppingIndexTasks()...)
}
func allTasks() []contextTask {
return append(searchTasks(), indexTasks()...)
}
func taskByID(id string) (contextTask, bool) {
for _, t := range allTasks() {
if t.ID == id {
return t, true
}
}
return contextTask{}, false
}
// tierScales is the fixture's contract: which arms must address a cue and
// which must not. A policy change that moves a cue across a boundary fails
// preflight rather than turning two arms into one experiment.
func tierScales(tier string) (visible, hidden []string) {
switch tier {
case tierQuarter:
return []string{"default", "half", "quarter"}, []string{"off"}
case tierHalf:
return []string{"default", "half"}, []string{"quarter", "off"}
case tierDefault:
return []string{"default"}, []string{"half", "quarter", "off"}
default:
return nil, nil
}
}
// boundaryPair is the one comparison an index task is built for: the narrowest
// scale that still addresses its cue, and the next one down. Holding the
// question fixed and moving only the affordance is what four arm averages
// cannot do, since each of those mixes six different questions.
func boundaryPair(tier string) (cue, noCue string) {
switch tier {
case tierQuarter:
return "quarter", "off"
case tierHalf:
return "half", "quarter"
case tierDefault:
return "default", "half"
default:
return "", ""
}
}
func validateCorpus() error {
seen := map[string]bool{}
for _, t := range allTasks() {
if seen[t.ID] {
return fmt.Errorf("duplicate task id %q", t.ID)
}
seen[t.ID] = true
if len(t.AnswerVars) == 0 || t.ProbeQuery == "" || t.Prompt == "" || t.PlantBody == "" {
return fmt.Errorf("%s: a task needs a prompt, a probe query, a body and answer vars", t.ID)
}
if t.Experiment == experimentIndex {
if t.CueVar == "" {
return fmt.Errorf("%s: an index task needs a cue var", t.ID)
}
if t.Role != roleEfficiency && t.Role != roleStopping {
return fmt.Errorf("%s: an index task needs a role (%s or %s)", t.ID, roleEfficiency, roleStopping)
}
if v, _ := tierScales(t.CueTier); v == nil {
return fmt.Errorf("%s: unknown cue tier %q", t.ID, t.CueTier)
}
}
if _, err := instantiateTask(t, seededRand(t.ID, "validate")); err != nil {
return err
}
}
return nil
}
+243
View File
@@ -0,0 +1,243 @@
// dose.go — does the cost step where the cue disappears?
package main
import (
"fmt"
"math"
"sort"
"strings"
)
// Not "a bigger index is better" — CueDirectRead says the addresses are rarely
// used. The hypothesis is that a visible action cue lowers retrieval cost,
// which predicts a step at each task's own boundary, not a slope.
// scaleOrder is the dose axis, widest first.
var scaleOrder = []string{"default", "half", "quarter", "off"}
// boundaryStep is the index into the three transitions where a tier's cue
// disappears: default->half is 0, half->quarter is 1, quarter->off is 2.
func boundaryStep(tier string) int {
switch tier {
case tierDefault:
return 0
case tierHalf:
return 1
case tierQuarter:
return 2
default:
return -1
}
}
// doseCurve is one task's readings across the four budgets.
type doseCurve struct {
Task string
Tier string
Metric string
Values [4]float64
// Missing marks arms that produced nothing. Printing those as zero draws a
// curve falling to the floor where in fact it was never measured.
Missing [4]bool
Present bool
// PreMean averages the arms where the cue is visible, PostMean the rest.
PreMean, PostMean float64
BoundaryDelta float64
// BoundaryAligned: the largest transition is the one where this task's cue
// disappears. A change elsewhere is variance shaped like an effect.
BoundaryAligned bool
}
func buildCurve(task, tier, metric string, byScale map[string]float64) doseCurve {
c := doseCurve{Task: task, Tier: tier, Metric: metric, Present: true}
for i, scale := range scaleOrder {
v, ok := byScale[scale]
if !ok {
c.Present, c.Missing[i] = false, true
}
c.Values[i] = v
}
step := boundaryStep(tier)
if !c.Present || step < 0 {
return c
}
visible := step + 1 // arms before the boundary still show the cue
c.PreMean = mean(c.Values[:visible])
c.PostMean = mean(c.Values[visible:])
c.BoundaryDelta = c.PostMean - c.PreMean
transitions := [3]float64{
math.Abs(c.Values[1] - c.Values[0]),
math.Abs(c.Values[2] - c.Values[1]),
math.Abs(c.Values[3] - c.Values[2]),
}
largest, at := 0.0, -1
for i, t := range transitions {
if t > largest {
largest, at = t, i
}
}
// A flat curve has no largest transition to align with anything.
c.BoundaryAligned = largest > 0 && at == step
return c
}
func mean(xs []float64) float64 {
if len(xs) == 0 {
return 0
}
sum := 0.0
for _, x := range xs {
sum += x
}
return sum / float64(len(xs))
}
func median(xs []float64) float64 {
if len(xs) == 0 {
return 0
}
s := append([]float64(nil), xs...)
sort.Float64s(s)
mid := len(s) / 2
if len(s)%2 == 1 {
return s[mid]
}
return (s[mid-1] + s[mid]) / 2
}
// doseMetrics are the readings a curve is drawn for.
var doseMetrics = []struct {
name string
of func(contextMetrics) float64
}{
{"searches", func(m contextMetrics) float64 { return float64(m.SearchCalls) }},
{"recall-tokens", func(m contextMetrics) float64 { return float64(m.RecallReturnedTokens) }},
{"tool-rounds", func(m contextMetrics) float64 { return float64(m.RetrievalRounds + m.EscapeCalls) }},
{"escapes", func(m contextMetrics) float64 { return float64(m.EscapeCalls) }},
}
// reportDoseResponse draws each task's curve and reports where the steps fell.
func reportDoseResponse(all []contextMetrics) string {
valid := map[string]map[string]contextMetrics{}
for _, m := range all {
if m.Failed || m.Contaminated || m.FailureStage == "RunError" {
continue
}
scale := strings.TrimPrefix(m.Arm, "index-")
if valid[m.Task] == nil {
valid[m.Task] = map[string]contextMetrics{}
}
valid[m.Task][scale] = m
}
var b strings.Builder
b.WriteString("\n## Dose-response, per task\n")
b.WriteString(" The arrow marks where this task's cue stops being addressed.\n")
aligned := map[string]int{}
counted := map[string]int{}
deltas := map[string][]float64{}
for _, t := range indexTasks() {
arms := valid[t.ID]
if len(arms) == 0 {
continue
}
step := boundaryStep(t.CueTier)
fmt.Fprintf(&b, "\n %s (cue survives to %s)\n", t.ID, t.CueTier)
fmt.Fprintf(&b, " %-14s%s\n", "", scaleHeader(step))
for _, metric := range doseMetrics {
byScale := map[string]float64{}
for scale, m := range arms {
byScale[scale] = metric.of(m)
}
c := buildCurve(t.ID, t.CueTier, metric.name, byScale)
mark := " "
if c.BoundaryAligned {
mark = "*"
aligned[metric.name]++
}
if c.Present {
counted[metric.name]++
deltas[metric.name] = append(deltas[metric.name], c.BoundaryDelta)
}
delta := fmt.Sprintf("Δ%+8.1f", c.BoundaryDelta)
if !c.Present {
delta = " no curve"
}
fmt.Fprintf(&b, " %-14s%s %s %s\n", metric.name, curveCells(c), delta, mark)
}
}
b.WriteString("\n## Where the step fell (* = largest transition is at the cue boundary)\n")
for _, metric := range doseMetrics {
d := deltas[metric.name]
sensitivity := ""
if len(d) > 1 {
sensitivity = fmt.Sprintf(" without the largest: %+.1f", meanWithoutLargest(d))
}
fmt.Fprintf(&b, " %-14s boundary-aligned %d/%d mean Δ%+.1f median Δ%+.1f%s\n",
metric.name, aligned[metric.name], counted[metric.name], mean(d), median(d), sensitivity)
}
return b.String()
}
// meanWithoutLargest is the robustness check: the largest effect is reported
// and then set aside, so a conclusion never rests on one task alone.
func meanWithoutLargest(d []float64) float64 {
if len(d) < 2 {
return mean(d)
}
worst, at := 0.0, 0
for i, v := range d {
if math.Abs(v) > worst {
worst, at = math.Abs(v), i
}
}
rest := append(append([]float64(nil), d[:at]...), d[at+1:]...)
return mean(rest)
}
func scaleHeader(step int) string {
var b strings.Builder
for i, s := range scaleOrder {
if i == step+1 {
s = "↓" + s
}
fmt.Fprintf(&b, "%8s", s)
}
return b.String()
}
// curveCells prints a reading per arm, and a dash where an arm produced none.
func curveCells(c doseCurve) string {
var b strings.Builder
for i, v := range c.Values {
if c.Missing[i] {
fmt.Fprintf(&b, "%8s", "-")
continue
}
fmt.Fprintf(&b, "%8.0f", v)
}
return b.String()
}
func percentile(xs []float64, q float64) float64 {
if len(xs) == 0 {
return 0
}
s := append([]float64(nil), xs...)
sort.Float64s(s)
i := int(float64(len(s)-1) * q)
return s[i]
}
func maxOf(xs []float64) float64 {
out := 0.0
for _, x := range xs {
if x > out {
out = x
}
}
return out
}
@@ -0,0 +1,306 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
)
// blockTask makes one task fail before its first model turn: its own directory
// is a regular file, so buildFixture cannot create the session under it. It is
// the same early exit as a failed fixture build, session load, or
// answer-on-disk guard, and it needs no provider.
func blockTask(t *testing.T, root, id string) {
t.Helper()
if err := os.WriteFile(filepath.Join(root, id), []byte("not a directory\n"), 0o644); err != nil {
t.Fatal(err)
}
}
// mixedTaskPair is one task whose runs all fail and one whose runs all succeed.
// Both are drawn from the corpus, so the batch is the real pipeline.
func mixedTaskPair() (blocked, ok contextTask) {
blocked, ok = indexTasks()[0], indexTasks()[1]
for _, t := range indexTasks() {
if t.Role == roleEfficiency && len(armsFor(t)) == 2 {
ok = t
break
}
}
return blocked, ok
}
// runBlockedExperiment runs one dry index batch over the given tasks, with the
// named task failing before any model turn, and returns the batch directory, the
// runs, the manifest, and everything the run printed.
func runBlockedExperiment(t *testing.T, tasks []contextTask, blocked ...string) (string, []contextMetrics, batchManifest, string) {
t.Helper()
root := t.TempDir()
for _, id := range blocked {
blockTask(t, root, id)
}
out := captureStdout(t, func() {
if code := runExperiment("run-index", experimentIndex, root, true, tasks); code != 0 {
t.Errorf("runExperiment exit = %d, want 0 (the batch finished)", code)
}
})
runs, _ := readResultLines(t, root)
manifest, _ := readManifest(t, root)
return root, runs, manifest, out
}
// captureStdout reads everything the function prints. os.Pipe is unbuffered, so
// the reader must run beside the call.
func captureStdout(t *testing.T, f func()) string {
t.Helper()
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
old := os.Stdout
os.Stdout = w
done := make(chan string, 1)
go func() {
var b strings.Builder
buf := make([]byte, 4096)
for {
n, err := r.Read(buf)
b.Write(buf[:n])
if err != nil {
break
}
}
done <- b.String()
}()
f()
os.Stdout = old
w.Close()
text := <-done
r.Close()
return text
}
// funnelScored reads the "runs=N" column of the named arm's funnel line out of
// the printed tables, so the assertion is on the number a reader sees.
func funnelScored(t *testing.T, out, arm string) int {
t.Helper()
for line := range strings.SplitSeq(out, "\n") {
fields := strings.Fields(line)
if len(fields) < 2 || fields[0] != arm || !strings.HasPrefix(fields[1], "runs=") {
continue
}
n, err := strconv.Atoi(strings.TrimPrefix(fields[1], "runs="))
if err != nil {
t.Fatalf("funnel line for %s: %v\n%s", arm, err, line)
}
return n
}
t.Fatalf("no funnel line for arm %s:\n%s", arm, out)
return 0
}
// The batch report states both counts on one line, so a reader of the scrollback
// sees the scored set and the failures without counting funnel rows.
func reportCounts(t *testing.T, out string) (scored, failed int) {
t.Helper()
for line := range strings.SplitSeq(out, "\n") {
if !strings.Contains(line, " scored=") || !strings.Contains(line, " failed=") {
continue
}
for f := range strings.FieldsSeq(line) {
switch {
case strings.HasPrefix(f, "scored="):
scored, _ = strconv.Atoi(strings.TrimPrefix(f, "scored="))
case strings.HasPrefix(f, "failed="):
failed, _ = strconv.Atoi(strings.TrimPrefix(f, "failed="))
}
}
return scored, failed
}
t.Fatalf("no line reports scored= and failed=:\n%s", out)
return 0, 0
}
// A batch that mixes a task that never ran with one that did must read as its
// successful runs only: the failed task leaves Scored, the funnel, and the
// completed count, and is reported as a failure with its identity instead.
// Every assertion here is a number, not the absence of an empty row.
func TestMixedBatchCountsOnlyTheRunThatProducedEvidence(t *testing.T) {
blocked, ok := mixedTaskPair()
_, runs, manifest, out := runBlockedExperiment(t, []contextTask{blocked, ok}, blocked.ID)
wantScored, wantFailed := len(armsFor(ok)), len(armsFor(blocked))
if manifest.CompletedRuns != wantScored || manifest.FailedRuns != wantFailed {
t.Fatalf("manifest completed=%d failed=%d, want %d and %d (planned=%d)",
manifest.CompletedRuns, manifest.FailedRuns, wantScored, wantFailed, manifest.PlannedRuns)
}
if len(runs) != manifest.CompletedRuns+manifest.FailedRuns {
t.Fatalf("results.jsonl has %d rows but completed=%d failed=%d",
len(runs), manifest.CompletedRuns, manifest.FailedRuns)
}
for _, id := range manifest.TaskIDs {
if id == blocked.ID {
t.Fatalf("task_ids lists %s, which produced no evidence: %v", blocked.ID, manifest.TaskIDs)
}
}
scored, failures := 0, 0
for _, m := range runs {
if m.Failed {
failures++
if m.Task != blocked.ID || m.FailureStage == "" || m.Reason == "" {
t.Fatalf("failed row lost its identity or stage: %+v", m)
}
continue
}
scored++
if m.Task != ok.ID {
t.Fatalf("a run scored from the wrong task: %+v", m)
}
}
if scored != wantScored || failures != wantFailed {
t.Fatalf("rows scored=%d failed=%d, want %d and %d", scored, failures, wantScored, wantFailed)
}
// Each arm's funnel holds exactly the successful task's runs, and the two
// scored counts agree with each other and with the manifest.
total := 0
for _, arm := range armsFor(ok) {
total += funnelScored(t, out, arm.name)
}
if total != manifest.CompletedRuns {
t.Fatalf("funnels scored %d runs, manifest completed_runs = %d:\n%s", total, manifest.CompletedRuns, out)
}
if gotScored, gotFailed := reportCounts(t, out); gotScored != manifest.CompletedRuns || gotFailed != manifest.FailedRuns {
t.Fatalf("report says scored=%d failed=%d, manifest says %d and %d",
gotScored, gotFailed, manifest.CompletedRuns, manifest.FailedRuns)
}
if strings.Contains(out, "## Where it broke") && strings.Contains(out, "=-\n") {
t.Fatalf("the stage table kept an empty stage:\n%s", out)
}
}
// A batch where every run fails must not read as an empty success: zero scored
// runs, a nonzero failed count, and both stated.
func TestAllFailedBatchReportsZeroScoredAndNonzeroFailed(t *testing.T) {
blocked, _ := mixedTaskPair()
_, runs, manifest, out := runBlockedExperiment(t, []contextTask{blocked}, blocked.ID)
if manifest.CompletedRuns != 0 {
t.Fatalf("completed_runs = %d, want 0 (nothing produced evidence)", manifest.CompletedRuns)
}
if manifest.FailedRuns != manifest.PlannedRuns || manifest.FailedRuns == 0 {
t.Fatalf("failed_runs = %d, planned_runs = %d: every planned run failed",
manifest.FailedRuns, manifest.PlannedRuns)
}
if len(manifest.TaskIDs) != 0 {
t.Fatalf("task_ids = %v, want none", manifest.TaskIDs)
}
for i, m := range runs {
if !m.Failed || m.FailureStage == "" || m.Task != blocked.ID {
t.Fatalf("results.jsonl line %d is not a marked failure: %+v", i+1, m)
}
}
total := 0
for _, arm := range armsFor(blocked) {
total += funnelScored(t, out, arm.name)
}
if total != 0 {
t.Fatalf("funnels scored %d runs although nothing ran:\n%s", total, out)
}
if gotScored, gotFailed := reportCounts(t, out); gotScored != 0 || gotFailed != manifest.FailedRuns {
t.Fatalf("report says scored=%d failed=%d, want 0 and %d", gotScored, gotFailed, manifest.FailedRuns)
}
}
// The control: with nothing blocked, a batch is what it always was. Every run
// is completed with a stage, two identical batches write byte-identical results
// and the same manifest header, and the new failure field stays out of it.
func TestAllSuccessfulBatchKeepsItsArtifacts(t *testing.T) {
tasks := indexTasks()
first, runs, manifest, out := runBlockedExperiment(t, tasks)
second, again, repeat, _ := runBlockedExperiment(t, tasks)
if manifest.CompletedRuns != len(runs) || manifest.FailedRuns != 0 {
t.Fatalf("completed=%d failed=%d rows=%d, want every row completed",
manifest.CompletedRuns, manifest.FailedRuns, len(runs))
}
if manifest.CompletedRuns != manifest.PlannedRuns {
t.Fatalf("completed=%d planned=%d, want all planned", manifest.CompletedRuns, manifest.PlannedRuns)
}
for i, m := range runs {
if m.Failed || m.Task == "" || m.Arm == "" || m.FailureStage == "" {
t.Fatalf("results.jsonl line %d is not a completed run: %+v", i+1, m)
}
}
if gotScored, gotFailed := reportCounts(t, out); gotScored != manifest.CompletedRuns || gotFailed != 0 {
t.Fatalf("report says scored=%d failed=%d, want %d and 0", gotScored, gotFailed, manifest.CompletedRuns)
}
if !reflect.DeepEqual(runs, again) {
t.Fatal("two identical successful batches scored differently")
}
// Byte-for-byte on the runs. Trajectories carry per-message wall-clock
// stamps, so they are compared field by field instead.
if got, want := readFile(t, second, "results.jsonl"), readFile(t, first, "results.jsonl"); got != want {
t.Fatalf("an unchanged batch wrote different results.jsonl:\n%s\n---\n%s", want, got)
}
gotTraj, wantTraj := withoutStamps(t, readFile(t, second, "trajectories.jsonl")),
withoutStamps(t, readFile(t, first, "trajectories.jsonl"))
if gotTraj != wantTraj {
t.Fatalf("an unchanged batch wrote different trajectories:\n%s\n---\n%s", wantTraj, gotTraj)
}
repeat.StartedAt, repeat.FinishedAt = manifest.StartedAt, manifest.FinishedAt
repeat.TrajectoriesSHA256 = manifest.TrajectoriesSHA256
if !reflect.DeepEqual(repeat, manifest) {
t.Fatalf("an unchanged batch wrote a different manifest:\n%+v\n---\n%+v", manifest, repeat)
}
if repeat.FailedRuns != 0 {
t.Fatalf("a batch with no failure reports failed_runs=%d", repeat.FailedRuns)
}
}
// withoutStamps drops the createdAt stamp every message carries, which is the
// one field a dry batch cannot reproduce between two runs.
func withoutStamps(t *testing.T, text string) string {
t.Helper()
var b strings.Builder
for line := range strings.SplitSeq(strings.TrimSpace(text), "\n") {
var row map[string]any
if err := json.Unmarshal([]byte(line), &row); err != nil {
t.Fatal(err)
}
stripStamp(row)
out, err := json.Marshal(row)
if err != nil {
t.Fatal(err)
}
b.Write(out)
b.WriteByte('\n')
}
return b.String()
}
func stripStamp(v any) {
switch typed := v.(type) {
case map[string]any:
delete(typed, "createdAt")
delete(typed, "workDurationMs")
for _, child := range typed {
stripStamp(child)
}
case []any:
for _, child := range typed {
stripStamp(child)
}
}
}
func readFile(t *testing.T, root, name string) string {
t.Helper()
data, err := os.ReadFile(filepath.Join(root, name))
if err != nil {
t.Fatal(err)
}
return string(data)
}
+205
View File
@@ -0,0 +1,205 @@
package main
import (
"context"
"fmt"
"io/fs"
"os"
"path/filepath"
"reasonix/internal/state/sessionstore"
"strings"
"reasonix/internal/contract/ablation"
"reasonix/internal/contract/event"
"reasonix/internal/contract/provider"
"reasonix/internal/contract/tool"
"reasonix/internal/runtime/agent"
)
// A fixture is a real session that has really been compacted: no bench-only
// state shape, so the loader and the projection restore have to work on it.
const (
fixtureWindow = 60000
fixtureGenerations = 5
// fillerBytes per unit is what makes a fold necessary rather than optional.
fillerBytes = 6000
)
// digestProvider answers a summarizer without naming anything the corpus
// plants. A digest that happened to quote the answer would leave it visible
// and silently turn the task into a no-op.
type digestProvider struct{ calls int }
func (p *digestProvider) Name() string { return "fixture-digest" }
func (p *digestProvider) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) {
p.calls++
ch := make(chan provider.Chunk, 2)
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Earlier work\nEarlier work continued across several files; nothing outstanding was recorded."}
ch <- provider.Chunk{Type: provider.ChunkDone}
close(ch)
return ch, nil
}
// fillerCall is one ordinary read whose only job is to take up room — in the
// window, and in the fold index ahead of a cue.
func fillerCall(sess *sessionstore.Session, n int) {
id := fmt.Sprintf("f%04d", n)
sess.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
{ID: id, Name: "read_file", Arguments: fmt.Sprintf(`{"path":"internal/mod%03d/handler%03d.go"}`, n%37, n)},
}})
sess.Add(provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "read_file", Content: fillerBody(n)})
}
func fillerBody(n int) string {
var b strings.Builder
fmt.Fprintf(&b, "package mod%03d\n\n", n%37)
for b.Len() < fillerBytes {
fmt.Fprintf(&b, "func handler%03dStep%02d() error { return nil }\n", n, b.Len()%64)
}
return b.String()
}
// builtFixture is one task's history after real folds, plus where its target
// ended up.
type builtFixture struct {
Session *sessionstore.Session
State sessionstore.CompactionState
Target int
Path string
Instance fixtureInstance
}
// buildFixture plays a task's history through the real agent and folds it at
// the given arm, writing the session and its context state where the agent
// itself would.
func buildFixture(f fixtureInstance, arm ablation.Set, sessionPath string) (builtFixture, error) {
t := f.Task
sess := sessionstore.NewSession("You are a coding agent.")
sess.Add(provider.Message{Role: provider.RoleUser, Content: "Work through the transport and scheduler backlog."})
n := 0
at := -1
a := agent.New(&digestProvider{}, tool.NewRegistry(), sess, agent.Options{
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
SessionPath: sessionPath, KeepPolicy: agent.KeepErrors, Ablation: arm,
}, event.Discard)
for gen := range fixtureGenerations {
if gen == t.PlantAfterGen {
at = f.plant(sess)
}
for range 8 {
fillerCall(sess, n)
n++
}
if _, err := a.CompactNow(context.Background(), agent.CompactRequest{}); err != nil {
return builtFixture{}, fmt.Errorf("%s: generation %d fold: %w", t.ID, gen, err)
}
}
if at < 0 {
return builtFixture{}, fmt.Errorf("%s: PlantAfterGen %d is past the last generation", t.ID, t.PlantAfterGen)
}
// projectionCoversTail cannot revalidate a projection covering the whole
// transcript: it needs a live tail, which a real session always has because
// work continues after a fold. Neutral text — the tail is provider-visible.
sess.Add(provider.Message{Role: provider.RoleAssistant, Content: "Backlog swept; ready for the next question."})
if err := sess.Save(sessionPath); err != nil {
return builtFixture{}, fmt.Errorf("%s: save session: %w", t.ID, err)
}
state, ok, err := sessionstore.LoadCompactionState(sessionPath)
if err != nil {
return builtFixture{}, fmt.Errorf("%s: load context state: %w", t.ID, err)
}
if !ok || len(state.Projection.Messages) == 0 {
return builtFixture{}, fmt.Errorf("%s: no projection was installed", t.ID)
}
if state.Projection.CoveredCount <= at {
return builtFixture{}, fmt.Errorf("%s: target #%d is not inside the folded region (covered %d)",
t.ID, at, state.Projection.CoveredCount)
}
return builtFixture{Session: sess, State: state, Target: at, Path: sessionPath, Instance: f}, nil
}
// visibleContext is the model-visible view: the stored projection spliced with
// everything canonical that came after it.
func visibleContext(f builtFixture) []provider.Message {
canonical := f.Session.Snapshot()
out := append([]provider.Message(nil), f.State.Projection.Messages...)
if n := f.State.Projection.CoveredCount; n >= 0 && n < len(canonical) {
out = append(out, canonical[n:]...)
}
return out
}
// visibleText is everything the model would read, as one string.
func visibleText(f builtFixture) string {
var b strings.Builder
for _, m := range visibleContext(f) {
b.WriteString(m.Content)
b.WriteString("\n")
for _, tc := range m.ToolCalls {
b.WriteString(tc.Name)
b.WriteString(" ")
b.WriteString(string(tc.Arguments))
b.WriteString("\n")
}
}
return b.String()
}
// sealFixture removes the fixture from disk once the agent holds it in memory.
// The canonical transcript is the answer, and the sandbox mounts the host
// read-only by design — so a model that finds the path reads the answer, which
// one did. The agent has already loaded the session and its sidecar by the time
// this runs; a failed autosave afterwards costs the benchmark nothing.
func sealFixture(sessionPath string) {
// Every file, not two named ones: a session writes an event log and a
// context sidecar beside the transcript, and session.events.jsonl still
// carried the answer after the other two were sealed by name.
dir := filepath.Dir(sessionPath)
entries, err := os.ReadDir(dir)
if err != nil {
return
}
for _, e := range entries {
if !e.IsDir() {
_ = os.Remove(filepath.Join(dir, e.Name()))
}
}
// Deny the directory too, so an autosave cannot recreate what was removed.
_ = os.Chmod(dir, 0o500)
}
// unsealFixture restores the directory so the harness can clean up after
// itself.
func unsealFixture(sessionPath string) {
_ = os.Chmod(filepath.Dir(sessionPath), 0o700)
}
// answerOnDisk reports any readable file under dir holding a current-run
// answer. Two named files were sealed because two leaks were found in them;
// this scans the directory, so a path that moves — an autosave, a recovery
// tombstone, a future sidecar — fails here instead of on a paid run.
func answerOnDisk(dir string, markers []string) (string, string) {
var atPath, found string
_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() || atPath != "" {
return nil //nolint:nilerr // an unreadable entry is not a leak
}
body, readErr := os.ReadFile(path)
if readErr != nil {
return nil
}
for _, marker := range markers {
if len(marker) >= 6 && strings.Contains(string(body), marker) {
atPath, found = path, marker
return filepath.SkipAll
}
}
return nil
})
return atPath, found
}
+139
View File
@@ -0,0 +1,139 @@
// instance.go — a task template becomes one run's concrete history.
package main
import (
"crypto/rand"
"fmt"
"hash/fnv"
"io"
mathrand "math/rand"
"strings"
)
// The invariant this file exists for: no literal sufficient to satisfy
// AnswerRecovered for a concrete run may exist in this repository before that
// run begins. Grepping the corpus reveals the question, never the answer.
// varKind is how one placeholder's value is generated. Values stay shaped like
// real engineering records — a UUID would defeat the leak but turn every task
// into "find the random string".
type varKind string
const (
varCodename varKind = "codename" // cobalt-k7m4q9
varInt varKind = "int" // 731
varDecimal varKind = "decimal" // 0.0437
varDuration varKind = "duration" // 43ms
)
// varSpec declares one placeholder. Min/Max bound the numeric kinds.
type varSpec struct {
Name string
Kind varKind
Word string // codename stem, e.g. "cobalt"
Min int
Max int
}
// codenameEntropy is how much of a codename is unguessable. Five base32 digits
// is about 25 bits: far past what a model can reach by knowing the stem.
const codenameEntropy = 5
const codenameAlphabet = "abcdefghijkmnpqrstuvwxyz23456789"
// fixtureVars is one run's instantiated values.
type fixtureVars map[string]string
// newVars generates values for every spec. A nil rng means deterministic.
func newVars(specs []varSpec, rng *mathrand.Rand) fixtureVars {
out := fixtureVars{}
for _, spec := range specs {
switch spec.Kind {
case varCodename:
out[spec.Name] = spec.Word + "-" + randomToken(rng, codenameEntropy)
case varInt:
out[spec.Name] = fmt.Sprint(randomBetween(rng, spec.Min, spec.Max))
case varDecimal:
out[spec.Name] = fmt.Sprintf("0.%04d", randomBetween(rng, spec.Min, spec.Max))
case varDuration:
out[spec.Name] = fmt.Sprintf("%dms", randomBetween(rng, spec.Min, spec.Max))
}
}
return out
}
func randomToken(rng *mathrand.Rand, n int) string {
var b strings.Builder
for range n {
b.WriteByte(codenameAlphabet[randomBetween(rng, 0, len(codenameAlphabet)-1)])
}
return b.String()
}
func randomBetween(rng *mathrand.Rand, lo, hi int) int {
if hi <= lo {
return lo
}
return lo + rng.Intn(hi-lo+1)
}
// seededRand is the deterministic generator preflight and tests use, so a
// fixture's tier calibration and its golden state reproduce exactly.
func seededRand(parts ...string) *mathrand.Rand {
h := fnv.New64a()
for _, p := range parts {
h.Write([]byte(p))
h.Write([]byte{0})
}
return mathrand.New(mathrand.NewSource(int64(h.Sum64())))
}
// liveRand is the generator a paid run uses: every run gets values that existed
// nowhere before it started.
func liveRand() *mathrand.Rand {
rng, err := liveRandFrom(rand.Reader)
if err != nil {
panic("contextbench: no entropy for a live run: " + err.Error())
}
return rng
}
// liveRandFrom seeds a generator from an entropy source, so what a run's values
// are unpredictable from is the source and nothing else.
func liveRandFrom(entropy io.Reader) (*mathrand.Rand, error) {
var seed [8]byte
if _, err := io.ReadFull(entropy, seed[:]); err != nil {
return nil, err
}
var n int64
for _, b := range seed {
n = n<<8 | int64(b)
}
return mathrand.New(mathrand.NewSource(n)), nil
}
// instantiate substitutes {{name}} and refuses anything left over. The grammar
// is one identifier in double braces: no conditionals, no loops, no defaults.
// This is fixture data, not a template language.
func instantiate(text string, vars fixtureVars) (string, error) {
out := text
for name, value := range vars {
out = strings.ReplaceAll(out, "{{"+name+"}}", value)
}
if strings.Contains(out, "{{") || strings.Contains(out, "}}") {
return "", fmt.Errorf("unresolved placeholder in %q", firstPlaceholder(out))
}
return out, nil
}
func firstPlaceholder(text string) string {
i := strings.Index(text, "{{")
if i < 0 {
return text
}
j := strings.Index(text[i:], "}}")
if j < 0 {
return text[i:min(i+40, len(text))]
}
return text[i : i+j+2]
}
+144
View File
@@ -0,0 +1,144 @@
// ContextRetrievalBench builds the fixtures the long-horizon retrieval
// experiments run against, and proves they are worth paying a provider for.
//
// -mode=calibrate find each index task's lead-call count for its cue tier
// -mode=preflight build every task under every arm and check the contract
//
// Nothing here calls a provider. The experiments themselves do; this is what
// says they would measure what they claim to.
package main
import (
"flag"
"fmt"
"os"
"sort"
"strings"
)
func main() {
mode := flag.String("mode", "preflight", "calibrate | preflight | snippet | adversarial | run-search | run-boundary | run-index")
work := flag.String("work", "", "directory for built fixtures (default: a temp dir)")
only := flag.String("task", "", "limit to one task id")
dry := flag.Bool("dry", false, "run modes: drive the pipeline with a scripted provider instead of paying for one")
flag.Parse()
if err := validateCorpus(); err != nil {
fmt.Fprintln(os.Stderr, "corpus:", err)
os.Exit(2)
}
root := *work
if root == "" {
dir, err := os.MkdirTemp("", "context-retrieval-*")
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(2)
}
defer os.RemoveAll(dir)
root = dir
}
tasks := allTasks()
if *only != "" {
t, ok := taskByID(*only)
if !ok {
fmt.Fprintf(os.Stderr, "unknown task %q\n", *only)
os.Exit(2)
}
tasks = []contextTask{t}
}
switch *mode {
case "calibrate":
os.Exit(runCalibrate(tasks, root))
case "preflight":
os.Exit(runPreflight(tasks, root))
case "run-search":
os.Exit(runExperiment(*mode, experimentSearch, root, *dry, tasks))
case "run-boundary":
os.Exit(runBoundaries(*mode, root))
case "snippet":
os.Exit(runSnippetAudit(root, 100))
case "adversarial":
os.Exit(runAdversarial(root))
case "run-index":
batch := tasks
if *only == "" {
// A batch measures the efficiency substrate only. Naming one task
// with -task runs that task, role included, so the stopping corpus
// stays measurable on its own.
batch = efficiencyTasks()
}
os.Exit(runExperiment(*mode, experimentIndex, root, *dry, batch))
default:
fmt.Fprintf(os.Stderr, "unknown mode %q\n", *mode)
os.Exit(2)
}
}
func runPreflight(tasks []contextTask, root string) int {
var all []finding
for _, t := range tasks {
found, err := checkTask(t, root)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", t.ID, err)
return 2
}
all = append(all, found...)
status := "ok"
if len(found) > 0 {
status = fmt.Sprintf("%d problem(s)", len(found))
}
fmt.Printf("%-24s %-6s gen=%-2d %s\n", t.ID, t.Experiment, t.PlantAfterGen, status)
}
if len(all) == 0 {
fmt.Printf("\n%d fixtures ready.\n", len(tasks))
return 0
}
fmt.Fprintf(os.Stderr, "\n%d preflight problem(s):\n", len(all))
sort.Slice(all, func(i, j int) bool { return all[i].Task < all[j].Task })
for _, f := range all {
fmt.Fprintln(os.Stderr, " ", f)
}
return 1
}
// runCalibrate reports, for each index task, the lead-call counts at which its
// cue is still addressed by each budget. The corpus freezes one number from
// this; it is never computed at run time, or the implementation under test
// would be moving its own goalposts.
func runCalibrate(tasks []contextTask, root string) int {
for _, t := range tasks {
if t.Experiment != experimentIndex {
continue
}
var rows []string
for lead := range fixtureGenerations {
probe := t
probe.PlantAfterGen = lead
var visibleAt []string
for _, scale := range armScales {
inst, ierr := instantiateTask(probe, seededRand(t.ID, scale, "calibrate"))
if ierr != nil {
fmt.Fprintln(os.Stderr, ierr)
return 2
}
f, err := buildFixture(inst, armFor(scale, false),
fmt.Sprintf("%s/calib/%s/%d-%s/session.jsonl", root, t.ID, lead, scale))
if err != nil {
fmt.Fprintf(os.Stderr, "%s gen=%d %s: %v\n", t.ID, lead, scale, err)
return 2
}
if strings.Contains(visibleText(f), inst.CueMarker) {
visibleAt = append(visibleAt, scale)
}
}
rows = append(rows, fmt.Sprintf("planted after gen %d, cue visible at: %s", lead, strings.Join(visibleAt, ",")))
}
fmt.Printf("\n%s (wants tier %s)\n", t.ID, t.CueTier)
for _, r := range rows {
fmt.Println(" ", r)
}
}
return 0
}
+189
View File
@@ -0,0 +1,189 @@
package main
import (
"bytes"
"fmt"
"maps"
mathrand "math/rand"
"os/exec"
"slices"
"strings"
"testing"
)
// The same checks the preflight mode runs, under go test: a compaction change
// that invalidates the fixtures fails here rather than three hundred API calls
// later.
func TestCorpusIsWellFormed(t *testing.T) {
if err := validateCorpus(); err != nil {
t.Fatal(err)
}
if got := len(searchTasks()); got != 6 {
t.Errorf("search experiment has %d tasks, want 6", got)
}
if got := len(indexTasks()); got != 12 {
t.Errorf("index experiment has %d tasks, want 12", got)
}
stopping := map[string]bool{}
tiers := map[string]int{}
for _, task := range indexTasks() {
if task.Role == roleStopping {
stopping[task.ID] = true
continue
}
tiers[task.CueTier]++
}
if !stopping["i03-coalescing"] || !stopping["i04-recovery-fence"] {
t.Errorf("the stopping corpus is %v; it must keep i03-coalescing and i04-recovery-fence", stopping)
}
// Two per tier, over the efficiency substrate, is what makes the staircase
// readable: the same task can be compared across the boundary its cue sits
// on, instead of two different questions being compared across two arms.
for _, tier := range []string{tierQuarter, tierHalf, tierDefault} {
if tiers[tier] != 2 {
t.Errorf("tier %s has %d tasks, want 2", tier, tiers[tier])
}
}
}
// The role decides which batch a task enters; the literal lists are only where
// it is declared, so the two must not drift.
func TestIndexListsMatchTheirRoles(t *testing.T) {
for _, task := range append(append(indexQuarterTasks(), indexHalfTasks()...), indexDefaultTasks()...) {
if task.Role != roleEfficiency {
t.Errorf("%s sits in an efficiency list with role %q", task.ID, task.Role)
}
}
for _, task := range stoppingIndexTasks() {
if task.Role != roleStopping {
t.Errorf("%s sits in the stopping list with role %q", task.ID, task.Role)
}
}
}
// The invariant the whole instantiation layer exists for: no literal a scorer
// needs may be readable in this repository before a run starts.
func TestNoAnswerLiteralExistsInTheRepository(t *testing.T) {
root, err := repoRoot()
if err != nil {
t.Skip(err)
}
for _, task := range allTasks() {
inst, err := instantiateTask(task, seededRand(task.ID, "leak-check"))
if err != nil {
t.Fatal(err)
}
for _, marker := range append(inst.AnswerMarkers, inst.CueMarker) {
if len(marker) < 6 {
continue // a bare number is not a literal anyone can grep for
}
if found, where := grepRepo(t, root, marker); found {
t.Errorf("%s: %q is readable at %s before the run", task.ID, marker, where)
}
}
}
}
// A run's values are unpredictable from a previous run only if the live
// generator is seeded from entropy. Comparing answers across two instances
// cannot assert that: a task whose answer is a bounded number collides by
// chance (1/8601 to 1/286 per task), so the check is on the seed source.
func TestLiveRandDrawsFromEntropy(t *testing.T) {
a, b := liveRand(), liveRand()
for range 4 {
if a.Int63() != b.Int63() {
return
}
}
t.Fatal("two live generators produced the same stream: they are seeded alike")
}
// The source alone decides an instance: the same entropy gives the same
// answers, different entropy gives different values.
func TestLiveRandSourceDecidesTheInstance(t *testing.T) {
same := func() *mathrand.Rand {
rng, err := liveRandFrom(bytes.NewReader(bytes.Repeat([]byte{7}, 8)))
if err != nil {
t.Fatal(err)
}
return rng
}
other, err := liveRandFrom(bytes.NewReader(bytes.Repeat([]byte{9}, 8)))
if err != nil {
t.Fatal(err)
}
if _, err := liveRandFrom(bytes.NewReader(nil)); err == nil {
t.Fatal("an empty entropy source was accepted")
}
for _, task := range allTasks() {
a, _ := instantiateTask(task, same())
b, _ := instantiateTask(task, same())
if !slices.Equal(a.AnswerMarkers, b.AnswerMarkers) || !maps.Equal(a.Vars, b.Vars) {
t.Errorf("%s: the same entropy gave different instances", task.ID)
}
c, _ := instantiateTask(task, other)
if maps.Equal(a.Vars, c.Vars) {
t.Errorf("%s: different entropy gave the same values", task.ID)
}
}
}
// A preflight that could not reproduce would calibrate against one run's
// nonces and report about another's.
func TestSeededInstancesReproduce(t *testing.T) {
for _, task := range allTasks() {
a, _ := instantiateTask(task, seededRand(task.ID, "x"))
b, _ := instantiateTask(task, seededRand(task.ID, "x"))
if !slices.Equal(a.AnswerMarkers, b.AnswerMarkers) || a.body != b.body {
t.Errorf("%s did not reproduce under the same seed", task.ID)
}
}
}
// An unresolved placeholder would put "{{marker}}" in the transcript and score
// every run wrong; it fails the fixture instead.
func TestUnresolvedPlaceholderIsRefused(t *testing.T) {
if _, err := instantiate("value is {{missing}}", fixtureVars{"other": "x"}); err == nil {
t.Fatal("an unresolved placeholder was accepted")
}
}
func repoRoot() (string, error) {
out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
if err != nil {
return "", fmt.Errorf("not in a git checkout: %w", err)
}
return strings.TrimSpace(string(out)), nil
}
// grepRepo searches tracked files only: build artifacts and temp fixtures are
// not what a fresh checkout hands anyone.
func grepRepo(t *testing.T, root, needle string) (bool, string) {
t.Helper()
cmd := exec.Command("git", "grep", "-l", "-F", needle)
cmd.Dir = root
out, err := cmd.Output()
if err != nil {
return false, "" // exit 1 means no match
}
return true, strings.TrimSpace(strings.SplitN(string(out), "\n", 2)[0])
}
func TestEveryFixturePassesPreflight(t *testing.T) {
if testing.Short() {
t.Skip("builds and folds 36 sessions")
}
root := t.TempDir()
for _, task := range allTasks() {
t.Run(task.ID, func(t *testing.T) {
found, err := checkTask(task, root)
if err != nil {
t.Fatal(err)
}
for _, f := range found {
t.Error(f)
}
})
}
}
+290
View File
@@ -0,0 +1,290 @@
package main
import (
"encoding/json"
"fmt"
"path/filepath"
"slices"
"strings"
"reasonix/internal/base/retrieval"
"reasonix/internal/contract/ablation"
"reasonix/internal/contract/provider"
)
// Preflight is what makes a real run worth paying for. An answer the model can
// already read, or a cue not where the matrix says, turns an arm into a
// different experiment without saying so.
// armScales are the fold-index arms an index task is measured across.
var armScales = []string{"default", "half", "quarter", "off"}
// experimentArm is one measured configuration of a task.
type experimentArm struct {
name string
scale string
searchOff bool
}
// armsFor is the single declaration of which arms a task runs at, so preflight
// and the run itself cannot check one set and measure another.
func armsFor(t contextTask) []experimentArm {
if t.Experiment == experimentSearch {
return []experimentArm{
{"search-on", "default", false},
{"search-off", "default", true},
}
}
arms := make([]experimentArm, 0, len(armScales))
for _, scale := range armScales {
arms = append(arms, experimentArm{"index-" + scale, scale, false})
}
return arms
}
func scaleOf(name string) ablation.FoldIndexScale {
s, err := ablation.ParseFoldIndexScale(name)
if err != nil {
panic("unknown scale " + name) // the list above is a constant
}
return s
}
// armFor is the arm a task is measured under at one fold-index scale.
func armFor(scale string, searchOff bool) ablation.Set {
set := ablation.Set{}
if searchOff {
set = ablation.New(ablation.RecallSearch)
}
return set.WithFoldIndex(scaleOf(scale))
}
type finding struct {
Task, Arm, Problem string
}
func (f finding) String() string { return fmt.Sprintf("%s [%s]: %s", f.Task, f.Arm, f.Problem) }
// checkTask builds the task under every arm it will be run at and reports what
// does not hold.
func checkTask(t contextTask, root string) ([]finding, error) {
var out []finding
for _, arm := range armsFor(t) {
// Deterministic values: a preflight has to reproduce, and a tier
// calibration measured against one run's nonces is measured against
// nothing.
inst, err := instantiateTask(t, seededRand(t.ID, arm.name, "preflight"))
if err != nil {
return nil, err
}
dir := filepath.Join(root, t.ID, arm.name)
f, err := buildFixture(inst, armFor(arm.scale, arm.searchOff), filepath.Join(dir, "session.jsonl"))
if err != nil {
return nil, err
}
visible := visibleText(f)
// The answer must be unreadable in every arm, or the task measures
// reading rather than recall.
for _, marker := range inst.AnswerMarkers {
if len(marker) < 3 {
continue // a bare number is not a nonce; the whole set still has to match
}
if strings.Contains(visible, marker) {
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("answer marker %q is already visible", marker)})
}
}
// The corpus has to be findable at all, or a miss in a real run says
// nothing about the model.
rank, err := probeRank(f, inst.ProbeQuery)
if err != nil {
return nil, err
}
switch {
case rank == 0:
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("probe query %q does not find the target at all", inst.ProbeQuery)})
case rank > 5:
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("probe query %q ranks the target %d", inst.ProbeQuery, rank)})
}
if t.Experiment == experimentIndex {
out = append(out, checkCueVisibility(t, inst, arm.scale, arm.name, visible)...)
} else if line := indexLineFor(f, f.Target); line != "" {
out = append(out, finding{t.ID, arm.name,
"a search task is addressed by the fold index, so it measures the index and not search: " + line})
}
// The projection is what the agent believes. This is what the model is
// told, and it is the one that has to hold.
req, err := captureRequest(f, armFor(arm.scale, arm.searchOff), inst.Prompt)
if err != nil {
return nil, fmt.Errorf("%s [%s]: capture: %w", t.ID, arm.name, err)
}
out = append(out, checkCapturedRequest(t, inst, arm.name, arm.scale, arm.searchOff, req)...)
}
return out, nil
}
// checkCapturedRequest holds one arm's real provider request to its contract:
// the answer unreadable, the cue exactly where its tier says, the recall tool
// offering only what this arm can do, and no host-side field surviving the
// boundary.
func checkCapturedRequest(t contextTask, inst fixtureInstance, armName, scale string, searchOff bool, req provider.Request) []finding {
var out []finding
for _, marker := range inst.AnswerMarkers {
if len(marker) < 3 {
continue // a bare number is not a nonce; the set as a whole still is
}
for _, at := range findInSurface(req, marker) {
out = append(out, finding{t.ID, armName, fmt.Sprintf("answer marker %q leaked at %s", marker, at)})
}
}
if t.Experiment == experimentIndex {
wantVisible, wantHidden := tierScales(t.CueTier)
where := findInSurface(req, inst.CueMarker)
switch {
case contains(wantVisible, scale) && len(where) == 0:
out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is not provider-visible; the tier expects it", inst.CueMarker)})
case contains(wantHidden, scale) && len(where) > 0:
out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is provider-visible at %s: %s", inst.CueMarker, where[0], scale)})
}
}
schema, ok := recallToolSchema(req)
if !ok {
out = append(out, finding{t.ID, armName, "recall is not in the provider's tool list"})
return append(out, boundaryFindings(t, armName, req)...)
}
params := ""
if b, err := json.Marshal(schema.Parameters); err == nil {
params = string(b)
}
// Both arms keep the read half: the ablation removes search, not recall.
if !strings.Contains(params, `"positions"`) {
out = append(out, finding{t.ID, armName, "recall lost its positions parameter"})
}
hasQuery := strings.Contains(params, `"query"`)
switch {
case searchOff && hasQuery:
out = append(out, finding{t.ID, armName, "recall still offers a query parameter"})
case !searchOff && !hasQuery:
out = append(out, finding{t.ID, armName, "recall has no query parameter"})
}
if searchOff {
for _, phrase := range searchWording {
for _, at := range findInSurface(req, phrase) {
out = append(out, finding{t.ID, armName, fmt.Sprintf("search is offered to an arm without it: %q at %s", phrase, at)})
}
}
}
return append(out, boundaryFindings(t, armName, req)...)
}
func boundaryFindings(t contextTask, armName string, req provider.Request) []finding {
var out []finding
for _, at := range localOnlyLeaks(req) {
out = append(out, finding{t.ID, armName, "host-only field crossed the provider boundary at " + at})
}
return out
}
// checkCueVisibility holds an index task to its tier: the cue is addressed at
// the scales the matrix names and at no others. A policy change that moves it
// fails here rather than turning two arms into one experiment.
func checkCueVisibility(t contextTask, inst fixtureInstance, scale, armName, visible string) []finding {
wantVisible, wantHidden := tierScales(t.CueTier)
has := strings.Contains(visible, inst.CueMarker)
switch {
case contains(wantVisible, scale) && !has:
return []finding{{t.ID, armName, fmt.Sprintf("cue %q should be addressed at %s and is not", inst.CueMarker, scale)}}
case contains(wantHidden, scale) && has:
return []finding{{t.ID, armName, fmt.Sprintf("cue %q should be gone at %s and is not", inst.CueMarker, scale)}}
}
return nil
}
func contains(list []string, want string) bool { return slices.Contains(list, want) }
// probeRank runs the task's own query against the folded region the way recall
// would, and reports where the target lands.
func probeRank(f builtFixture, query string) (int, error) {
terms, err := retrieval.QueryTerms(query)
if err != nil {
return 0, err
}
canonical := f.Session.Snapshot()
covered := min(f.State.Projection.CoveredCount, len(canonical))
docs := make([]map[string]int, 0, covered)
positions := make([]int, 0, covered)
lengths := make([]int, 0, covered)
for i, m := range canonical[:covered] {
var text strings.Builder
text.WriteString(m.Content)
for _, tc := range m.ToolCalls {
text.WriteString(" " + tc.Name + " " + string(tc.Arguments))
}
tokens := retrieval.Tokens(text.String())
if len(tokens) == 0 {
continue
}
docs = append(docs, retrieval.Counts(tokens))
positions = append(positions, i)
lengths = append(lengths, len(tokens))
}
df := retrieval.DocumentFrequency(docs)
avg := 0.0
for _, l := range lengths {
avg += float64(l)
}
if len(lengths) > 0 {
avg /= float64(len(lengths))
}
type hit struct {
pos int
score float64
}
var hits []hit
for i, counts := range docs {
if s := retrieval.BM25Score(counts, lengths[i], terms, df, len(docs), avg); s > 0 {
hits = append(hits, hit{positions[i], s})
}
}
for i := range hits {
for j := i + 1; j < len(hits); j++ {
if hits[j].score > hits[i].score {
hits[i], hits[j] = hits[j], hits[i]
}
}
}
// A tool result answers at its caller's position, so either counts.
for i, h := range hits {
if h.pos == f.Target || h.pos == f.Target+1 {
return i + 1, nil
}
}
return 0, nil
}
// indexLineFor returns the fold-index line addressing a position, if any.
func indexLineFor(f builtFixture, pos int) string {
want := fmt.Sprintf("#%d ", pos)
for line := range strings.SplitSeq(visibleIndexOnly(f), "\n") {
if strings.Contains(line, want) {
return strings.TrimSpace(line)
}
}
return ""
}
// visibleIndexOnly is just the fold-index sections of the projection.
func visibleIndexOnly(f builtFixture) string {
var b strings.Builder
for _, m := range visibleContext(f) {
if i := strings.Index(m.Content, "## Folded work index"); i >= 0 {
b.WriteString(m.Content[i:])
b.WriteString("\n")
}
}
return b.String()
}
@@ -0,0 +1,252 @@
// reformulation.go — how a query changed, from token sets alone.
package main
import (
"fmt"
"slices"
"sort"
"strings"
"reasonix/internal/base/retrieval"
)
// The host probe ranks every target first, so the engine is not what makes a
// model take more than one attempt. These classes are set arithmetic over the
// same tokenizer recall uses: no judge, nothing read out of prose.
const (
reformFirst = "First" // nothing to compare against yet
reformNearDup = "NearDuplicate" // the same query again, give or take a word
reformNarrowing = "Narrowing" // kept everything and added terms
reformBroadening = "Broadening" // dropped terms, added none
reformAnchored = "HistoricalAnchor" // reached for a term the question does not contain
reformShift = "SynonymShift" // traded terms for others
)
// nearDuplicateOverlap is where "reworded" stops and "asked again" begins.
const nearDuplicateOverlap = 0.8
func terms(s string) []string {
return retrieval.Unique(retrieval.Tokens(s))
}
func termSet(s string) map[string]bool {
out := map[string]bool{}
for _, t := range terms(s) {
out[t] = true
}
return out
}
func jaccard(a, b map[string]bool) float64 {
if len(a) == 0 && len(b) == 0 {
return 1
}
shared := 0
for t := range a {
if b[t] {
shared++
}
}
union := len(a) + len(b) - shared
if union == 0 {
return 0
}
return float64(shared) / float64(union)
}
// classifyReformulation names how this query differs from the one before it.
func classifyReformulation(previous, current, question string) string {
if previous == "" {
return reformFirst
}
prev, cur := termSet(previous), termSet(current)
if jaccard(prev, cur) >= nearDuplicateOverlap {
return reformNearDup
}
added, dropped := diffTerms(cur, prev), diffTerms(prev, cur)
// A term the question never used had to come from the model's own memory of
// the history, which is a different move from rephrasing what it was asked.
ask := termSet(question)
for _, t := range added {
if !ask[t] {
return reformAnchored
}
}
switch {
case len(dropped) == 0:
return reformNarrowing
case len(added) == 0:
return reformBroadening
default:
return reformShift
}
}
func diffTerms(from, minus map[string]bool) []string {
var out []string
for t := range from {
if !minus[t] {
out = append(out, t)
}
}
sort.Strings(out)
return out
}
// classifySearches labels each attempt and records which of the host probe's
// terms it left out. The probe is the oracle: it ranks the target first, so a
// query missing its terms says what the model did not think to ask for.
func classifySearches(m *contextMetrics, inst fixtureInstance) {
probe := terms(inst.ProbeQuery)
previous := ""
for i := range m.Searches {
s := &m.Searches[i]
s.Reformulation = classifyReformulation(previous, s.Query, inst.Prompt)
have := termSet(s.Query)
for _, term := range probe {
if !have[term] {
s.MissingProbeTerms = append(s.MissingProbeTerms, term)
}
}
previous = s.Query
}
}
// queryAudit is the reformulation funnel across a set of runs.
type queryAudit struct {
Runs, RunsThatSearched int
FirstQueryHit int
HitBySecond, HitByThird int
NeverHit int
TotalSearches int
PostHitSearches int
HitThenRead int
EscapeBeforeFirstSearch int
Reformulations map[string]int
MissedProbeTerms map[string]int
}
func auditQueries(runs []contextMetrics) queryAudit {
a := queryAudit{Reformulations: map[string]int{}, MissedProbeTerms: map[string]int{}}
for _, r := range runs {
if r.Failed || r.Contaminated {
continue
}
a.Runs++
a.TotalSearches += len(r.Searches)
a.PostHitSearches += r.PostHitSearches
if r.EscapeBeforeFirstRecall > 0 {
a.EscapeBeforeFirstSearch++
}
if len(r.Searches) == 0 {
continue
}
a.RunsThatSearched++
hitAt := 0
for i, s := range r.Searches {
a.Reformulations[s.Reformulation]++
for _, term := range s.MissingProbeTerms {
a.MissedProbeTerms[term]++
}
if s.TargetRank > 0 && hitAt == 0 {
hitAt = i + 1
}
}
switch {
case hitAt == 1:
a.FirstQueryHit++
case hitAt == 2:
a.HitBySecond++
case hitAt >= 3:
a.HitByThird++
default:
a.NeverHit++
}
if hitAt > 0 && r.TargetRead {
a.HitThenRead++
}
}
return a
}
func (a queryAudit) report() string {
var b strings.Builder
b.WriteString("\n## Query reformulation\n")
fmt.Fprintf(&b, " runs=%d searched=%d searches=%d (%.2f per searching run)\n",
a.Runs, a.RunsThatSearched, a.TotalSearches, ratio(a.TotalSearches, a.RunsThatSearched))
fmt.Fprintf(&b, " target found by query 1: %d by 2: %d by 3+: %d never: %d\n",
a.FirstQueryHit, a.HitBySecond, a.HitByThird, a.NeverHit)
fmt.Fprintf(&b, " found then read: %d/%d searches issued after the target was already back: %d\n",
a.HitThenRead, a.FirstQueryHit+a.HitBySecond+a.HitByThird, a.PostHitSearches)
fmt.Fprintf(&b, " reached for the workspace before searching: %d\n", a.EscapeBeforeFirstSearch)
b.WriteString(" reformulations: " + countsLine(a.Reformulations) + "\n")
b.WriteString(" probe terms most often absent: " + topCounts(a.MissedProbeTerms, 8) + "\n")
return b.String()
}
func ratio(n, d int) float64 {
if d == 0 {
return 0
}
return float64(n) / float64(d)
}
func countsLine(counts map[string]int) string {
keys := make([]string, 0, len(counts))
for k := range counts {
keys = append(keys, k)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, k := range keys {
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
}
return strings.Join(parts, " ")
}
func topCounts(counts map[string]int, limit int) string {
type kv struct {
k string
n int
}
all := make([]kv, 0, len(counts))
for k, n := range counts {
all = append(all, kv{k, n})
}
sort.Slice(all, func(i, j int) bool {
if all[i].n != all[j].n {
return all[i].n > all[j].n
}
return all[i].k < all[j].k
})
parts := make([]string, 0, limit)
for _, e := range all[:min(limit, len(all))] {
parts = append(parts, fmt.Sprintf("%s=%d", e.k, e.n))
}
if len(parts) == 0 {
return "(none)"
}
return strings.Join(parts, " ")
}
// queryLines lists every attempt, so a surprising audit can be read back.
func queryLines(runs []contextMetrics) string {
var b strings.Builder
b.WriteString("\n## Every query, in order\n")
for _, r := range runs {
for _, s := range r.Searches {
rank := "miss"
if s.TargetRank > 0 {
rank = fmt.Sprintf("rank %d", s.TargetRank)
}
fmt.Fprintf(&b, " %-24s %-14s #%d %-17s %-9s %q\n",
r.Task, r.Arm, s.Ordinal, s.Reformulation, rank, s.Query)
if len(s.MissingProbeTerms) > 0 && s.TargetRank == 0 {
fmt.Fprintf(&b, " %-24s %-14s missing probe terms: %s\n",
"", "", strings.Join(slices.Clone(s.MissingProbeTerms), " "))
}
}
}
return b.String()
}
+569
View File
@@ -0,0 +1,569 @@
package main
import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"reasonix/internal/state/sessionstore"
"slices"
"sort"
"strings"
"time"
"reasonix/internal/assembly/boot"
"reasonix/internal/contract/ablation"
"reasonix/internal/contract/provider"
"reasonix/internal/contract/tool"
_ "reasonix/internal/model/openai"
"reasonix/internal/runtime/agent"
_ "reasonix/internal/tools/builtin"
)
// A run is one recall question against a fixture the preflight has already
// cleared. Scoring reads the transcript the turn appended — tool names and
// their JSON arguments — so nothing depends on how the model narrated itself.
const (
runBaseURL = "https://api.deepseek.com"
runModel = "deepseek-chat"
runTimeout = 4 * time.Minute
)
func realProvider() (provider.Provider, error) {
key := os.Getenv("DEEPSEEK_API_KEY")
if key == "" {
return nil, fmt.Errorf("a real run needs DEEPSEEK_API_KEY")
}
return provider.New("openai", provider.Config{
Name: "contextbench", BaseURL: runBaseURL, Model: runModel, APIKey: key,
})
}
// adversarialPrompts tell the model what it is running and send it after the
// answer everywhere except the conversation. The question is not whether it
// obeys the experiment: it is whether the answer is reachable at all by a model
// that knows the mechanism and is actively attacking the host.
var adversarialPrompts = map[string]string{
experimentSearch: "You are running a benchmark. Find the expected answer for this task from the " +
"Reasonix repository, benchmark source, fixture generator, test files, scratch files, or " +
"session metadata. Do not use recall unless necessary. Report the exact hidden values.",
experimentIndex: "Inspect benchmarks/context-retrieval, the git repository, session files, temporary " +
"files, and any benchmark metadata you can access. Recover the concrete instantiated answer for " +
"this run without relying on the folded conversation. Report the exact values.",
}
// runAdversarial sends one task per experiment after the answer through every
// host-readable surface. It passes when nothing but recall could produce it —
// a run that gives up and recalls has still proven the attack failed.
func runAdversarial(root string) int {
root, err := filepath.Abs(root)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 2
}
restore, ierr := isolateWorkspace()
if ierr != nil {
fmt.Fprintln(os.Stderr, "isolate workspace:", ierr)
return 2
}
defer restore()
p, perr := realProvider()
if perr != nil {
fmt.Fprintln(os.Stderr, perr)
return 2
}
leaked := 0
for _, id := range []string{"r01-retry-boundary", "i01-transport-fallback"} {
t, ok := taskByID(id)
if !ok {
fmt.Fprintf(os.Stderr, "unknown task %q\n", id)
return 2
}
m, err := runOne(p, t, armFor("default", false), "adversarial", root, false, false,
adversarialPrompts[t.Experiment])
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", id, err)
m = failedMetrics(t, "adversarial", err)
}
verdict := "isolated"
if m.EscapeFoundAnswer {
verdict = "LEAKED via " + m.LeakedVia
leaked++
}
fmt.Printf("%-24s %-24s escapes=%-3d recall=%d answered=%v tools=%v\n",
id, verdict, m.EscapeCalls, m.SearchCalls+m.ReadCalls, m.AnswerRecovered, m.UnexpectedWorkTools)
if m.EscapeFoundAnswer {
fmt.Printf(" args: %s\n output: %s\n", m.LeakedArgs, m.LeakedOutput)
}
}
if leaked > 0 {
fmt.Fprintf(os.Stderr, "\n%d of 2 adversarial runs reached an answer outside recall\n", leaked)
return 1
}
fmt.Println("\nNo answer was reachable outside recall.")
return 0
}
// runOne asks one task under one arm and scores what the model did.
func runOne(p provider.Provider, t contextTask, arm ablation.Set, armName, root string, cueVisible, dry bool, promptOverride ...string) (contextMetrics, error) {
// Live runs get values that existed nowhere before the run started; a dry
// run stays reproducible.
rng := liveRand()
if dry {
rng = seededRand(t.ID, armName, "dry")
}
inst, err := instantiateTask(t, rng)
if err != nil {
return contextMetrics{}, err
}
f, err := buildFixture(inst, arm, filepath.Join(root, t.ID, armName, "session.jsonl"))
if err != nil {
return contextMetrics{}, err
}
sess, err := sessionstore.LoadSession(f.Path)
if err != nil {
return contextMetrics{}, fmt.Errorf("load session: %w", err)
}
reg := tool.NewRegistry()
for _, x := range tool.Builtins() {
reg.Add(x)
}
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
var usage usageSink
a := agent.New(p, reg, sess, agent.Options{
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
}, &usage)
a.LoadProjectionSidecar(f.Path)
// From here the transcript lives only in memory. On disk it is the answer.
sealFixture(f.Path)
defer unsealFixture(f.Path)
dir := filepath.Dir(f.Path)
if at, marker := answerOnDisk(dir, inst.AnswerMarkers); at != "" {
return contextMetrics{}, fmt.Errorf("%s [%s]: %q is readable at %s before the run", t.ID, armName, marker, at)
}
before := len(sess.Snapshot())
ctx, cancel := context.WithTimeout(context.Background(), runTimeout)
defer cancel()
prompt := inst.Prompt
if len(promptOverride) > 0 && promptOverride[0] != "" {
prompt = promptOverride[0]
}
runErr := a.Run(ctx, prompt)
if at, marker := answerOnDisk(dir, inst.AnswerMarkers); at != "" {
return contextMetrics{}, fmt.Errorf("%s [%s]: %q was written to %s during the run", t.ID, armName, marker, at)
}
appended := sess.Snapshot()[before:]
m := scoreRun(appended, inst, f.Target, armName, cueVisible)
m.trajectory = appended
if u, requests := usage.snapshot(); requests > 0 {
m.UsageReportedRequests, m.PromptTokens, m.CompletionTokens = requests, u.PromptTokens, u.CompletionTokens
m.CacheHitTokens, m.CacheMissTokens, m.UsageEstimated = u.CacheHitTokens, u.CacheMissTokens, u.Estimated
}
m.scoreAnswer(finalAnswer(appended), inst)
if runErr != nil {
m.FailureStage = "RunError"
m.TimeoutReason = timeoutReason(m)
}
return m, runErr
}
// failedMetrics is what a run that never reached a transcript leaves behind. It
// carries the run's identity and the stage that ended it, so the failure stays a
// countable fact; Failed keeps it out of every statistic.
func failedMetrics(t contextTask, armName string, err error) contextMetrics {
return contextMetrics{
Task: t.ID, Arm: armName, Failed: true,
FailureStage: stageNoEvidence, Reason: err.Error(),
}
}
// finalAnswer is the last assistant text the turn produced.
func finalAnswer(msgs []provider.Message) string {
for _, m := range slices.Backward(msgs) {
if m.Role == provider.RoleAssistant && strings.TrimSpace(m.Content) != "" {
return m.Content
}
}
return ""
}
// isolateWorkspace moves the process into an empty directory before any task
// runs. The corpus lives in this repository, so an agent left here greps the
// answers out of corpus.go — one did, quoting the file's own line about them
// being nonces. Fixtures stay outside it too: a readable session file is the
// canonical transcript, answer included.
func isolateWorkspace() (func(), error) {
dir, err := os.MkdirTemp("", "contextbench-workspace-*")
if err != nil {
return nil, err
}
readme := "This workspace is empty on purpose. The question is about earlier conversation,\nnot about any file here.\n"
if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte(readme), 0o644); err != nil {
return nil, err
}
previous, err := os.Getwd()
if err != nil {
return nil, err
}
if err := os.Chdir(dir); err != nil {
return nil, err
}
return func() {
_ = os.Chdir(previous)
_ = os.RemoveAll(dir)
}, nil
}
// runBoundaries runs only the pair each index task is built for: the narrowest
// scale still addressing its cue, and the next one down. Each pair moves the
// affordance and nothing else, where default-vs-off moves the whole index.
// The two arms get separate nonces: reusing one would leave the first arm's
// answer in a host-readable history for the second to find.
func runBoundaries(mode, root string) int {
started := time.Now()
planned := 0
repo, _ := os.Getwd()
root, err := filepath.Abs(root)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 2
}
restore, ierr := isolateWorkspace()
if ierr != nil {
fmt.Fprintln(os.Stderr, "isolate workspace:", ierr)
return 2
}
defer restore()
p, perr := realProvider()
if perr != nil {
fmt.Fprintln(os.Stderr, perr)
return 2
}
var all []contextMetrics
var cueSide, noCueSide []contextMetrics
for _, t := range efficiencyTasks() {
cue, noCue := boundaryPair(t.CueTier)
for _, side := range []struct {
scale string
has bool
}{{cue, true}, {noCue, false}} {
name := "index-" + side.scale
planned++
m, err := runOne(p, t, armFor(side.scale, false), name, root, side.has, false)
if err != nil {
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, name, err)
m = failedMetrics(t, name, err)
}
all = append(all, m)
if side.has {
cueSide = append(cueSide, m)
} else {
noCueSide = append(noCueSide, m)
}
fmt.Printf("%-24s %-14s cue=%-3v %-14s search=%d read=%d direct=%-5v recall-tok=%-5d%s escape=%d\n",
t.ID, name, side.has, m.FailureStage, m.SearchCalls, m.ReadCalls,
m.CueDirectRead, m.RecallReturnedTokens, promptTokenSuffix(m), m.EscapeCalls)
}
}
reportDeltas(cueSide, noCueSide)
fmt.Print(reportStopping(all))
return writeResults(root, batchInfo{Mode: mode, Planned: planned, Started: started, Repo: repo}, all)
}
// reportDeltas is the paired reading: each task against itself, then pooled.
func reportDeltas(cueSide, noCueSide []contextMetrics) {
fmt.Printf("\n## Paired deltas (cue present − cue absent)\n")
fmt.Printf(" %-24s %-10s %-10s %-9s %-11s %s\n",
"task", "recovery", "directread", "searches", "recall-tok", "escapes")
for i := range cueSide {
if i >= len(noCueSide) {
break
}
a, b := cueSide[i], noCueSide[i]
fmt.Printf(" %-24s %-10s %-10s %-9d %-11d %d\n", a.Task,
deltaBool(a.AnswerRecovered, b.AnswerRecovered),
deltaBool(a.CueDirectRead, b.CueDirectRead),
a.SearchCalls-b.SearchCalls,
a.RecallReturnedTokens-b.RecallReturnedTokens,
a.EscapeCalls-b.EscapeCalls)
}
fmt.Println("\n## Pooled")
fmt.Println(" ", summarize("cue-present", cueSide).line())
fmt.Println(" ", summarize("cue-absent", noCueSide).line())
fmt.Println("\n## Where each side looked first")
fmt.Printf(" %-14s %s\n", "cue-present", countsLine(summarize("x", cueSide).Routes))
fmt.Printf(" %-14s %s\n", "cue-absent", countsLine(summarize("x", noCueSide).Routes))
}
func deltaBool(a, b bool) string {
switch {
case a == b:
return "same"
case a:
return "+cue"
default:
return "-cue"
}
}
// runExperiment runs one experiment's tasks across its arms. A dry run drives
// the same pipeline with a scripted provider: it proves fixture, run, scoring
// and report hold together before any of it is paid for.
func runExperiment(mode, experiment, root string, dry bool, tasks []contextTask) int {
started := time.Now()
planned := 0
repo, _ := os.Getwd()
root, err := filepath.Abs(root)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 2
}
restore, err := isolateWorkspace()
if err != nil {
fmt.Fprintln(os.Stderr, "isolate workspace:", err)
return 2
}
defer restore()
var p provider.Provider = &scriptedProvider{}
if !dry {
real, rerr := realProvider()
if rerr != nil {
fmt.Fprintln(os.Stderr, rerr)
return 2
}
p = real
}
byArm := map[string][]contextMetrics{}
var all []contextMetrics
if experiment == experimentIndex {
fmt.Print(fingerprint(dry))
}
for _, t := range tasks {
if t.Experiment != experiment {
continue
}
// Arms are shuffled inside each task block. Running six defaults, then
// six halves, would make provider drift collinear with the axis under
// test: a bad minute would land entirely on one arm.
for _, arm := range shuffledArms(t) {
cueVisible := t.Experiment == experimentIndex && cueExpectedVisible(t, arm.scale)
planned++
m, err := runOne(p, t, armFor(arm.scale, arm.searchOff), arm.name, root, cueVisible, dry)
if err != nil {
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, arm.name, err)
m = failedMetrics(t, arm.name, err)
}
byArm[arm.name] = append(byArm[arm.name], m)
all = append(all, m)
fmt.Printf("%-24s %-14s %-18s search=%d hit=%d read=%d recall-tok=%d%s %v\n",
t.ID, arm.name, m.FailureStage, m.SearchCalls, m.TargetSearchHits, m.ReadCalls,
m.RecallReturnedTokens, promptTokenSuffix(m), m.UnexpectedWorkTools)
}
}
reportFunnels(byArm)
if experiment == experimentIndex {
fmt.Print(reportDoseResponse(all))
}
fmt.Print(reportStopping(all))
fmt.Print(auditQueries(all).report())
fmt.Print(queryLines(all))
if experiment == experimentIndex {
reportBoundaries(all)
}
return writeResults(root, batchInfo{Mode: mode, Planned: planned, Started: started, Dry: dry, Repo: repo}, all)
}
func cueExpectedVisible(t contextTask, scale string) bool {
visible, _ := tierScales(t.CueTier)
return contains(visible, scale)
}
func reportFunnels(byArm map[string][]contextMetrics) {
names := make([]string, 0, len(byArm))
for name := range byArm {
names = append(names, name)
}
sort.Strings(names)
fmt.Println("\n## Funnel")
for _, name := range names {
fmt.Println(" ", summarize(name, byArm[name]).line())
}
fmt.Println("\n## Where it looked first")
for _, name := range names {
f := summarize(name, byArm[name])
fmt.Printf(" %-14s %s\n", name, countsLine(f.Routes))
}
fmt.Println("\n## Where it broke")
for _, name := range names {
f := summarize(name, byArm[name])
stages := make([]string, 0, len(f.Stages))
for stage, n := range f.Stages {
stages = append(stages, fmt.Sprintf("%s=%d", stage, n))
}
sort.Strings(stages)
fmt.Printf(" %-12s %s\n", name, strings.Join(stages, " "))
}
}
// writeTrajectories saves what each turn produced, after the whole batch is
// done. Three analyses in a row have needed a re-run because something was
// counted and not kept — query text, then snippet contents. It is written at
// the end on purpose: during the batch these files are the answer.
func writeTrajectories(root string, all []contextMetrics) error {
f, err := os.Create(filepath.Join(root, "trajectories.jsonl"))
if err != nil {
return err
}
defer f.Close()
for _, m := range all {
if err := writeJSONLine(f, map[string]any{
"task": m.Task, "arm": m.Arm, "messages": m.trajectory,
}); err != nil {
return err
}
}
return nil
}
func writeResults(root string, info batchInfo, all []contextMetrics) int {
path := filepath.Join(root, "results.jsonl")
f, err := os.Create(path)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 2
}
defer f.Close()
scored, failed := 0, 0
for _, m := range all {
if err := writeJSONLine(f, m); err != nil {
fmt.Fprintln(os.Stderr, err)
return 2
}
if m.Failed {
failed++
} else {
scored++
}
}
if err := writeTrajectories(root, all); err != nil {
fmt.Fprintln(os.Stderr, "write trajectories:", err)
return 2
}
if err := writeBatchManifest(root, info, all); err != nil {
fmt.Fprintln(os.Stderr, "write batch manifest:", err)
return 2
}
// Both counts are named: a reader who sees only the batch size cannot tell
// measurements from attempts.
fmt.Printf("\n%d runs written to %s: scored=%d failed=%d (with trajectories.jsonl and batch.json)\n",
len(all), path, scored, failed)
return 0
}
// reportBoundaries prints each index task across its own cue boundary, then the
// two sides pooled. Task difficulty is held constant inside a row, which is
// what a four-arm average cannot do.
func reportBoundaries(all []contextMetrics) {
byKey := map[string]contextMetrics{}
scored, failed := 0, 0
for _, m := range all {
if m.Failed {
// No transcript, no boundary to compare: count the failure instead of
// printing the zero it left behind as a measurement.
failed++
continue
}
scored++
byKey[m.Task+"|"+m.Arm] = m
}
if scored == 0 {
fmt.Printf("\n## Across each task's cue boundary\n no scored runs: scored=%d failed=%d (nothing to compare)\n", scored, failed)
return
}
var cueSide, noCueSide []contextMetrics
fmt.Println("\n## Across each task's cue boundary")
fmt.Printf(" %-24s %-16s %-4s %-10s %-9s %-12s %-11s %s\n",
"task", "arm", "cue", "recovered", "cue-read", "search/task", "recall-tok", "escape")
for _, t := range indexTasks() {
cue, noCue := boundaryPair(t.CueTier)
for _, side := range []struct {
scale string
has bool
}{{cue, true}, {noCue, false}} {
m, ok := byKey[t.ID+"|index-"+side.scale]
if !ok {
continue
}
if side.has {
cueSide = append(cueSide, m)
} else {
noCueSide = append(noCueSide, m)
}
mark := "no"
if side.has {
mark = "yes"
}
fmt.Printf(" %-24s %-16s %-4s %-10v %-9v %-12d %-11d %d\n",
t.ID, "index-"+side.scale, mark, m.AnswerRecovered, m.RecallReadWithoutSearch,
m.SearchCalls, m.RecallReturnedTokens, m.EscapeCalls)
}
}
fmt.Println("\n## Pooled across boundaries")
fmt.Println(" ", summarize("cue-present", cueSide).line())
fmt.Println(" ", summarize("cue-absent", noCueSide).line())
}
// shuffledArms permutes a task's arms deterministically, so the order is
// reproducible and still uncorrelated with the axis being measured.
func shuffledArms(t contextTask) []experimentArm {
arms := armsFor(t)
rng := seededRand(t.ID, "arm-order")
rng.Shuffle(len(arms), func(i, j int) { arms[i], arms[j] = arms[j], arms[i] })
return arms
}
// fingerprint records what produced a batch. Not a gate — but batch effects
// have reversed three conclusions in this experiment already, and a table with
// no provenance cannot be compared with the next one.
func fingerprint(dry bool) string {
model, endpoint := runModel, runBaseURL
if dry {
model, endpoint = "scripted", "none"
}
commit := "unknown"
if out, err := exec.Command("git", "rev-parse", "--short", "HEAD").Output(); err == nil {
commit = strings.TrimSpace(string(out))
}
return fmt.Sprintf("## Batch\n model=%s endpoint=%s window=%d generations=%d commit=%s started=%s\n",
model, endpoint, fixtureWindow, fixtureGenerations, commit, time.Now().UTC().Format(time.RFC3339))
}
// timeoutReason separates a run that never found its answer from one that found
// it and kept going. Only the first is a case for a longer budget; the second
// would just get more time to circle. A batch reporting only "context deadline
// exceeded" cannot tell an arm that made retrieval harder from an arm that
// happened to draw a slow minute.
func timeoutReason(m contextMetrics) string {
switch {
case m.TargetHeldAtEnd && m.RoundsAfterTarget >= 2:
return "PostTargetRunaway"
case m.TargetHeldAtEnd:
return "TargetHeldLate"
case m.SearchCalls >= 3:
return "SearchSpiral"
case m.SearchCalls+m.ReadCalls+m.EscapeCalls == 0:
return "NoToolActivity"
default:
return "Unknown"
}
}
+95
View File
@@ -0,0 +1,95 @@
package main
import (
"context"
"encoding/json"
"regexp"
"slices"
"strconv"
"strings"
"reasonix/internal/contract/provider"
)
// scriptedProvider walks the chain a competent model would: search, read the
// best hit, answer with what came back. It proves the pipeline holds together
// for nothing, so a real run's first failure is about the model.
type scriptedProvider struct{}
func (p *scriptedProvider) Name() string { return "contextbench-scripted" }
var (
scriptedHit = regexp.MustCompile(`(?m)^#(\d+) `)
scriptedRecall = regexp.MustCompile(`(?s)^#\d+\n(.*)`)
)
// Stream decides from the request alone, never from a call counter: one
// provider serves every task in a run, and a counter would make the second task
// start where the first left off.
func (p *scriptedProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
ch := make(chan provider.Chunk, 3)
defer close(ch)
// Honour the schema, as a model does: an arm without a query parameter is
// an arm that cannot search, and pretending otherwise scores a refusal as
// a failed search.
if !recallAccepts(req, "query") {
ch <- provider.Chunk{Type: provider.ChunkText, Text: "I have no address for that part of the history."}
ch <- provider.Chunk{Type: provider.ChunkDone}
return ch, nil
}
// The last message, not the last tool result anywhere: the fixture's own
// history is full of tool results, and one of those is not this turn's.
last := ""
if n := len(req.Messages); n > 0 && req.Messages[n-1].Role == provider.RoleTool {
last = req.Messages[n-1].Content
}
switch {
case last == "":
args, _ := json.Marshal(map[string]string{"query": lastUserText(req)})
ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{
ID: "s1", Name: "recall", Arguments: string(args),
}}
case strings.Contains(last, "Folded context matching"):
positions := scriptedHit.FindAllStringSubmatch(last, 1)
if len(positions) == 0 {
ch <- provider.Chunk{Type: provider.ChunkText, Text: "Nothing in the folded history matches."}
break
}
n, _ := strconv.Atoi(positions[0][1])
args, _ := json.Marshal(map[string][]int{"positions": {n}})
ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{
ID: "r1", Name: "recall", Arguments: string(args),
}}
default:
// Answer with what recall returned, so a correct chain scores as one.
body := last
if m := scriptedRecall.FindStringSubmatch(last); len(m) > 1 {
body = m[1]
}
ch <- provider.Chunk{Type: provider.ChunkText, Text: "From the folded history: " + body}
}
ch <- provider.Chunk{Type: provider.ChunkDone}
return ch, nil
}
// recallAccepts reports whether recall is offered with the given parameter.
func recallAccepts(req provider.Request, param string) bool {
for _, t := range req.Tools {
if t.Name != "recall" {
continue
}
b, err := json.Marshal(t.Parameters)
return err == nil && strings.Contains(string(b), `"`+param+`"`)
}
return false
}
func lastUserText(req provider.Request) string {
for _, m := range slices.Backward(req.Messages) {
if m.Role == provider.RoleUser {
return m.Content
}
}
return ""
}
+182
View File
@@ -0,0 +1,182 @@
// snippet.go — does a rank-1 hit actually hand over the answer?
package main
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"reasonix/internal/base/retrieval"
"reasonix/internal/contract/ablation"
"reasonix/internal/contract/event"
"reasonix/internal/contract/tool"
"reasonix/internal/runtime/agent"
)
// TargetHit is not EvidenceSufficient: a rank-1 hit can still hand over a
// window too short to answer with, and searching again after that is correct.
// Measured through the shipped recall path, not a copy of it.
// snippetWidths are the windows to compare. 240 is what ships.
var snippetWidths = []int{120, 180, 240, 320, 480}
// historicalQueries are the queries these tasks actually drew in the pilots,
// kept because a probe query is the host's phrasing and the model's was not.
var historicalQueries = map[string][]string{
"i03-coalescing": {"coalesce window marker probe", "job completion coalescing window"},
"i04-recovery-fence": {"recovery fence marker epoch", "expected epoch", "recfence quartz"},
}
type snippetReading struct {
Task string
Query string
Width int
Covered int
Total int
Runes int
Rank int
Complete bool
}
// runSnippetAudit answers the question offline, over many instantiations so a
// single seed's lengths cannot decide it.
func runSnippetAudit(root string, seeds int) int {
fmt.Printf("## Snippet coverage (%d instantiations per task)\n", seeds)
fmt.Println(" Does the window a rank-1 hit returns contain every value the task scores on?")
byTaskWidth := map[string]map[int][]snippetReading{}
for _, t := range allTasks() {
byTaskWidth[t.ID] = map[int][]snippetReading{}
for seed := range seeds {
inst, err := instantiateTask(t, seededRand(t.ID, "snippet", fmt.Sprint(seed)))
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 2
}
queries := append([]string{inst.ProbeQuery}, historicalQueries[t.ID]...)
for _, q := range queries {
for _, width := range snippetWidths {
r := measureSnippet(inst, q, width)
byTaskWidth[t.ID][width] = append(byTaskWidth[t.ID][width], r)
}
}
}
}
fmt.Printf("\n %-24s %-5s", "task", "vals")
for _, w := range snippetWidths {
fmt.Printf("%9d", w)
}
fmt.Println(" mean runes @240")
for _, t := range allTasks() {
vals := len(t.AnswerVars)
fmt.Printf(" %-24s %-5d", t.ID, vals)
for _, w := range snippetWidths {
fmt.Printf("%8.0f%%", 100*completeRate(byTaskWidth[t.ID][w]))
}
fmt.Printf(" %.0f\n", meanRunes(byTaskWidth[t.ID][240]))
}
reportShippedPath(root)
return 0
}
// measureSnippet renders one hit through the same MakeSnippet the recall path
// uses, then asks whether every scored value survived the window.
func measureSnippet(inst fixtureInstance, query string, width int) snippetReading {
terms, err := retrieval.QueryTerms(query)
if err != nil {
return snippetReading{Task: inst.Task.ID, Query: query, Width: width, Total: len(inst.AnswerMarkers)}
}
// The tool result is the document the answer lives in; a call's arguments
// are a separate document and never carry it.
snippet := retrieval.MakeSnippet(inst.body, query, terms, width)
covered := 0
for _, marker := range inst.AnswerMarkers {
if strings.Contains(strings.ToLower(snippet), strings.ToLower(marker)) {
covered++
}
}
return snippetReading{
Task: inst.Task.ID, Query: query, Width: width,
Covered: covered, Total: len(inst.AnswerMarkers),
Runes: len([]rune(snippet)), Complete: covered == len(inst.AnswerMarkers),
}
}
// reportShippedPath cross-checks the table against the real recall tool: a
// fixture, a fold, and a search through the agent that ships.
func reportShippedPath(root string) {
fmt.Println("\n## Through the shipped recall path (width as shipped)")
for _, t := range allTasks() {
inst, err := instantiateTask(t, seededRand(t.ID, "snippet-live"))
if err != nil {
continue
}
f, err := buildFixture(inst, ablation.Set{}, filepath.Join(root, "snippet", t.ID, "session.jsonl"))
if err != nil {
fmt.Printf(" %-24s fixture: %v\n", t.ID, err)
continue
}
a := agent.New(&digestProvider{}, tool.NewRegistry(), f.Session, agent.Options{
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
SessionPath: f.Path, KeepPolicy: agent.KeepErrors,
}, event.Discard)
a.LoadProjectionSidecar(f.Path)
queries := append([]string{inst.ProbeQuery}, historicalQueries[t.ID]...)
for _, q := range queries {
res, err := a.RecallContext(context.Background(), tool.RecallRequest{Query: q})
if err != nil {
fmt.Printf(" %-24s %-34q recall: %v\n", t.ID, q, err)
continue
}
covered := 0
for _, marker := range inst.AnswerMarkers {
if strings.Contains(strings.ToLower(res.Text), strings.ToLower(marker)) {
covered++
}
}
fmt.Printf(" %-24s %-34q %d/%d values in the result\n",
t.ID, truncate(q, 32), covered, len(inst.AnswerMarkers))
}
}
}
func completeRate(rs []snippetReading) float64 {
if len(rs) == 0 {
return 0
}
n := 0
for _, r := range rs {
if r.Complete {
n++
}
}
return float64(n) / float64(len(rs))
}
func meanRunes(rs []snippetReading) float64 {
if len(rs) == 0 {
return 0
}
total := 0
for _, r := range rs {
total += r.Runes
}
return float64(total) / float64(len(rs))
}
// answerCoverage is what a run should have recorded all along: how much of the
// scored answer a search actually handed over. rank=1 with coverage 1/3 is not
// a stopping failure.
func answerCoverage(text string, markers []string) int {
covered := 0
lower := strings.ToLower(text)
for _, marker := range markers {
if strings.Contains(lower, strings.ToLower(marker)) {
covered++
}
}
return covered
}
+59
View File
@@ -0,0 +1,59 @@
package main
import (
"fmt"
"sync"
"reasonix/internal/contract/event"
"reasonix/internal/contract/provider"
)
// usageSink reads what a run cost from the events the agent already emits. Every
// billed completion reports event.Usage — the round path, compaction, command
// routing and failure diagnosis all do — so the bench reads the same record the
// product reads instead of wrapping a provider and counting a second time.
type usageSink struct {
mu sync.Mutex
requests int
usage provider.Usage
}
func (s *usageSink) Emit(e event.Event) {
if e.Kind != event.Usage || e.Usage == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
// One event is one completion that reported usage; RequestCount covers a
// completion that merged attempts. A request that failed before reporting
// leaves no event, so this is not a count of attempts.
if e.Usage.RequestCount > 0 {
s.requests += e.Usage.RequestCount
} else {
s.requests++
}
u := *e.Usage
s.usage.PromptTokens += u.PromptTokens
s.usage.CompletionTokens += u.CompletionTokens
s.usage.TotalTokens += u.TotalTokens
s.usage.CacheHitTokens += u.CacheHitTokens
s.usage.CacheMissTokens += u.CacheMissTokens
s.usage.ReasoningTokens += u.ReasoningTokens
s.usage.Estimated = s.usage.Estimated || u.Estimated
}
func (s *usageSink) snapshot() (provider.Usage, int) {
s.mu.Lock()
defer s.mu.Unlock()
return s.usage, s.requests
}
// promptTokenSuffix renders what a run cost only when something reported it. A
// dry run reports nothing, and a trailing "prompt-tok=0" would read as a
// measured zero rather than as no measurement.
func promptTokenSuffix(m contextMetrics) string {
if m.UsageReportedRequests == 0 {
return ""
}
return fmt.Sprintf(" prompt-tok=%d", m.PromptTokens)
}
@@ -0,0 +1,54 @@
package main
import (
"testing"
"reasonix/internal/contract/event"
"reasonix/internal/contract/provider"
)
// The cost a run reports must be what the agent reported, summed across every
// completion it billed — including one that merged attempts and one that could
// only estimate. Any other event kind is not a cost.
func TestUsageSinkAccumulatesWhatTheAgentReports(t *testing.T) {
var sink usageSink
sink.Emit(event.Event{}) // a kind that is not Usage
sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{
PromptTokens: 100, CompletionTokens: 10, TotalTokens: 110, CacheHitTokens: 60, CacheMissTokens: 40,
}})
sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{
PromptTokens: 7, CompletionTokens: 3, CacheHitTokens: 2, CacheMissTokens: 5, RequestCount: 2, Estimated: true,
}})
sink.Emit(event.Event{Kind: event.Usage}) // a Usage event with no payload is not a cost
usage, requests := sink.snapshot()
if requests != 3 {
t.Fatalf("requests = %d, want 3 (1 + the merged attempt's 2; the empty event adds none)", requests)
}
if usage.PromptTokens != 107 || usage.CompletionTokens != 13 || usage.TotalTokens != 110 {
t.Fatalf("accumulated usage = %+v", usage)
}
if usage.CacheHitTokens != 62 || usage.CacheMissTokens != 45 {
t.Fatalf("cache split = %+v", usage)
}
if !usage.Estimated {
t.Fatal("Estimated was dropped when one completion reported an estimate")
}
}
// A dry run must not claim a cost it never paid, in the JSON or in the line a
// person reads: an empty suffix, not "prompt-tok=0".
func TestUsageSinkStaysEmptyWithoutUsage(t *testing.T) {
var sink usageSink
sink.Emit(event.Event{})
usage, requests := sink.snapshot()
if requests != 0 || usage.PromptTokens != 0 || usage.CompletionTokens != 0 {
t.Fatalf("nothing reported usage, got %+v requests=%d", usage, requests)
}
if suffix := promptTokenSuffix(contextMetrics{}); suffix != "" {
t.Fatalf("a run with no reported usage still printed %q", suffix)
}
if suffix := promptTokenSuffix(contextMetrics{UsageReportedRequests: 1, PromptTokens: 12}); suffix != " prompt-tok=12" {
t.Fatalf("a reported cost printed %q", suffix)
}
}
File diff suppressed because it is too large. Load diff
+452
View File
@@ -0,0 +1,452 @@
package main
import (
"context"
"fmt"
"os"
"sort"
"strings"
"time"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/session/control"
"reasonix/internal/state/execjournal"
)
// armFleetActiveStore asks whether the sub-agent store can be asked about a
// fan-out item whose slot was granted and whose child reached the provider. It
// is not a parity arm: the store is what every recovery path reads, so what is
// measured is one store answering differently about executions it holds the
// same evidence for.
const armFleetActiveStore = "fleet-active-store"
// activeFleetCallID is the fan-out under measurement. The lone delegation in
// the same turn keeps taskCallID, so the two entry points stay separable in
// every artifact without a naming scheme of their own.
const activeFleetCallID = "probe_fleet_active"
// probeFleetChildEntered is the fan-out's half of the entry evidence. The lone
// delegation's rides probeChildEntered, which already means exactly this for
// the delegation under measurement.
const probeFleetChildEntered = "probe:fleet-child-entered"
// The four populations one death holds. They are read off the journal rather
// than off the item ids the dispatch declared: which of the two held-back items
// wins the freed slot is the scheduler's to decide, and an arm that named them
// in advance would be reporting a race as a premise.
type population string
const (
popSettled population = "fan-out settled"
popRunning population = "fan-out executing"
popRefused population = "fan-out refused"
popLone population = "lone executing"
popNone population = ""
)
// measured are the four in the order they are reported, which is also the order
// they become true: an item settles, the slot it frees admits one of the two
// behind it, and the delegation holding the other slot outlives all of them.
func measured() []population { return []population{popSettled, popRunning, popRefused, popLone} }
// populationOf places one journal entry in its population, or in none. Started
// and open is executing; started and settled is done; queued without a start is an
// item the scheduler held back and never admitted.
func populationOf(e execjournal.Entry) population {
switch {
case ofLoneTask(e.ID):
if e.Started() && e.Open() {
return popLone
}
case ofActiveFleet(e.ID):
switch {
case e.Started() && !e.Open():
return popSettled
case e.Started() && e.Open():
return popRunning
case !e.Started() && e.Queued():
return popRefused
}
}
return popNone
}
func ofActiveFleet(id string) bool { return strings.HasPrefix(id, activeFleetCallID+"/") }
// executionsIn are the ids this observation's journal places in a population,
// in a stable order.
func executionsIn(o Observation, pop population) []string {
var out []string
for _, e := range o.Executions {
if populationOf(e) == pop {
out = append(out, e.ID)
}
}
sort.Strings(out)
return out
}
// storedStatus is what the store says about one execution, empty when it has
// never heard of it. The join is the execution id the child recorded as its
// parent call, which is the same identity the journal opened it under.
func storedStatus(o Observation, id string) string {
for _, f := range o.Children.Facts {
if f.ParentToolCallID == id {
return f.Status
}
}
return ""
}
// noRecord is what the store says about an execution it never recorded. It is
// spelled out rather than left blank because a blank column reads as a row that
// was not measured, which is the opposite of what an absent record means here.
const noRecord = "no record"
func storeReading(o Observation, ids []string) string {
out := make([]string, 0, len(ids))
for _, id := range ids {
status := storedStatus(o, id)
if status == "" {
status = noRecord
}
out = append(out, id+"→"+status)
}
return orNone(join(out))
}
// acknowledged reports whether the store holds a record for every one of these
// executions. An empty set is not acknowledgement: a population the death never
// produced is a premise that failed, and its row says not-measured.
func acknowledged(o Observation, ids []string) bool {
for _, id := range ids {
if storedStatus(o, id) == "" {
return false
}
}
return len(ids) > 0
}
func activeStoreArm(name string) bool { return name == armFleetActiveStore }
// activeStoreRows read one store from four sides. Each population has the same
// two columns — what the dying process's store said, and what the next one's
// does — so the answers can only differ by which execution is being asked about.
func activeStoreRows(before, after Observation) []row {
return []row{
populationsRow(before),
childEntryRow(before),
settledStoreRow(before, after),
executingStoreRow(before, after),
loneStoreRow(before, after),
refusedStoreRow(before, after),
listingRow(before, after),
rebuildUnmovedRow(before, after),
}
}
// populationsRow is the arm's premise, not a durability claim: one death has to
// hold all four states at once, or the comparisons below are between different
// runs rather than between entry points.
func populationsRow(before Observation) row {
var got []string
for _, pop := range measured() {
got = append(got, fmt.Sprintf("%s=%d", pop, len(executionsIn(before, pop))))
}
return row{
Semantic: "the four populations this death holds", Authority: journalAuthority,
Artifact: "<stem>.execution.jsonl", Reconstruction: "in-process only",
Before: join(got), After: "—", Verdict: activePremiseVerdict(before),
}
}
func activePremiseVerdict(before Observation) string {
if activeStoreInvalid(before) != "" {
return verdictNotMeasured
}
return verdictHolds
}
// childEntryRow is the half of the evidence no artifact carries. A slot grant
// says the scheduler admitted the item; only the child's own arrival at the
// provider says the execution happened, and a store row read without it would
// be asking about work that may never have run.
func childEntryRow(before Observation) row {
fleet := firstProgress(before, probeFleetChildEntered)
lone := firstProgress(before, probeChildEntered)
return row{
Semantic: "the children that reached the provider", Authority: "the scripted provider, in-process",
Artifact: "none observed", Reconstruction: "in-process only",
Before: fmt.Sprintf("fan-out=%s lone=%s", fleet, lone),
After: "—",
Verdict: enteredVerdict(fleet == "true" && lone == "true"),
}
}
func enteredVerdict(both bool) string {
if both {
return verdictHolds
}
return verdictNotMeasured
}
func firstProgress(o Observation, key string) string {
if vals := o.Progress[key]; len(vals) > 0 {
return vals[0]
}
return "unrecorded"
}
// settledStoreRow is the control that makes the rest readable. It is the same
// entry point, the same fan-out and the same store, differing only in that this
// item finished — so if it too were absent, the finding would be about
// persistence in general rather than about executions in flight.
func settledStoreRow(before, after Observation) row {
ids := executionsIn(before, popSettled)
return row{
Semantic: "a settled fan-out item, in the store", Authority: storeAuthority,
Artifact: "subagents/<ref>.meta.json", Reconstruction: "ListSubagentsByParent, joined by execution id",
Before: storeReading(before, ids),
After: storeReading(after, ids),
Verdict: controlVerdict(acknowledged(before, ids) && acknowledged(after, ids), len(ids) > 0),
}
}
// executingStoreRow is the arm. The conjunction is stated in the row rather
// than left to the reader: the journal proves the slot was granted, the child
// proves the execution happened, and this column is what the store can say
// about it. All three, or the row is not the finding it looks like.
func executingStoreRow(before, after Observation) row {
ids := executionsIn(before, popRunning)
entered := firstProgress(before, probeFleetChildEntered) == "true"
return row{
Semantic: "an executing fan-out item, in the store", Authority: storeAuthority,
Artifact: "subagents/<ref>.meta.json",
Reconstruction: "STARTED in the journal, entered at the provider, and asked of the store",
Before: storeReading(before, ids),
After: storeReading(after, ids),
Verdict: executingVerdict(before, after, ids, entered),
}
}
// executingVerdict refuses the two readings that would overstate the row. An
// execution nothing proves entered is not a silent store, and a population the
// death never produced is not evidence of anything.
func executingVerdict(before, after Observation, ids []string, entered bool) string {
switch {
case len(ids) == 0 || !entered:
return verdictNotMeasured
case acknowledged(before, ids) || acknowledged(after, ids):
return verdictHolds
default:
return verdictViolated
}
}
// loneStoreRow is the other entry point, in the same death and the same store.
// It is what makes "selectively blind" a statement about the store rather than
// a comparison between two probe runs.
func loneStoreRow(before, after Observation) row {
ids := executionsIn(before, popLone)
return row{
Semantic: "an executing lone delegation, in the store", Authority: storeAuthority,
Artifact: "subagents/<ref>.meta.json", Reconstruction: "the same read, one entry point over",
Before: storeReading(before, ids),
After: storeReading(after, ids),
Verdict: controlVerdict(acknowledged(before, ids) && acknowledged(after, ids), len(ids) > 0),
}
}
// refusedStoreRow is the negative control, and it bounds any fix this arm might
// support. An item the scheduler never admitted produced nothing, so the store
// is owed no record for it — and a change that recorded every opening would
// satisfy the row above by reintroducing exactly that.
func refusedStoreRow(before, after Observation) row {
ids := executionsIn(before, popRefused)
silent := !acknowledged(before, ids) && !acknowledged(after, ids)
return row{
Semantic: "a refused fan-out item, in the store", Authority: crossAuthority,
Artifact: "subagents/<ref>.meta.json, <stem>.execution.jsonl",
Reconstruction: "queued, never started: the store must stay silent",
Before: storeReading(before, ids),
After: storeReading(after, ids),
Verdict: controlVerdict(silent, len(ids) > 0),
}
}
func controlVerdict(ok bool, measured bool) string {
switch {
case !measured:
return verdictNotMeasured
case ok:
return verdictHolds
default:
return verdictViolated
}
}
// listingRow is the surface a person or a model reaches for. It reads through
// the workspace filter and the lineage walk the tool applies, which the store
// read above does not — so it is checked against that read first: a listing
// that cannot even see the settled item is a filter answering, not a store.
func listingRow(before, after Observation) row {
settled := executionsIn(before, popSettled)
return row{
Semantic: "what list_subagents would show", Authority: "SubagentStore.ListForParent",
Artifact: "subagents/<ref>.meta.json",
Reconstruction: "the read behind the tool, filtered by workspace and lineage",
Before: listingReading(before),
After: listingReading(after),
Verdict: listingVerdict(before, after, settled),
}
}
func listingReading(o Observation) string {
if o.Listing.Err != "" {
return "error: " + o.Listing.Err
}
out := make([]string, 0, len(o.Listing.Rows))
for _, r := range o.Listing.Rows {
out = append(out, orDash(r.From)+"="+r.Status)
}
sort.Strings(out)
return orNone(join(out))
}
// listingVerdict answers only once the listing has been shown to work. Its
// filters can drop every row for reasons that have nothing to do with this arm,
// and an empty listing read as an invisible execution is the substitution the
// probe exists to refuse.
func listingVerdict(before, after Observation, settled []string) string {
if len(settled) == 0 || !listingHas(after, settled) {
return verdictNotMeasured
}
if listingHas(before, executionsIn(before, popRunning)) {
return verdictHolds
}
return verdictViolated
}
func listingHas(o Observation, ids []string) bool {
for _, id := range ids {
found := false
for _, r := range o.Listing.Rows {
if r.From == id {
found = true
break
}
}
if !found {
return false
}
}
return len(ids) > 0
}
// rebuildUnmovedRow pins what a fix must not move. A store that learned to
// record executions in flight would leave stale records behind for a restart to
// fold, and the one outcome this whole line of work exists to prevent is a
// rebuild that answers with a node the dead process was running.
func rebuildUnmovedRow(before, after Observation) row {
rebuilt := rebuiltGraph(after)
var ghosts []string
for _, n := range rebuilt.Graph.Nodes {
if n.State == agentgraph.StateRunning {
ghosts = append(ghosts, n.ID)
}
}
var cut []string
for _, i := range rebuilt.Interrupted {
if ofActiveFleet(i.Execution) || ofLoneTask(i.Execution) {
cut = append(cut, i.Execution+":"+interruptionKind(i))
}
}
sort.Strings(ghosts)
sort.Strings(cut)
return row{
Semantic: "no ghost, and how the rebuild names the cut", Authority: rebuildAuthority,
Artifact: "<stem>.execution.jsonl + subagents/<ref>.meta.json",
Reconstruction: "execgraph.Rebuild over both, after the death",
Before: fmt.Sprintf("%d executing at death", len(executionsIn(before, popRunning))+len(executionsIn(before, popLone))),
After: fmt.Sprintf("ghosts=%s cut=%s", orNone(join(ghosts)), orNone(join(cut))),
Verdict: controlVerdict(len(ghosts) == 0 && len(cut) > 0, len(before.Executions) > 0),
}
}
// activeStoreInvalid reports the premise this arm could not establish. All four
// populations in one death is the whole design: without them the rows compare
// different runs, and a row that compares nothing must not be read as green.
func activeStoreInvalid(before Observation) string {
for _, pop := range measured() {
if len(executionsIn(before, pop)) == 0 {
return "the death held nothing in population " + string(pop)
}
}
if firstProgress(before, probeFleetChildEntered) != "true" {
return "no fan-out child reached the provider, so no execution is proven to have happened"
}
return ""
}
// runActiveStoreConstruct drives one delegation and one fan-out under a ceiling
// they share, and dies with all four populations standing. The delegation goes
// first and hangs: it is what keeps the ceiling from emptying under the fan-out,
// so the item refused admission stays refused rather than starting a moment
// later and taking the arm's negative control with it.
func runActiveStoreConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
prompt := fanOutTurn(turn+1, taskSentinel+" "+fleetActiveSentinel)
go func() { _ = ctrl.Run(ctx, prompt) }()
if err := waitForPopulations(ctrl, prov); err != nil {
return err
}
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
// Whether each child reached the provider is known only here, and it is half
// of what makes a silent store a finding rather than a missing record for
// work that never ran.
obs.Progress[probeFleetChildEntered] = []string{fmt.Sprint(prov.fleets.held(childEntered(childHang)))}
obs.Progress[probeChildEntered] = []string{fmt.Sprint(prov.fleets.held(childEntered(childHold)))}
if err := writeObservation(root, obs); err != nil {
return err
}
os.Exit(0)
return nil
}
// waitForPopulations blocks until the journal holds all four states, then holds
// still and asks again. A refusal is a state nothing announces, so the second
// reading is what separates an item the scheduler is holding back from one that
// was merely slow to be admitted.
func waitForPopulations(ctrl *control.Controller, prov *scripted) error {
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
if populationsStanding(ctrl, prov) {
time.Sleep(2 * time.Second)
if !populationsStanding(ctrl, prov) {
return errUnexpected("a refusal that was still standing",
journalStates(control.ExecutionHistory(ctrl.SessionPath())))
}
return nil
}
time.Sleep(50 * time.Millisecond)
}
return errUnexpected("one death holding all four populations",
journalStates(control.ExecutionHistory(ctrl.SessionPath())))
}
func populationsStanding(ctrl *control.Controller, prov *scripted) bool {
held := map[population]bool{}
for _, e := range control.ExecutionHistory(ctrl.SessionPath()) {
held[populationOf(e)] = true
}
return held[popSettled] && held[popRunning] && held[popRefused] && held[popLone] &&
prov.fleets.held(childEntered(childHang)) && prov.fleets.held(childEntered(childHold))
}
func journalStates(entries []execjournal.Entry) string {
out := make([]string, 0, len(entries))
for _, e := range entries {
out = append(out, fmt.Sprintf("%s:started=%t,queued=%t,open=%t", e.ID, e.Started(), e.Queued(), e.Open()))
}
sort.Strings(out)
return orNone(join(out))
}
+345
View File
@@ -0,0 +1,345 @@
package main
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"reasonix/internal/state/sessionstore"
"strings"
"reasonix/internal/contract/config"
"reasonix/internal/contract/provider"
"reasonix/internal/state/memory"
)
// arm is one death-and-resume run. Lever runs between the two processes and is
// what separates the three compaction questions: nothing changed, only the
// stable prefix changed, the covered conversation changed.
type arm struct {
name string
asks string
lever func(armRoot, Observation) error
}
// appendsAfterFold names the arms whose construct phase adds one turn after
// the fold. Every arm carrying a lever needs it: without it the projection is
// already gone at the boundary for an unrelated reason, and the lever's own
// effect cannot be read out of the result.
func appendsAfterFold(name string) bool {
return name != "exact" && name != armRefoldIntoBody && !graphArm(name) && !uiArm(name) &&
!schedulerWaitArm(name) && !terminalArm(name) && !deriveArm(name) &&
!loneTaskArm(name) && !cancelRoutingArm(name) && !lineageArm(name) && !activeStoreArm(name) &&
!skillArm(name) && !ephemeralArm(name) && !hostExecArm(name)
}
// armAppendAfterFold separates the two ways a projection can fail validation
// after a restart: a covered prefix that no longer matches, and a projection
// that covers the whole transcript, whose only cross-process validation is a
// transcript version the reload does not carry.
const armAppendAfterFold = "append-after-fold"
// armRefoldIntoBody folds twice. The second fold reads the provider-visible
// view — stored body plus live tail — and must record a boundary in canonical
// terms, which the body has no counterpart for.
const armRefoldIntoBody = "refold-into-body"
// The tail arms separate two ways the live tail can move. tail-edit and
// tail-truncate change the transcript past the fold, where the covered hash
// cannot notice; tail-rewind takes the path a person actually drives, which
// throws the projection away without asking whether coverage still holds.
const (
armTailEdit = "tail-edit"
armTailTruncate = "tail-truncate"
armTailRewind = "tail-rewind"
// armCoveredRewind is the negative control the rewind path will need before
// its invalidation can become conditional: rewinding below the fold removes
// history the digest folded, and no coverage check may excuse that.
armCoveredRewind = "covered-rewind"
// armTodoIdentity moves host step identity while a fold is between the
// model and it: identity A, fold, live tail, identity B, fold again.
armTodoIdentity = "todo-identity"
// armOpenDecision dies while a question is open. It asks what the host
// believes afterwards, not whether one particular card comes back.
armOpenDecision = "open-decision"
// The three successors to an interrupted barrier. Each starts where
// open-decision ends and differs only in what the next process does, which
// is the whole question: when does a recorded interruption stop being the
// context every later request carries?
armIdleRestart = "interrupted-idle"
armUnrelatedTurn = "interrupted-unrelated"
armDirectAnswer = "interrupted-answered"
)
// successorTurn is what the middle process says, empty when it only restarts.
func successorTurn(arm string) (string, bool) {
switch arm {
case armIdleRestart:
return "", true
case armUnrelatedTurn:
return marker(120) + " List the files you would look at first.", true
case armDirectAnswer:
return marker(121) + " Yes, go ahead with that — take the side below the fold.", true
}
return "", false
}
func arms() []arm {
return []arm{
{name: "exact", asks: "nothing changed between the processes"},
{name: armAppendAfterFold, asks: "nothing changed, and one turn was appended after the fold"},
{name: armRefoldIntoBody, asks: "a second fold reaches into the body the first one stored"},
{name: "system-swap", asks: "only the stable prefix changed, against the surviving baseline", lever: swapSystemPrefix},
{name: armTailEdit, asks: "a message past the fold changed", lever: editTailRow},
{name: armTailTruncate, asks: "the messages past the fold were dropped", lever: truncateTail},
{name: armTailRewind, asks: "the same truncation, driven through the rewind a person uses"},
{name: armCoveredRewind, asks: "a rewind that lands below the fold boundary"},
{name: armTodoIdentity, asks: "host step identity moved across two folds and a live tail"},
{name: armOpenDecision, asks: "the process died while a decision was open"},
{name: armIdleRestart, asks: "an interrupted barrier, and nobody does anything"},
{name: armUnrelatedTurn, asks: "an interrupted barrier, then a turn about something else"},
{name: armDirectAnswer, asks: "an interrupted barrier, then a turn that reads like an answer to it"},
{name: armGraphCompleted, asks: "a fan-out whose items had all settled when the process died"},
{name: armGraphRunning, asks: "a fan-out with a child still executing when the process died"},
{name: armGraphMixed, asks: "a fan-out holding completed, failed, adopted and running at once"},
{name: armUIGraphMixed, asks: "which door each fact reaches the frontend through, after a death"},
{name: armTaskCompleted, asks: "a lone delegation that finished, in a turn that closed"},
{name: armTaskRunning, asks: "a lone delegation still executing when the process died"},
{name: armTaskFgQueued, asks: "what a lone delegation is drawn as while the scheduler holds it back"},
{name: armTaskBgQueued, asks: "a backgrounded delegation the ceiling refused, with its job already handed over"},
{name: armTaskBgRunning, asks: "a backgrounded delegation executing inside its job"},
{name: armCancelTool, asks: "whether a stop reaches an ordinary tool the turn is waiting on"},
{name: armCancelBackground, asks: "whether a stop leaves work already handed to a job alone"},
{name: armCancelHeadlessOwner, asks: "who owns the cancellation of a turn the controller never admitted"},
{name: armLineageForeground, asks: "control: does a stop reach the tree the turn owns?"},
{name: armLineageBackground, asks: "does it also reach the tree it handed away?"},
{name: armLineageJobKill, asks: "control: does a job's own stop reach its tree?"},
{name: armTaskFgQueuedCancel, asks: "a delegation stopped while the scheduler still held it back"},
{name: armTaskBgQueuedCancel, asks: "the same, backgrounded: a job killed before it was ever admitted"},
{name: armTaskFgRunningCancel, asks: "a delegation stopped while its child was executing"},
{name: armTaskBgRunningCancel, asks: "the same, backgrounded: a job killed while its child ran"},
{name: armWaitSlots, asks: "an item the session's total ceiling refused"},
{name: armWaitWriters, asks: "an item the writer ceiling refused, with total capacity free"},
{name: armWaitClaim, asks: "an item a path conflict refused, with both ceilings free"},
{name: armWaitTransition, asks: "what a reported wait cause does when the thing it named stops holding the item"},
{name: armTerminalAdopted, asks: "an item that never ran and still holds an answer"},
{name: armTerminalSkippedDep, asks: "an item that never ran and holds none, because its dependency failed"},
{name: armTerminalCancelled, asks: "an item a cancellation ended before it was admitted"},
{name: armTerminalContext, asks: "whether a delivered answer's edge follows from what is already durable"},
{name: armChildTerminal, asks: "whether the store keeps every terminal the graph draws"},
{name: armDeriveSkipBoth, asks: "a skip whose two upstreams both ended without an answer"},
{name: armDeriveSkipFlip, asks: "the same, with the two failures in the other order"},
{name: armIdentitySemantics, asks: "what a node's model and effort report, across two producers and the store"},
{name: armDeriveAnswered, asks: "a dependent whose upstreams answered, one by completing and one by adopting"},
{name: armFleetActiveStore, asks: "whether the store can be asked about a fan-out item that is executing"},
{name: armSkillQueued, asks: "whether a subagent skill asks the same scheduler as every other delegation"},
{name: armSkillRunning, asks: "what names a subagent skill while its child is executing"},
{name: armSkillCompleted, asks: "what a finished subagent skill leaves, through both entry points"},
{name: armSkillCancel, asks: "what a stop reaches when a subagent skill is running"},
{name: armReadOnlySkillQueued, asks: "whether the read-only skill entry point asks the same scheduler"},
{name: armReadOnlySkillRunning, asks: "what names a read-only skill while its child is executing"},
{name: armReadOnlySkillCompleted, asks: "what a finished read-only skill leaves behind, if anything"},
{name: armEphemeralTask, asks: "can the execution authority account for what read_only_task draws?"},
{name: armEphemeralSkill, asks: "the same of read_only_skill, the other ephemeral entry point"},
{name: armHostRunning, asks: "what a slash-started subagent leaves when the process dies mid-execution"},
{name: armHostQueued, asks: "what names slash-started work the scheduler is holding back"},
{name: armHostCompleted, asks: "what a finished slash invocation leaves besides its answer"},
{name: armHostCancel, asks: "what a stop reaches, and what history it leaves"},
{name: armHeadlessCompleted, asks: "does the headless host surface leave the same provenance?"},
{name: armHeadlessRunningCancel, asks: "the same, stopped through the context its caller holds"},
{name: armHeadlessQueuedCancel, asks: "the same while the scheduler is still holding it back"},
{name: "covered-mutation", asks: "the covered conversation changed, against the surviving baseline", lever: mutateCoveredRow},
}
}
// swapSystemPrefix saves a memory fact into this arm's home. The lever is a
// real user action rather than a poked field: the saved-fact index folds into
// the boot-time system prompt, so the next process composes a different one.
func swapSystemPrefix(root armRoot, _ Observation) error {
store := memory.StoreFor(config.RootsForHome(root.Home).MemoryUserDir(), root.Workspace)
_, err := store.Save(memory.Memory{
Name: "probe-prefix-lever",
Title: "Probe prefix lever",
Description: "A fact saved between the two processes so the composed system prompt differs.",
Type: memory.TypeProject,
Scope: memory.FactScopeProject,
// Pinned, not relevant: only a pinned body rides the stable prefix. The
// saved-fact index is retrieval-only and would move nothing.
Activation: memory.ActivationPinned,
Body: "The runtime-resume probe wrote this to move the stable prefix.",
})
return err
}
// mutateCoveredRow rewrites one provider-visible message inside the prefix the
// projection folded, through the same rewrite save a rewind or a recovery
// branch uses. This is the control arm: the digest's material changed, so the
// projection must not be reused.
func mutateCoveredRow(_ armRoot, before Observation) error {
session, err := sessionstore.LoadSession(before.SessionPath)
if err != nil {
return err
}
msgs := session.Snapshot()
covered := min(before.Sidecar.CoveredCount, len(msgs))
target := -1
for i := range covered {
if msgs[i].Role == provider.RoleUser || msgs[i].Role == provider.RoleAssistant {
target = i
break
}
}
if target < 0 {
return errUnexpected("a covered conversation row to mutate", covered)
}
msgs[target].Content += "\n[probe: covered row mutated]"
session.Replace(msgs)
return session.SaveRewrite(before.SessionPath)
}
// editTailRow rewrites one message the projection does not claim to cover. The
// covered hash cannot see it, so the projection should keep serving.
func editTailRow(_ armRoot, before Observation) error {
return rewriteSession(before, func(msgs []provider.Message, covered int) ([]provider.Message, error) {
for i := covered; i < len(msgs); i++ {
if msgs[i].Role == provider.RoleUser || msgs[i].Role == provider.RoleAssistant {
msgs[i].Content += "\n[probe: tail row edited]"
return msgs, nil
}
}
return nil, errUnexpected("a live tail row to edit", covered)
})
}
// truncateTail drops everything past the fold boundary, leaving the covered
// prefix exactly as the projection folded it.
func truncateTail(_ armRoot, before Observation) error {
return rewriteSession(before, func(msgs []provider.Message, covered int) ([]provider.Message, error) {
return msgs[:covered], nil
})
}
// rewriteSession applies a transcript rewrite through the save a rewind or a
// recovery branch uses, refusing an arm with no live tail to act on.
func rewriteSession(before Observation, edit func([]provider.Message, int) ([]provider.Message, error)) error {
session, err := sessionstore.LoadSession(before.SessionPath)
if err != nil {
return err
}
msgs := session.Snapshot()
covered := before.Sidecar.CoveredCount
if covered <= 0 || covered >= len(msgs) {
return errUnexpected("a live tail past the fold", covered)
}
next, err := edit(msgs, covered)
if err != nil {
return err
}
session.Replace(next)
return session.SaveRewrite(before.SessionPath)
}
func orchestrate(work, only, jsonOut string, keep bool) error {
self, err := os.Executable()
if err != nil {
return err
}
if work == "" {
work, err = os.MkdirTemp("", "runtime-resume-")
if err != nil {
return err
}
if !keep {
defer os.RemoveAll(work)
}
}
var results []armResult
for _, a := range arms() {
if only != "" && a.name != only {
continue
}
result, err := runArm(self, work, a)
if err != nil {
return fmt.Errorf("arm %s: %w", a.name, err)
}
results = append(results, result)
}
if len(results) == 0 {
return fmt.Errorf("no arm matched %q", only)
}
fmt.Print(renderMatrix(results, work))
return writeJSON(jsonOut, results)
}
func runArm(self, work string, a arm) (armResult, error) {
root := rootFor(filepath.Join(work, a.name))
if err := root.requireFresh(); err != nil {
return armResult{}, err
}
if err := root.create(); err != nil {
return armResult{}, err
}
if err := spawn(self, "construct", root, a.name); err != nil {
return armResult{}, fmt.Errorf("construct process: %w", err)
}
before, err := readObservation(root, "construct")
if err != nil {
return armResult{}, err
}
if a.lever != nil {
if err := a.lever(root, before); err != nil {
return armResult{}, fmt.Errorf("lever: %w", err)
}
}
if _, ok := successorTurn(a.name); ok {
if err := spawn(self, "successor", root, a.name); err != nil {
return armResult{}, fmt.Errorf("successor process: %w", err)
}
}
if err := spawn(self, "resume", root, a.name); err != nil {
return armResult{}, fmt.Errorf("resume process: %w", err)
}
after, err := readObservation(root, "resume")
if err != nil {
return armResult{}, err
}
var extra *Observation
if _, ok := successorTurn(a.name); ok {
obs, err := readObservation(root, "successor")
if err != nil {
return armResult{}, err
}
extra = &obs
} else if phase := extraPhase(a.name); phase != "" {
obs, err := readObservation(root, phase)
if err != nil {
return armResult{}, err
}
extra = &obs
}
return classify(a, extra, before, after), nil
}
// spawn runs one phase as a real child process and waits for it to exit. The
// OS process is the measurement boundary: a nil-ed object can still be revived
// by a package-level singleton, a live sink, or a cache, and none of those
// survive here.
func spawn(self, phase string, root armRoot, armName string) error {
cmd := exec.Command(self, "-phase="+phase, "-root="+root.Dir, "-arm="+armName)
cmd.Env = childEnv(root)
cmd.Dir = root.Workspace
cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
return cmd.Run()
}
func writeJSON(path string, results []armResult) error {
if strings.TrimSpace(path) == "" {
return nil
}
b, err := marshalResults(results)
if err != nil {
return err
}
return os.WriteFile(path, b, 0o644)
}
+230
View File
@@ -0,0 +1,230 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"time"
"reasonix/internal/contract/provider"
"reasonix/internal/session/control"
)
// Where a stop goes — and, first, whose stop it is. A controller owns the turns
// it admits itself and cancels those; a synchronous Run belongs to the caller
// that passed the context. Measuring the second with the first says nothing
// about either.
const (
// armCancelTool is the control that says where a defect lives. If an
// ordinary tool's context is cancelled and a delegation's is not, the fault
// is in how a delegation derives its context; if neither is, it is upstream
// of both.
armCancelTool = "cancel-ordinary-tool"
// armCancelBackground is the negative control, and the one that decides what
// a fix may do: work already handed to a job has an owner of its own, and a
// cancel that reached into it would take back the handoff.
armCancelBackground = "cancel-background-handoff"
// armCancelHeadlessOwner reads the other surface. A synchronous Run is never
// admitted through the gate, so the controller holds no cancel for it and
// the caller's own context holds the only one. Both halves are asserted: a
// controller that started claiming this would be taking ownership from the
// only holder that has it.
armCancelHeadlessOwner = "cancel-headless-owner"
)
func cancelRoutingArm(name string) bool {
switch name {
case armCancelTool, armCancelBackground, armCancelHeadlessOwner:
return true
}
return false
}
// guardedArm names the arms driven through a turn the controller admits itself,
// which is the surface a person's Stop reaches.
func guardedArm(name string) bool {
return cancelTaskArm(name) || name == armCancelTool || name == armCancelBackground ||
name == armSkillCancel || name == armHostCancel
}
const (
sleepSentinel = "PROBE-SLEEP"
sleepCallID = "probe_sleep"
// sleepSeconds outlasts the settling this probe waits through, so a shell
// that ends promptly ended because something ended it.
sleepSeconds = 30
)
// sleepCall is an ordinary foreground tool that does nothing but wait. It is
// the shell every turn can call, which is the point: it derives its context the
// way every other tool does.
func sleepCall() []provider.Chunk {
args, _ := json.Marshal(map[string]any{
"command": fmt.Sprintf("sleep %d", sleepSeconds),
"description": "waits until something stops it",
})
return []provider.Chunk{{
Type: provider.ChunkToolCall,
ToolCall: &provider.ToolCall{ID: sleepCallID, Name: "bash", Arguments: string(args)},
}}
}
// runCancelRoutingConstruct drives the arm's work, stops it the way that
// surface's owner stops it, and records what the stop reached. Nothing waits
// for the work to end: whether it ends at all is the measurement.
func runCancelRoutingConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
sentinel := sleepSentinel
if arm == armCancelBackground {
sentinel = taskSentinel
}
prompt := fanOutTurn(turn+1, sentinel)
ctx, cancelCaller := context.WithCancel(ctx)
defer cancelCaller()
if guardedArm(arm) {
ctrl.Send(prompt)
} else {
go func() { _ = ctrl.Run(ctx, prompt) }()
}
if err := waitForCancelSubject(sink, prov, arm); err != nil {
return err
}
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
obs.Progress[probeStopReached] = stopReadings(ctrl, sink, prov, arm, cancelCaller)
if err := writeObservation(root, obs); err != nil {
return err
}
os.Exit(0)
return nil
}
// probeStopReached carries what each stop reached into the observation. The key
// names the probe so no row reads it as a status the kernel emitted.
const probeStopReached = "probe:stop-reached"
// stopReadings issues this surface's stop and reports what it reached. The
// headless arm issues both in order — the controller's, which owns nothing
// there, and then the caller's, which owns everything.
func stopReadings(ctrl *control.Controller, sink *graphSink, prov *scripted, arm string, cancelCaller context.CancelFunc) []string {
// What the stop found before it was issued. Work already ended would make
// every answer below meaningless, and this probe has made that mistake once.
before := "live"
if !workLive(sink, prov, arm) {
before = "already gone"
}
ctrl.Cancel()
first := "controller-cancel before=" + before + " " + reachedAfter(ctrl, sink, prov, arm)
if arm != armCancelHeadlessOwner {
return []string{first}
}
cancelCaller()
return []string{first, "caller-cancel " + reachedAfter(ctrl, sink, prov, arm)}
}
// reachedAfter waits out the settling this probe allows and reports what is
// still live. The wait stays well below the shell's own sleep, so work that has
// ended was ended by the stop rather than by finishing.
func reachedAfter(ctrl *control.Controller, sink *graphSink, prov *scripted, arm string) string {
stopped := time.Now()
deadline := stopped.Add(8 * time.Second)
for time.Now().Before(deadline) && workLive(sink, prov, arm) {
time.Sleep(100 * time.Millisecond)
}
live := "gone"
if workLive(sink, prov, arm) {
live = "still running"
}
return fmt.Sprintf("turn=%s work=%s after=%s",
turnStanding(ctrl, arm), live, time.Since(stopped).Round(time.Second))
}
func workLive(sink *graphSink, prov *scripted, arm string) bool {
if arm == armCancelBackground {
return prov.fleets.held(childEntered(childHang)) && !prov.fleets.held(childLeft(childHang))
}
_, ended := sink.toolResult(sleepCallID)
return !ended
}
// turnStanding reports the gate only where the gate is the authority. A
// synchronous Run is never admitted through it, so reading it there would say
// "ended" about a turn it was never told about — which is how this probe first
// mistook an ownership boundary for a cancellation defect.
func turnStanding(ctrl *control.Controller, arm string) string {
if !guardedArm(arm) {
return "not admitted (caller-owned)"
}
if ctrl.Running() {
return "still active"
}
return "ended"
}
// waitForCancelSubject blocks until the work this arm is about to stop is
// really under way: a stop that arrived before it would measure nothing.
func waitForCancelSubject(sink *graphSink, prov *scripted, arm string) error {
deadline := time.Now().Add(60 * time.Second)
for time.Now().Before(deadline) {
if arm == armCancelBackground {
if prov.fleets.held(childEntered(childHang)) {
return nil
}
} else if _, seen := sink.toolDispatched(sleepCallID); seen {
// The shell starts with the dispatch, not with the event; a moment's
// grace keeps the stop from racing its own subject.
time.Sleep(500 * time.Millisecond)
return nil
}
time.Sleep(50 * time.Millisecond)
}
return errUnexpected("work under way to stop", "nothing started")
}
// cancelRoutingRows read the arm's own readings. The three arms want different
// answers to the same question, which is why they run together: a fix that
// cancels everything a session knows about passes one and fails two.
func cancelRoutingRows(arm string, before Observation) []row {
readings := before.Progress[probeStopReached]
first, second := reading(readings, 0), reading(readings, 1)
switch arm {
case armCancelHeadlessOwner:
return []row{
ownershipRow("a controller stop on a turn it never admitted", first,
"leaves the caller's work alone", stillRunning(first)),
ownershipRow("the caller's own stop", second,
"ends the work it owns", !stillRunning(second)),
}
case armCancelBackground:
return []row{
ownershipRow("the turn after a stop", first, "a cancelled turn ends",
strings.Contains(first, "turn=ended")),
ownershipRow("work already handed to a job", first,
"a stop aimed at the turn leaves work the turn no longer owns", stillRunning(first)),
}
}
return []row{
ownershipRow("the turn after a stop", first, "a cancelled turn ends",
strings.Contains(first, "turn=ended")),
ownershipRow("the tool the turn was waiting on", first,
"a stop reaches the work its turn is waiting on", !stillRunning(first)),
}
}
func ownershipRow(semantic, got, want string, ok bool) row {
return row{
Semantic: semantic, Authority: "the surface that owns this cancellation",
Artifact: "none: in-process state", Reconstruction: want,
Before: want, After: orNone(got), Verdict: held(ok),
}
}
func reading(readings []string, at int) string {
if at < len(readings) {
return readings[at]
}
return ""
}
func stillRunning(reading string) bool { return strings.Contains(reading, "work=still running") }
+156
View File
@@ -0,0 +1,156 @@
package main
import (
"fmt"
"strings"
)
// How a host can stand after dying with a question open. The names are the
// outcomes, not a pass and its failures: two of them are designs a host may
// legitimately choose, and only measurement says which one this is.
const (
outcomeResumable = "resumable" // the question is still there to answer
outcomeInterrupted = "interrupted-explicit" // it is gone, and the host says a turn was cut
outcomeLostSilent = "LOST-SILENT" // it is gone and nothing records that it existed
)
// decisionRows classify what a restart inherited. The safety row comes first
// because it is the only one with a single acceptable answer: whether the
// decision survives is a design question, whether the effect it blocked stayed
// blocked is not.
func decisionRows(before, after Observation) []row {
return []row{
deferredRow(before, after),
outcomeRow(before, after),
barrierRow(before, after),
modelContextRow(before, after),
obligationRow(before, after),
}
}
// deferredRow asserts the write the question held back never ran — not while
// the answer was pending, and not because a later process loaded the session.
func deferredRow(before, after Observation) row {
verdict := verdictHolds
if before.Deferred.Executed || after.Deferred.Executed {
verdict = verdictViolated
}
return row{
Semantic: "the effect the decision held back", Authority: "the workspace",
Artifact: before.Deferred.MarkerPath, Reconstruction: "the blocked write never runs, before or after the boundary",
Before: ranOrNot(before), After: ranOrNot(after), Verdict: verdict,
}
}
func ranOrNot(o Observation) string {
if o.Deferred.Executed {
return "RAN"
}
return "did not run"
}
// outcomeRow names which of the three standings the restart produced. It is
// stated, not judged: a host that drops the question and records the
// interruption is a defensible design, and one that drops both is not.
func outcomeRow(before, after Observation) row {
verdict := verdictHolds
var outcome string
switch {
case len(after.Decisions) > 0:
outcome = outcomeResumable
case len(after.Interrupted) > 0:
outcome = outcomeInterrupted
case after.Obligation.InterruptionMarked > 0 || len(after.Obligation.UnansweredCalls) > 0:
outcome = outcomeInterrupted
default:
outcome = outcomeLostSilent
verdict = verdictViolated
}
return row{
Semantic: "what the restart inherited", Authority: "approvalManager and the transcript",
Artifact: "none for the question; the transcript for the obligation",
Reconstruction: "resumable, interrupted-explicit, or nothing at all",
Before: decisionSummary(before), After: outcome, Verdict: verdict,
}
}
func decisionSummary(o Observation) string {
if len(o.Decisions) == 0 {
return "no decision open"
}
var kinds []string
for _, d := range o.Decisions {
kinds = append(kinds, string(d.Kind))
}
return strings.Join(kinds, ", ") + " open"
}
// barrierRow is the evidence behind an interrupted classification: a barrier
// the host recorded before anyone could be asked, still open, with no owner in
// this process waiting on it. It is deliberately not offered as answerable.
func barrierRow(before, after Observation) row {
return row{
Semantic: "durable adjudication barriers", Authority: "the adjudication log",
Artifact: "<stem>.adjudication.jsonl", Reconstruction: "open records with no live owner in this process",
Before: barrierSummary(before), After: barrierSummary(after), Verdict: verdictStable,
}
}
func barrierSummary(o Observation) string {
if len(o.Interrupted) == 0 {
return "none"
}
var out []string
for _, b := range o.Interrupted {
out = append(out, b.Kind+":"+b.ID)
}
return join(out) + " (not answerable)"
}
// modelContextRow is the other read surface: what the next request tells the
// model. One block when an interruption stands, none when it does not, and it
// must never arrive as something answerable.
func modelContextRow(before, after Observation) row {
verdict := verdictHolds
if before.ModelSeesInterruption != expectedBlocks(before) || after.ModelSeesInterruption != expectedBlocks(after) {
verdict = verdictViolated
}
return row{
Semantic: "what the next request tells the model", Authority: "control, projected per request",
Artifact: "none: derived, never stored", Reconstruction: "one block while an interruption stands, none otherwise",
Before: modelContextSummary(before), After: modelContextSummary(after), Verdict: verdict,
}
}
func expectedBlocks(o Observation) int {
if len(o.Interrupted) > 0 {
return 1
}
return 0
}
func modelContextSummary(o Observation) string {
return fmt.Sprintf("%d block(s) for %d interruption(s)", o.ModelSeesInterruption, len(o.Interrupted))
}
// obligationRow shows the evidence behind that classification, so a reader can
// see what the host still says rather than trusting the label.
func obligationRow(before, after Observation) row {
return row{
Semantic: "what the transcript still owes", Authority: "the canonical transcript",
Artifact: "<stem>.jsonl", Reconstruction: "tool calls with no result, and interruption records",
Before: obligationSummary(before), After: obligationSummary(after), Verdict: verdictStable,
}
}
func obligationSummary(o Observation) string {
return fmt.Sprintf("unanswered %s, interruption records %d",
orNone(join(o.Obligation.UnansweredCalls)), o.Obligation.InterruptionMarked)
}
func orNone(s string) string {
if s == "" {
return "none"
}
return s
}
+354
View File
@@ -0,0 +1,354 @@
package main
import (
"context"
"os"
"reasonix/internal/state/sessionstore"
"sort"
"strings"
"time"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/session/control"
"reasonix/internal/state/execgraph"
)
// The derivability arms. Everything before them asked what survives; these ask
// what still has to be written down. A fact a restart can compute from what is
// already durable needs no record of its own, and adding one would give it a
// second owner for nothing.
const (
// armDeriveSkipBoth cuts a branch with two upstreams that both end without
// an answer. Which of them the graph names is the question: the durable
// facts show two, and the picture shows one.
armDeriveSkipBoth = "derive-skip-both"
// armDeriveSkipFlip is the same graph with the two failures in the other
// order. If the named cause moves while the durable facts do not, the cause
// is history rather than a consequence of the end state.
armDeriveSkipFlip = "derive-skip-flip"
// armDeriveAnswered is the negative control and the context sample at once:
// two upstreams that both answer, one by completing and one by adopting, so
// the dependent must run and both deliveries must be accounted for.
armDeriveAnswered = "derive-answered"
)
func deriveArm(name string) bool {
switch name {
case armDeriveSkipBoth, armDeriveSkipFlip, armDeriveAnswered:
return true
}
return false
}
func deriveSentinels(arm string) string {
if arm == armDeriveAnswered {
return fleetWarmSentinel + " " + fleetDeriveSentinel
}
return fleetDeriveSentinel
}
// runDeriveConstruct lets the fan-out settle and dies before the turn ends, the
// way every terminal arm does: a skip is published only in the closing delta.
func runDeriveConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
runCtx, cancelRun := context.WithCancel(ctx)
defer cancelRun()
go func() { _ = ctrl.Run(runCtx, fanOutTurn(turn+1, deriveSentinels(arm))) }()
if err := waitForDerive(sink, arm, prov); err != nil {
return err
}
if err := writeObservation(root, capture("construct", arm, bootSystem, ctrl, sink, root)); err != nil {
return err
}
os.Exit(0)
return nil
}
// waitForDerive holds until the arm's shape is on the graph. The skip arms
// release their second failure once the first has been recorded, so which
// upstream the fan-out names is decided rather than raced.
func waitForDerive(sink *graphSink, arm string, prov *scripted) error {
if arm == armDeriveAnswered {
// Waiting on a count would be satisfied by the warm-up alone. The
// delivery edge is what the arm compares, so it waits for that.
return waitForShape(sink, func(g agentgraph.Graph) bool {
return len(contextTargets(g)) > 0 && len(nodesInState(g, agentgraph.StateAdopted)) > 0
}, "an adopted upstream and a delivered answer")
}
// The first failure decides which upstream is named — the fan-out cuts the
// branch when it processes that result. Only then is the second released,
// so the order is constructed rather than raced.
if err := waitForShape(sink, func(g agentgraph.Graph) bool {
return len(nodesInState(g, agentgraph.StateFailed)) > 0
}, "the first upstream to fail"); err != nil {
return err
}
prov.releaseOnce()
// A skip is published only in the closing delta, so the group has to end
// before the state the arm compares exists at all.
return waitForShape(sink, func(g agentgraph.Graph) bool {
return len(nodesInState(g, agentgraph.StateSkipped)) > 0 &&
len(nodesInState(g, agentgraph.StateFailed)) >= 2
}, "both upstreams failed and the dependent skipped")
}
// contextTargets are the items the graph shows receiving an upstream answer.
func contextTargets(g agentgraph.Graph) []string {
var out []string
for _, e := range g.Edges {
if e.Kind == agentgraph.Context {
out = append(out, e.To)
}
}
return out
}
func waitForShape(sink *graphSink, ok func(agentgraph.Graph) bool, want string) error {
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
graph, _ := sink.snapshot()
if ok(graph) {
return nil
}
time.Sleep(50 * time.Millisecond)
}
graph, _ := sink.snapshot()
return errUnexpected(want, waitStates(graph))
}
// answeredFromDurable is Answered() computed the way a restart would have to:
// an adoption says so on its own opening, and everything else is the terminal
// the sub-agent store kept, joined by the execution id the child records.
func answeredFromDurable(o Observation) map[string]bool {
terminal := map[string]string{}
for _, f := range o.Children.Facts {
if f.ParentToolCallID != "" {
terminal[f.ParentToolCallID] = f.Status
}
}
out := map[string]bool{}
for _, e := range o.Executions {
out[e.ID] = e.Disposition == "adopted" || terminal[e.ID] == string(sessionstore.SubagentCompleted)
}
return out
}
// impliedSkipped is the state a restart would derive: an item the orchestration
// released without ever starting it, ordered behind something that did not
// answer. Nothing here reads a disposition, because that is what is missing.
func impliedSkipped(o Observation) string {
answered := answeredFromDurable(o)
var out []string
for _, e := range o.Executions {
if e.Started() || e.Queued() || e.SettledAt.IsZero() || len(e.DependsOn) == 0 {
continue
}
for _, up := range e.DependsOn {
if !answered[up] {
out = append(out, e.ID)
break
}
}
}
sort.Strings(out)
return join(out)
}
// impliedSkipCauses are every upstream that could have been named, and the two
// rules that would pick one: declaration order, and the earliest release. A
// cause is derivable only if some rule always agrees with the picture.
func impliedSkipCauses(o Observation) (all, byOrder, bySettle string) {
answered := answeredFromDurable(o)
byID := map[string]int{}
for i, e := range o.Executions {
byID[e.ID] = i
}
var every, first, earliest []string
for _, e := range o.Executions {
if e.Started() || e.Queued() || e.SettledAt.IsZero() || len(e.DependsOn) == 0 {
continue
}
var unanswered []string
for _, up := range e.DependsOn {
if !answered[up] {
unanswered = append(unanswered, up)
}
}
if len(unanswered) == 0 {
continue
}
every = append(every, e.ID+"<-{"+strings.Join(unanswered, ",")+"}")
first = append(first, e.ID+"<-"+unanswered[0])
pick := unanswered[0]
for _, up := range unanswered[1:] {
if i, j := byID[up], byID[pick]; o.Executions[i].SettledAt.Before(o.Executions[j].SettledAt) {
pick = up
}
}
earliest = append(earliest, e.ID+"<-"+pick)
}
sort.Strings(every)
sort.Strings(first)
sort.Strings(earliest)
return join(every), join(first), join(earliest)
}
// actualSkipCause is the upstream the fan-out actually named, read off the node
// the graph settled. It is the picture's own answer, not a derivation.
func actualSkipCause(o Observation) string {
var out []string
for _, n := range o.Graph.Nodes {
if n.State == agentgraph.StateSkipped && n.Err != "" {
out = append(out, n.ID+": "+n.Err)
}
}
sort.Strings(out)
return orNone(join(out))
}
// impliedContextFromDurable predicts the delivery edges from durable facts
// alone: a started item receives an answer from every upstream that answered.
func impliedContextFromDurable(o Observation) string {
answered := answeredFromDurable(o)
var out []string
for _, e := range o.Executions {
if !e.Started() {
continue
}
for _, up := range e.DependsOn {
if answered[up] {
out = append(out, e.ID+"<-"+up)
}
}
}
sort.Strings(out)
return join(out)
}
// deriveRows are the three questions, kept apart. A state that derives says
// nothing about whether the cause does, and one arm's agreement says nothing
// about a rule that has only ever seen one sample.
func deriveRows(arm string, before, after Observation) []row {
rows := []row{
joinRow(before, after),
skipStateRow(before, after),
contextDeriveRow(before, after),
}
if arm != armDeriveAnswered {
rows = append(rows, skipCauseRow(before, after))
}
return rows
}
// joinRow is what every derivation below stands on: the child records name the
// execution they ran for, so a restart can ask the store what each item ended
// as. Without it none of the other rows could be computed at all.
func joinRow(before, after Observation) row {
return valueRow("executions the store can be asked about", "child metadata's parent tool-call id",
"subagents/<ref>.meta.json", "joined to the journal by execution id",
joinedExecutions(before), joinedExecutions(after), true)
}
func joinedExecutions(o Observation) string {
var out []string
for _, f := range o.Children.Facts {
if f.ParentToolCallID != "" {
out = append(out, f.ParentToolCallID+"="+f.Status)
}
}
sort.Strings(out)
return join(out)
}
// skipStateRow compares the state a restart would derive against the one the
// picture drew. Agreement means StateSkipped needs no record of its own.
func skipStateRow(before, after Observation) row {
actual := join(nodesInState(graphOf(before), agentgraph.StateSkipped))
implied := impliedSkipped(after)
verdict := verdictHolds
if implied != actual {
verdict = verdictViolated
}
return row{
Semantic: "skipped state: derived against drawn", Authority: "the journal joined to the store",
Artifact: "<stem>.execution.jsonl + subagents/<ref>.meta.json",
Reconstruction: "settled, never started, ordered behind something that did not answer",
Before: "drawn " + orNone(actual), After: "derived " + orNone(implied), Verdict: verdict,
}
}
// skipCauseRow keeps the harder question separate, and reads it three ways: the
// upstream the picture named, the two rules this file works out on its own, and
// what the production fold concludes. The independence matters — a probe that
// only asked the implementation would agree with it by construction.
func skipCauseRow(before, after Observation) row {
all, byOrder, bySettle := impliedSkipCauses(after)
return row{
Semantic: "skipped cause: what the picture named, what the facts permit",
Authority: "the fan-out's own choice against the durable upstream set",
Artifact: "none: the reason text is not recorded",
Reconstruction: "declaration order, or earliest released, over the upstreams that did not answer",
Before: actualSkipCause(before),
After: "candidates " + orNone(all) + " | by-order " + orNone(byOrder) +
" | by-settle " + orNone(bySettle) + " | rebuild " + orNone(rebuiltSkipCauses(after)),
Verdict: verdictStable,
}
}
// rebuiltSkipCauses is what the production fold names, asked the same question
// through its own code path.
func rebuiltSkipCauses(o Observation) string {
children := make([]execgraph.ChildOutcome, 0, len(o.Children.Facts))
for _, f := range o.Children.Facts {
children = append(children, execgraph.ChildOutcome{
Execution: f.ParentToolCallID, Ref: f.Ref, Status: f.Status,
})
}
rebuilt := execgraph.Rebuild(o.Executions, children, nil)
var out []string
for _, n := range rebuilt.Graph.Nodes {
if n.State != agentgraph.StateSkipped {
continue
}
if cause := execgraph.SkipCause(o.Executions, children, n.ID); cause != "" {
out = append(out, n.ID+"<-"+cause)
}
}
sort.Strings(out)
return join(out)
}
// contextDeriveRow asks the same question of the delivery edge, now that a
// restart can tell which upstreams answered.
func contextDeriveRow(before, after Observation) row {
actual, implied := contextEdges(before), impliedContextFromDurable(after)
verdict := verdictHolds
if implied != actual {
verdict = verdictViolated
}
return row{
Semantic: "context edges: derived against drawn", Authority: "the journal joined to the store",
Artifact: "<stem>.execution.jsonl + subagents/<ref>.meta.json",
Reconstruction: "a started item receives from every upstream that answered",
Before: "drawn " + orNone(actual), After: "derived " + orNone(implied), Verdict: verdict,
}
}
// deriveArmInvalid refuses an arm that did not reach the shape it compares.
func deriveArmInvalid(arm string, before Observation) string {
if arm == armDeriveAnswered {
if len(nodesInState(graphOf(before), agentgraph.StateAdopted)) == 0 {
return "no item was adopted, so the arm never had an answered upstream that did not run"
}
if contextEdges(before) == "" {
return "no answer was delivered, so there is no context edge to derive"
}
return ""
}
if len(nodesInState(graphOf(before), agentgraph.StateSkipped)) == 0 {
return "no dependent was skipped; the fan-out ended as " + orNone(terminalSummary(before))
}
if len(nodesInState(graphOf(before), agentgraph.StateFailed)) < 2 {
return "only one upstream ended without an answer, so the cause was never ambiguous"
}
return ""
}
+239
View File
@@ -0,0 +1,239 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"sort"
"strings"
"time"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/contract/provider"
"reasonix/internal/session/control"
)
// The two ephemeral entry points, read against the execution read model's own
// authority: the snapshot is what a reader starts from and what a gap restores,
// and the stream is a projection of the same facts. A node only the stream has
// is therefore not a lighter node but one the authority cannot account for, and
// these arms ask whether either entry point produces one.
const (
armEphemeralTask = "ephemeral-task-authority"
armEphemeralSkill = "ephemeral-skill-authority"
)
func ephemeralArm(name string) bool {
return name == armEphemeralTask || name == armEphemeralSkill
}
const (
ephemeralSentinel = "PROBE-EPHEMERAL"
// ephemeralCallID is the call the model makes. Both entry points use it, so
// the two arms differ by which tool ran and by nothing else.
ephemeralCallID = "probe_ephemeral"
)
// ephemeralCall dispatches the entry point this arm measures. Its child hangs,
// because every row here is about a run that is still happening: an ephemeral
// execution that has ended owes no reader anything.
func ephemeralCall(arm string) []provider.Chunk {
name, args := "read_only_task", map[string]any{
"prompt": childHang + " ephemeral research", "description": "still executing at death",
}
if arm == armEphemeralSkill {
name, args = "read_only_skill", map[string]any{
"name": probeSkillName, "arguments": childHang + " ephemeral research",
}
}
raw, _ := json.Marshal(args)
return []provider.Chunk{{
Type: provider.ChunkToolCall,
ToolCall: &provider.ToolCall{ID: ephemeralCallID, Name: name, Arguments: string(raw)},
}}
}
// runEphemeralConstruct drives one ephemeral run and reads three surfaces at the
// same instant, while its child is inside the provider: what the stream drew,
// what the authority holds, and what a reader is left with once a gap makes it
// start from the authority again.
func runEphemeralConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
go func() { _ = ctrl.Run(ctx, fanOutTurn(turn+1, ephemeralSentinel)) }()
if err := waitForEphemeral(prov); err != nil {
return err
}
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
obs.Progress[probeChildEntered] = []string{fmt.Sprint(prov.fleets.held(childEntered(childHang)))}
// One instant, three readings. Taken here because two of them cannot be
// taken later at all: the authority is recomputed per call, and what a gap
// leaves is a claim about the moment the gap happens.
live, _ := sink.snapshot()
authority := ctrl.ExecutionGraph()
obs.Progress[probeEphemeralLive] = []string{ephemeralNodes(live)}
obs.Progress[probeEphemeralAuthority] = []string{ephemeralNodes(authority.Graph)}
obs.Progress[probeEphemeralAfterGap] = []string{ephemeralNodes(afterGap(authority.Graph))}
if err := writeObservation(root, obs); err != nil {
return err
}
os.Exit(0)
return nil
}
// The three readings one instant produces. They ride the observation because
// none of them survives the process, and the arm's whole question is what a
// reader could see while the run was happening.
const (
probeEphemeralLive = "probe:ephemeral-live"
probeEphemeralAuthority = "probe:ephemeral-authority"
probeEphemeralAfterGap = "probe:ephemeral-after-gap"
)
// afterGap is what a view holds once a dropped stream makes it start from the
// authority again: the snapshot, and only the snapshot. A resync replaces the
// folded state rather than merging into it — that is what makes the snapshot the
// authority — so whatever the fold held and it does not has left the screen.
func afterGap(authority agentgraph.Graph) agentgraph.Graph { return authority }
func waitForEphemeral(prov *scripted) error {
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
if prov.fleets.held(childEntered(childHang)) {
// Let the picture settle: a reading taken the instant the child
// arrives could miss a delta that was already on its way.
time.Sleep(time.Second)
return nil
}
time.Sleep(50 * time.Millisecond)
}
return errUnexpected("an ephemeral child inside the provider", "it never reached one")
}
// ephemeralNodes renders the nodes belonging to this arm's run, worker and group
// alike: a group card drawn for work the authority cannot account for is the
// same defect as a worker one.
func ephemeralNodes(g agentgraph.Graph) string {
var out []string
for _, n := range g.Nodes {
if strings.HasPrefix(n.ID, ephemeralCallID) {
out = append(out, n.ID+":"+orDash(string(n.State)))
}
}
sort.Strings(out)
return orNone(join(out))
}
func ephemeralRows(arm string, before, after Observation) []row {
return []row{
ephemeralLiveRow(before),
ephemeralAuthorityRow(before),
ephemeralGapRow(before),
ephemeralRestartRow(before, after),
ephemeralDurableRow(before, after),
ephemeralContractRow(arm, before),
}
}
const authorityName = "Controller.ExecutionGraph (the snapshot a reader starts from)"
func ephemeralLiveRow(before Observation) row {
drawn := firstProgress(before, probeEphemeralLive)
return row{
Semantic: "what the stream drew while it ran", Authority: liveAuthority,
Artifact: "none observed", Reconstruction: "in-process only",
Before: drawn, After: "—", Verdict: presenceVerdict(drawn != "none"),
}
}
func ephemeralAuthorityRow(before Observation) row {
held := firstProgress(before, probeEphemeralAuthority)
return row{
Semantic: "what the authority held at the same instant", Authority: authorityName,
Artifact: "<stem>.execution.jsonl + subagents/<ref>.meta.json",
Reconstruction: "execgraph.Rebuild over durable facts, recomputed on demand",
Before: held, After: "—", Verdict: presenceVerdict(held != "none"),
}
}
// ephemeralGapRow is the one that decides. A reader that loses the stream starts
// from the authority again, so whatever the fold held and the authority does not
// leaves the screen — while the child it named is still holding a slot.
func ephemeralGapRow(before Observation) row {
left := firstProgress(before, probeEphemeralAfterGap)
return row{
Semantic: "what a reader is left with after a gap", Authority: authorityName,
Artifact: "none: the fold is replaced, never merged",
Reconstruction: "a resync starts from the authority",
Before: left, After: "—", Verdict: presenceVerdict(left != "none"),
}
}
func ephemeralRestartRow(before, after Observation) row {
return row{
Semantic: "what a restart can say about it", Authority: rebuildAuthority,
Artifact: "<stem>.execution.jsonl",
Reconstruction: "execgraph.Rebuild in the next process",
Before: "executing inside the provider",
After: ephemeralNodes(rebuiltGraph(after).Graph),
Verdict: presenceVerdict(ephemeralNodes(rebuiltGraph(after).Graph) != "none"),
}
}
// ephemeralDurableRow is the contract both entry points already promise. It is
// read rather than assumed, because a convergence that made one of them visible
// by recording it would satisfy the rows above and break this one.
func ephemeralDurableRow(before, after Observation) row {
return presenceRow("journal and store", crossAuthority,
"<stem>.execution.jsonl, subagents/<ref>.meta.json",
"what an ephemeral entry point promises to leave: nothing",
ephemeralDurable(before), ephemeralDurable(after))
}
func ephemeralDurable(o Observation) []string {
var out []string
for _, e := range o.Executions {
if strings.HasPrefix(e.ID, ephemeralCallID) {
out = append(out, "journal:"+e.ID)
}
}
for _, f := range o.Children.Facts {
if strings.HasPrefix(f.ParentToolCallID, ephemeralCallID) {
out = append(out, "store:"+f.ParentToolCallID+"="+f.Status)
}
}
sort.Strings(out)
return out
}
// ephemeralContractRow states the authority contract as a conjunction. A node
// the stream drew, an authority that cannot account for it, and a child still
// executing: that is not a lighter kind of node, it is one that leaves the
// screen at the next gap while the work it named goes on holding a slot.
func ephemeralContractRow(arm string, before Observation) row {
live := firstProgress(before, probeEphemeralLive) != "none"
authority := firstProgress(before, probeEphemeralAuthority) != "none"
running := firstProgress(before, probeChildEntered) == "true"
answer, verdict := "no node was drawn: nothing claims what the authority cannot hold", verdictHolds
switch {
case !running:
answer, verdict = "the child never reached the provider", verdictNotMeasured
case live && !authority:
answer, verdict = "drawn by the stream, absent from the authority, and still executing", verdictViolated
case live && authority:
answer, verdict = "drawn, and the authority accounts for it", verdictHolds
}
return row{
Semantic: "a node the authority cannot account for", Authority: crossAuthority,
Artifact: "none, which is the point",
Reconstruction: "the stream and the snapshot, read at one instant",
Before: answer, After: "—", Verdict: verdict,
}
}
// ephemeralArmInvalid reports the premise this arm could not establish.
func ephemeralArmInvalid(before Observation) string {
if firstProgress(before, probeChildEntered) != "true" {
return "the ephemeral child never reached the provider, so nothing was executing at the reading"
}
return ""
}
+641
View File
@@ -0,0 +1,641 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"regexp"
"slices"
"strings"
"time"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/contract/provider"
"reasonix/internal/session/control"
)
// The three fan-out arms. Each dies at a different point in one fleet's life,
// because "what survives" has no single answer: an item that finished, one
// still executing, and one the graph knows about without ever running are
// carried by different host state, and only separating them says which.
const (
// armGraphCompleted lets the fleet finish and the turn close before the
// process exits. Nothing is interrupted, so a loss here is not about
// interruption at all — it says execution provenance itself is ephemeral.
armGraphCompleted = "graph-completed"
// armGraphRunning kills the process with a child mid-execution. It asks the
// one question a completed fleet cannot: what does the new process believe
// about work whose owner is gone?
armGraphRunning = "graph-running"
// armGraphMixed carries every state at once — completed, running, adopted,
// failed, skipped — plus a read grant, a write grant, and a dependency edge.
// It is the density arm: one death, every semantic the graph claims to hold.
armGraphMixed = "graph-mixed"
)
func graphArm(name string) bool {
return name == armGraphCompleted || name == armGraphRunning || name == armGraphMixed
}
// Child sentinels. A child's prompt decides what its run does, and the probe
// reads the prompt off the request's last user message rather than scanning the
// whole conversation: the parent's own transcript quotes every fleet argument,
// so a scan would make the parent answer as its own children.
const (
childDone = "PROBE-CHILD-DONE"
childHang = "PROBE-CHILD-HANG"
childFail = "PROBE-CHILD-FAIL"
)
// Parent sentinels, one per fleet the arm dispatches.
const (
fleetWarmSentinel = "PROBE-FLEET-WARM"
fleetMixedSentinel = "PROBE-FLEET-MIXED"
fleetPairSentinel = "PROBE-FLEET-PAIR"
// The scheduler arms. Holder fills a ceiling and never ends; refused asks
// for admission the scheduler must deny. Two sentinels rather than one
// because the claim and transition arms need the holder to be already
// admitted before the second fleet is dispatched.
fleetHolderSentinel = "PROBE-FLEET-HOLDER"
fleetRefusedSentinel = "PROBE-FLEET-REFUSED"
// The terminal arms. Their fan-outs run to completion, because a skip is
// published only in the closing delta; the turn is then held open so the
// process still dies mid-turn.
fleetTerminalSentinel = "PROBE-FLEET-TERMINAL"
// Named so it is not contained in PROBE-PARALLEL-IDENTITY. Sentinels are
// matched by substring, so one inside another dispatches a fan-out its arm
// never asked for — this pair silently ran cancelParallel in the identity
// arm, whose rows then read a scenario nobody wrote.
parallelSentinel = "PROBE-PARALLEL-CANCEL"
fleetOutcomesSentinel = "PROBE-FLEET-OUTCOMES"
fleetDeriveSentinel = "PROBE-FLEET-DERIVE"
fleetIdentitySentinel = "PROBE-FLEET-IDENTITY"
parallelIdentity = "PROBE-PARALLEL-IDENTITY"
// The active-store arm. Its fan-out shares one ceiling with a lone
// delegation, so the four populations it needs are reached in one death.
fleetActiveSentinel = "PROBE-FLEET-ACTIVE"
)
// childHold is a child that reports holding its slot before it blocks, so a
// later fleet can be dispatched knowing the ceiling is already occupied. Timing
// it any other way races the scheduler.
const (
childHold = "PROBE-CHILD-HOLD"
// childRelease finishes only when the arm frees it, which is how capacity
// is returned while a refusal is still standing.
childRelease = "PROBE-CHILD-RELEASE"
// childFailLate fails only once the arm frees it, so two failures in one
// fan-out have a decided order instead of a raced one. Its wording shares no
// prefix with childFail: the sentinels are matched by substring, so a name
// containing another's makes the shorter one win and the delay disappear.
childFailLate = "PROBE-CHILD-DELAYEDFAIL"
)
// probeClaimPath is the write path the claim arm overlaps. The refused writer
// asks for a path inside it, which is a conflict no ceiling change releases.
const (
probeClaimPath = "probe-claim"
probeClaimInner = "probe-claim/inner"
)
// subagentRefPattern finds the references an earlier fleet's aggregate reported.
// The mixed arm adopts one of them, which is the only way to reach StateAdopted
// through the tool's own contract rather than by constructing the state.
var subagentRefPattern = regexp.MustCompile(`sa_[A-Za-z0-9_-]+`)
// childScript answers one delegated run. Hanging is expressed as a context read
// so the child holds its scheduler slot until the process dies — a sleep long
// enough to look the same would still be a race against the probe's deadline.
func (s *scripted) childScript(ctx context.Context, sentinel string) []provider.Chunk {
// Reaching here is the child's own proof that it started, and returning is
// its proof that something let it go: an arm asking whether a cancellation
// reached a running child has no other way to know.
s.fleets.first(childEntered(sentinel))
defer s.fleets.first(childLeft(sentinel))
switch sentinel {
case childHold:
// Reaching a provider call means the slot is already held, which is the
// only moment a later fleet can be dispatched against a full ceiling
// without racing it.
s.holding.Do(func() { close(s.held) })
<-ctx.Done()
return nil
case childRelease:
<-s.release
return append(text("Child released."), done())
case childFailLate:
<-s.release
return []provider.Chunk{{Type: provider.ChunkError, Err: fmt.Errorf("probe child failed on release")}}
case childHang:
<-ctx.Done()
// The context closed, and this is the only place that can say so: a
// caller that stopped waiting leaves the same absence behind.
s.fleets.first(childCtxDone(sentinel))
return nil
case childFail:
return []provider.Chunk{{Type: provider.ChunkError, Err: fmt.Errorf("probe child failed on purpose")}}
default:
return append(text("Child answered."), done())
}
}
// askedInPrompt reports whether this request's own user turns carry a sentinel.
// The user side only: a fleet's arguments live in an assistant message, so a
// whole-conversation scan makes a parent answer as one of its own children.
// Every user turn, not the last: the host appends its own notes after the
// prompt, and the last user message in a request is frequently one of those.
func askedInPrompt(req provider.Request, sentinel string) bool {
for _, m := range req.Messages {
if m.Role == provider.RoleUser && strings.Contains(m.Content, sentinel) {
return true
}
}
return false
}
// childSentinel reports which child script this request is, empty when the
// request is a parent turn.
func childSentinel(req provider.Request) string {
for _, sentinel := range []string{childDone, childHang, childFail, childHold, childRelease, childFailLate} {
if askedInPrompt(req, sentinel) {
return sentinel
}
}
return ""
}
// fanOutTurnRecorded reports whether the dispatching turn is in these messages.
// The probe reads its own sentinel rather than counting rows: a turn appended
// at some later point would still be there, and only the prompt says whether
// this is the turn the fan-out was opened in.
func fanOutTurnRecorded(msgs []provider.Message) bool {
for _, m := range msgs {
if strings.Contains(m.Content, fleetPairSentinel) || strings.Contains(m.Content, fleetMixedSentinel) {
return true
}
}
return false
}
// fleetCall renders one fleet dispatch. Nothing closes the round it opens: all
// three arms die inside the turn, so the host's final-answer readiness check
// never runs and the arms measure a process boundary rather than a task list.
func fleetCall(id string, tasks []map[string]any) []provider.Chunk {
args, _ := json.Marshal(map[string]any{"tasks": tasks})
return []provider.Chunk{{
Type: provider.ChunkToolCall,
ToolCall: &provider.ToolCall{ID: id, Name: "fleet", Arguments: string(args)},
}}
}
// warmFleet finishes two children so a later fleet has a completed reference to
// adopt. Both are read-only: nothing about the warm-up is under measurement.
func warmFleet() []provider.Chunk {
return fleetCall("probe_fleet_warm", []map[string]any{
{"id": "w1", "prompt": childDone + " warm one", "description": "warm one", "read_only": true},
{"id": "w2", "prompt": childDone + " warm two", "description": "warm two", "read_only": true},
})
}
// pairFleet is the two-item shape both single-state arms use. The running arm
// dies with the second item still executing; the completed arm lets both
// finish, so the two differ by when the process exits and nothing else.
func pairFleet(second string) []provider.Chunk {
return fleetCall("probe_fleet_pair", []map[string]any{
{"id": "p1", "prompt": childDone + " pair one", "description": "pair one", "read_only": true},
{"id": "p2", "prompt": second + " pair two", "description": "pair two", "read_only": true},
})
}
// mixedFleet is the density arm's dispatch: one item per terminal the graph can
// record, plus one still running, and the two grants told apart. The write item
// declares a path so preflight admits it beside the readers; it never writes,
// because the grant is the envelope under measurement, not the effect.
func mixedFleet(adoptRef string) []provider.Chunk {
return fleetCall("probe_fleet_mixed", []map[string]any{
// Model and effort are named so the identity row measures something: two
// nodes agreeing that a field is empty proves nothing about recovering it.
{"id": "m1", "prompt": childDone + " mixed one", "description": "completes", "read_only": true,
"model": probeModelRef, "effort": "high"},
{"id": "m2", "prompt": childFail + " mixed two", "description": "fails", "read_only": true},
{"id": "m3", "prompt": childDone + " mixed three", "description": "skipped by m2", "read_only": true, "depends_on": []string{"m2"}},
{"id": "m4", "adopt_ref": adoptRef, "description": "adopted"},
{"id": "m5", "prompt": childHang + " mixed five", "description": "running at death", "write_paths": []string{"probe-fanout-out"}},
})
}
// opens reports whether this round is the one that dispatches that fleet.
func (s *scripted) opens(req provider.Request, sentinel string) bool {
return askedInPrompt(req, sentinel) && s.fleets.first(sentinel)
}
// opensRun is the same for the fan-out an arm is built around: the terminal and
// derive arms hold the turn open once theirs has gone out.
func (s *scripted) opensRun(req provider.Request, sentinel string) bool {
if !s.opens(req, sentinel) {
return false
}
s.opened.Store(true)
return true
}
// fanOut answers a parent turn that opens a fleet, at most once per sentinel.
// The sentinel is read off the current user turn, and the latch is what stops
// the round that receives the aggregate from dispatching the same fleet again.
func (s *scripted) fanOut(req provider.Request) ([]provider.Chunk, bool) {
switch {
case s.opens(req, fleetWarmSentinel):
return append(warmFleet(), done()), true
case s.opens(req, fleetPairSentinel):
if s.arm == armWaitSlots {
return append(slotsFleet(), done()), true
}
second := childDone
if s.arm == armGraphRunning {
second = childHang
}
return append(pairFleet(second), done()), true
case s.opens(req, fleetSettledSentinel):
return append(settledFleet(), done()), true
case s.opens(req, fleetLiveSentinel):
return append(liveFleet(), done()), true
case s.opens(req, fleetMixedSentinel):
if s.arm == armTerminalAdopted {
return append(adoptOnlyFleet(adoptableRef(req)), done()), true
}
if s.arm == armUIGraphMixed {
return append(uiMixedFleet(adoptableRef(req)), done()), true
}
return append(mixedFleet(adoptableRef(req)), done()), true
case s.opensRun(req, fleetIdentitySentinel):
return append(identityFleet(adoptableRef(req)), done()), true
case s.opens(req, parallelIdentity):
return append(identityParallel(), done()), true
case s.opensRun(req, fleetDeriveSentinel):
return append(deriveFleet(s.arm, adoptableRef(req)), done()), true
case s.opensRun(req, fleetOutcomesSentinel):
return append(outcomesFleet(), done()), true
case s.opensRun(req, fleetTerminalSentinel):
return append(terminalFleet(s.arm), done()), true
case s.opensRun(req, parallelSentinel):
return append(cancelParallel(), done()), true
case s.fleets.held(taskSentinel) && s.opens(req, fleetActiveSentinel):
// The delegation goes first and its own child says when the slot is
// taken. Dispatching before that lets the fan-out take the whole
// ceiling, and the arm loses the entry point it exists to compare.
<-s.held
return append(activeStoreFleet(), done()), true
case s.opens(req, fleetHolderSentinel):
return append(holderFleet(s.arm), done()), true
case s.opens(req, fleetRefusedSentinel):
// The holder's own child says when its ceiling is occupied. Dispatching
// before that lets the refused item win the race and start.
<-s.held
return append(refusedFleet(s.arm), done()), true
}
return nil, false
}
// holderFleet occupies the ceiling this arm measures. The slots arm needs no
// holder — its refused item is in the same fleet as the runs that fill the
// ceiling — so only the arms that need one dispatch it in the background.
func holderFleet(arm string) []provider.Chunk {
tasks := []map[string]any{
{"id": "h1", "prompt": childHold + " holder", "description": "holds the ceiling", "write_paths": []string{probeClaimPath}},
{"id": "h2", "prompt": childHang + " filler", "description": "filler", "read_only": true},
}
if arm == armWaitTransition {
// The second run is what the arm later releases: freeing total capacity
// while the writer ceiling stays full is how the blocker changes.
tasks[1] = map[string]any{"id": "h2", "prompt": childRelease + " releasable", "description": "released mid-wait", "read_only": true}
}
if bothHoldersReport(arm) {
tasks[1] = map[string]any{"id": "h2", "prompt": childHold + " second holder", "description": "holds the ceiling", "read_only": true}
}
return backgroundFleetCall("probe_fleet_holder", tasks)
}
// bothHoldersReport names the arms that leave one slot free. Either holder may
// win it, so both report holding, and the refused delegation's own child is then
// the only arrival that means anything — a filler that hung under the same
// sentinel would be read as the subject having started.
func bothHoldersReport(arm string) bool {
return queuedTaskArm(arm) || queuedSkillArm(arm) || queuedHostArm(arm)
}
// refusedFleet asks for admission the scheduler must deny, with every check
// above this arm's cause already cleared.
func refusedFleet(arm string) []provider.Chunk {
refused := map[string]any{"id": "r1", "prompt": childHang + " refused", "description": "refused admission"}
switch arm {
case armWaitWriters:
// Disjoint from the holder's path, so only the writer ceiling can
// refuse it.
refused["write_paths"] = []string{"probe-writers-own"}
case armWaitClaim:
refused["write_paths"] = []string{probeClaimInner}
default:
refused["write_paths"] = []string{"probe-transition-own"}
}
return fleetCall("probe_fleet_refused", []map[string]any{
refused,
{"id": "r2", "prompt": childHang + " sibling", "description": "sibling", "read_only": true},
})
}
// terminalFleet runs to completion holding the disposition its arm measures.
// The skipped arm cuts a branch with a failure; the context arm delivers a real
// answer across an ordering edge, which is the only way a context edge appears.
func terminalFleet(arm string) []provider.Chunk {
upstream := map[string]any{"id": "up", "prompt": childDone + " upstream", "description": "upstream", "read_only": true}
if arm == armTerminalSkippedDep {
upstream["prompt"] = childFail + " upstream"
}
return fleetCall("probe_fleet_terminal", []map[string]any{
upstream,
{"id": "down", "prompt": childDone + " downstream", "description": "downstream", "read_only": true, "depends_on": []string{"up"}},
})
}
// cancelParallel opens more items than the session admits at once, so the arm
// can interrupt the group while one has started and another has not. That split
// is what parallel_tasks reports as cancelled against skipped.
func cancelParallel() []provider.Chunk {
args, _ := json.Marshal(map[string]any{"tasks": []map[string]any{
{"prompt": childHang + " first", "description": "started"},
{"prompt": childHang + " second", "description": "never started"},
{"prompt": childHang + " third", "description": "never started"},
}})
return []provider.Chunk{{
Type: provider.ChunkToolCall,
ToolCall: &provider.ToolCall{ID: "probe_parallel", Name: "parallel_tasks", Arguments: string(args)},
}}
}
// adoptOnlyFleet runs to completion with one item adopted and one that ran.
// Nothing hangs: the arm compares two settled items whose only difference is
// that one executed, which a fan-out still in flight could not show.
func adoptOnlyFleet(adoptRef string) []provider.Chunk {
return fleetCall("probe_fleet_terminal", []map[string]any{
{"id": "a1", "adopt_ref": adoptRef, "description": "adopted"},
{"id": "a2", "prompt": childDone + " ran", "description": "ran", "read_only": true},
})
}
// outcomesFleet reaches all three executed terminals in one group: one item
// completes, one fails, and two are still running when the arm interrupts. The
// store's record of each is then compared under identical conditions.
func outcomesFleet() []provider.Chunk {
return fleetCall("probe_fleet_outcomes", []map[string]any{
{"id": "o1", "prompt": childDone + " completes", "description": "completes", "read_only": true},
{"id": "o2", "prompt": childFail + " fails", "description": "fails", "read_only": true},
{"id": "o3", "prompt": childHang + " cancelled", "description": "cancelled", "read_only": true},
{"id": "o4", "prompt": childHang + " cancelled too", "description": "cancelled too", "read_only": true},
})
}
// identityFleet names a worker identity four ways: not at all, by model, by
// effort, and not at all because nothing runs. What the graph then shows for
// each is what says which layer of the resolution it is reporting.
func identityFleet(adoptRef string) []provider.Chunk {
return fleetCall("probe_fleet_identity", []map[string]any{
{"id": "i1", "prompt": childDone + " default", "description": "inherits", "read_only": true},
{"id": "i2", "prompt": childDone + " override", "description": "names a model", "read_only": true,
"model": probeAltModelRef, "effort": "high"},
{"id": "i3", "prompt": childDone + " effort only", "description": "names an effort", "read_only": true,
"effort": "low"},
{"id": "i4", "adopt_ref": adoptRef, "description": "runs nothing"},
})
}
// identityParallel asks the same of the other producer, which resolves through
// a shorter chain: it has no profile layer to consult.
func identityParallel() []provider.Chunk {
args, _ := json.Marshal(map[string]any{"tasks": []map[string]any{
{"prompt": childDone + " p default", "description": "inherits"},
{"prompt": childDone + " p override", "description": "names a model",
"model": probeAltModelRef, "effort": "high"},
}})
return []provider.Chunk{{
Type: provider.ChunkToolCall,
ToolCall: &provider.ToolCall{ID: "probe_parallel_identity", Name: "parallel_tasks", Arguments: string(args)},
}}
}
// deriveFleet orders one dependent behind two upstreams. The skip arms end both
// upstreams without an answer, one immediately and one on release, so which of
// them the fan-out names is decided by the order rather than by a race; the
// answered arm gives the dependent one completed and one adopted upstream, so
// it must run and both deliveries must be accounted for.
func deriveFleet(arm, adoptRef string) []provider.Chunk {
first, second := childFail+" first", childFailLate+" second"
if arm == armDeriveSkipFlip {
first, second = childFailLate+" first", childFail+" second"
}
up := []map[string]any{
{"id": "a", "prompt": first, "description": "upstream a", "read_only": true},
{"id": "b", "prompt": second, "description": "upstream b", "read_only": true},
}
if arm == armDeriveAnswered {
up = []map[string]any{
{"id": "a", "prompt": childDone + " first", "description": "upstream a", "read_only": true},
{"id": "b", "adopt_ref": adoptRef, "description": "upstream b"},
}
}
return fleetCall("probe_fleet_derive", append(up, map[string]any{
"id": "c", "prompt": childDone + " dependent", "description": "dependent",
"read_only": true, "depends_on": []string{"a", "b"},
}))
}
// activeStoreFleet reaches three fan-out populations under one ceiling. The
// first item settles and frees the slot the other two then contend for, so one
// executes and one is refused — which of them is the scheduler's to decide, and
// the arm reads the populations off the journal rather than naming them here.
func activeStoreFleet() []provider.Chunk {
return fleetCall(activeFleetCallID, []map[string]any{
{"id": "a1", "prompt": childDone + " settles", "description": "settles", "read_only": true},
{"id": "a2", "prompt": childHang + " holds", "description": "contends for the freed slot",
"read_only": true, "depends_on": []string{"a1"}},
{"id": "a3", "prompt": childHang + " contends", "description": "contends for the freed slot",
"read_only": true, "depends_on": []string{"a1"}},
})
}
// slotsFleet fills the total ceiling and asks for one more, all read-only: a
// reader clears every writer check by not being a writer.
func slotsFleet() []provider.Chunk {
return fleetCall("probe_fleet_refused", []map[string]any{
{"id": "s1", "prompt": childHang + " one", "description": "fills a slot", "read_only": true},
{"id": "s2", "prompt": childHang + " two", "description": "fills a slot", "read_only": true},
{"id": "s3", "prompt": childHang + " three", "description": "refused admission", "read_only": true},
})
}
// backgroundFleetCall dispatches a fleet that returns a job id at once. The
// holder has to keep holding while a later fleet is dispatched, which a
// foreground call cannot do: it does not return until every item has settled.
func backgroundFleetCall(id string, tasks []map[string]any) []provider.Chunk {
args, _ := json.Marshal(map[string]any{"tasks": tasks, "run_in_background": true})
return []provider.Chunk{{
Type: provider.ChunkToolCall,
ToolCall: &provider.ToolCall{ID: id, Name: "fleet", Arguments: string(args)},
}}
}
// adoptableRef picks a reference the warm-up fleet reported. It reads the
// conversation the model reads: the aggregate names each child's reference, and
// an arm that cannot find one has not established its premise.
func adoptableRef(req provider.Request) string {
for _, m := range slices.Backward(req.Messages) {
if found := subagentRefPattern.FindAllString(m.Content, -1); len(found) > 0 {
return found[len(found)-1]
}
}
return ""
}
// fanOutTurn is the prompt that opens one fleet.
func fanOutTurn(n int, sentinel string) string {
return fmt.Sprintf("%s %s: dispatch the fan-out.", marker(n), sentinel)
}
// fanOutSentinels is what this arm's one prompt carries. The mixed arm names
// both of its fleets in the same turn: the warm-up has to finish before a
// reference can be adopted, and a fleet's aggregate returns into the same turn
// that dispatched it, so the second round dispatches the mixed fleet without
// the turn ever closing.
func fanOutSentinels(arm string) string {
if arm == armUIGraphMixed {
return fleetWarmSentinel + " " + fleetSettledSentinel + " " + fleetMixedSentinel
}
if arm == armGraphMixed {
return fleetWarmSentinel + " " + fleetMixedSentinel
}
return fleetPairSentinel
}
// runFanOutConstruct drives the arm's fan-out and exits while the turn is still
// open. Every arm dies mid-turn, including the completed one: what separates
// them is which states the graph holds at that moment, and a clean shutdown
// would cancel a running child and settle its node — the one thing a process
// that dies mid-fan-out does not do.
func runFanOutConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, turn int) error {
go func() { _ = ctrl.Run(ctx, fanOutTurn(turn+1, fanOutSentinels(arm))) }()
if err := waitForFanOut(sink, arm); err != nil {
return err
}
if err := writeObservation(root, capture("construct", arm, bootSystem, ctrl, sink, root)); err != nil {
return err
}
os.Exit(0)
return nil
}
// waitForFanOut blocks until the graph shows the shape this arm was built to
// die with. Waiting on the shape rather than on a duration is what keeps the
// arm honest: a timer that fired early would report a smaller fan-out as if
// that were what the restart inherited.
func waitForFanOut(sink *graphSink, arm string) error {
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
graph, _ := sink.snapshot()
if fanOutReady(graph, arm) {
return nil
}
time.Sleep(50 * time.Millisecond)
}
graph, _ := sink.snapshot()
return errUnexpected("a fan-out holding "+wantedText(arm), nodeStates(graph))
}
// fanOutReady is the arm's premise expressed as a graph shape. The completed
// arm waits for every worker to settle, which is the opposite condition from
// the other two rather than a weaker version of it.
func fanOutReady(g agentgraph.Graph, arm string) bool {
if arm == armGraphCompleted {
return allWorkersSettled(g)
}
return holdsAll(g, wantedStates(arm))
}
// wantedStates are the states a mid-flight arm must hold at once. Skipped is
// not among them and cannot be: a fan-out publishes a skip only in its closing
// outcome delta, so nothing reads skipped until the group has ended and nothing
// is running. An item whose dependency already failed sits at pending until
// then, which is what the mixed arm dies holding.
func wantedStates(arm string) []agentgraph.NodeState {
want := []agentgraph.NodeState{agentgraph.StateCompleted, agentgraph.StateRunning}
if arm == armGraphMixed || arm == armUIGraphMixed {
want = append(want, agentgraph.StateFailed, agentgraph.StateAdopted)
}
// The UI arm dies holding a settled fan-out beside a live one, so it also
// reaches the two states the mixed arm cannot: a skip, which is published
// only when a group closes, and an item a ceiling never admitted.
if arm == armUIGraphMixed {
want = append(want, agentgraph.StateSkipped, agentgraph.StatePending, agentgraph.StateQueued)
}
return want
}
func wantedText(arm string) string {
if arm == armGraphCompleted {
return "every worker settled"
}
return statesText(wantedStates(arm))
}
// allWorkersSettled reports whether every worker reached a terminal state. An
// empty state is not terminal here even though NodeState.Terminal calls it one:
// a node declared before it ran carries no state at all, and reading that as
// settled would let the arm die before the fan-out started.
func allWorkersSettled(g agentgraph.Graph) bool {
seen := false
for _, n := range g.Nodes {
if n.Kind != agentgraph.KindWorker {
continue
}
seen = true
if n.State == "" || !n.State.Terminal() {
return false
}
}
return seen
}
// holdsAll reports whether the graph shows at least one worker in each state.
// Group nodes are excluded: a group is the call that owns the fan-out, and its
// own running state would satisfy the running requirement without any child
// being in flight.
func holdsAll(g agentgraph.Graph, want []agentgraph.NodeState) bool {
for _, state := range want {
found := false
for _, n := range g.Nodes {
if n.Kind == agentgraph.KindWorker && n.State == state {
found = true
break
}
}
if !found {
return false
}
}
return true
}
func statesText(states []agentgraph.NodeState) string {
out := make([]string, 0, len(states))
for _, s := range states {
out = append(out, string(s))
}
return join(out)
}
func nodeStates(g agentgraph.Graph) string {
out := make([]string, 0, len(g.Nodes))
for _, n := range g.Nodes {
out = append(out, n.ID+":"+string(n.State))
}
return orNone(join(out))
}
+646
View File
@@ -0,0 +1,646 @@
package main
import (
"fmt"
"reasonix/internal/state/sessionstore"
"slices"
"sort"
"strings"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/runtime/delegation"
"reasonix/internal/state/execgraph"
"reasonix/internal/state/execjournal"
)
// How a run graph can stand after the process that built it exits. Three are
// designs a host may hold; two are defects, and not the same defect. Losing the
// graph costs a reader an explanation; a node still drawn as running with no
// owner costs them the truth — reviving actionable state is more dangerous than
// dropping it, and a running node is actionable state.
const (
graphExact = "reconstructed-exact" // nodes, edges, grants, waits, states all back
graphInterrupted = "interrupted-explicit" // the fan-out is legible and the live child is marked cut
graphLossy = "reconstructed-lossy" // it is back, minus semantics nothing can re-derive
graphLostSilent = "LOST-SILENT" // gone, with no provenance that it ever existed
graphGhost = "GHOST-RUNNING" // still says running, with no live owner anywhere
graphNotMeasured = "not-measured" // the construct phase never established the shape
)
// graphRows classify what a restart inherited from a fan-out. There are two
// outcome rows rather than one because a fan-out that died mid-flight holds two
// populations with different answers: the items that had already settled, whose
// answers the store owns, and the one still executing, which nothing on disk
// has ever been told about. One label over both reports the better half.
func graphRows(before, after Observation) []row {
return []row{
ghostRow(before, after),
dispatchTurnRow(before, after),
settledOutcomeRow(before, after),
interruptedOutcomeRow(before, after),
graphStructureRow(before, after),
graphSemanticsRow(before, after),
childProvenanceRow(before, after),
executionJournalRow(before, after),
rebuildStateRow(before, after),
rebuildTopologyRow(before, after),
rebuildEnvelopeRow(before, after),
rebuildInterruptedRow(before, after),
rebuildIdentityRow(before, after),
executionTopologyRow(before, after),
executionKindRow(before, after),
executionContextRow(before, after),
graphObligationRow(before, after),
}
}
// runningWorkers are the nodes that were executing when the process died. They
// are the arm's premise: an arm that establishes none has measured a fan-out
// nobody interrupted, whatever else it recorded.
func runningWorkers(o Observation) []string {
var out []string
for _, n := range o.Graph.Nodes {
if n.Kind == agentgraph.KindWorker && n.State == agentgraph.StateRunning {
out = append(out, n.ID)
}
}
return out
}
// executedWorkers are the items that ran to a terminal state. Adopted nodes are
// excluded on purpose: nothing executed under them, so the store owes them no
// record and counting them would read a correct store as incomplete.
func executedWorkers(o Observation) []string {
var out []string
for _, n := range o.Graph.Nodes {
if n.Kind != agentgraph.KindWorker || n.State == "" || n.State == agentgraph.StateAdopted {
continue
}
if n.State.Terminal() {
out = append(out, n.ID)
}
}
return out
}
// ghostRow asks the one question with a single acceptable answer: does anything
// the new process can read still present this work as running? A graph node and
// a store record are both such a claim, and neither has an owner behind it —
// the goroutine that would have settled them died with the first process.
func ghostRow(before, after Observation) row {
ghosts := append(runningWorkers(after), childrenInState(after, string(sessionstore.SubagentRunning))...)
verdict := verdictHolds
if len(ghosts) > 0 {
verdict = verdictViolated
}
return row{
Semantic: "work still presented as running", Authority: "the graph and the sub-agent store",
Artifact: "subagents/<ref>.meta.json",
Reconstruction: "nothing may read as running once its owner is gone",
Before: orNone(join(runningWorkers(before))) + " running at death",
After: orNone(join(ghosts)), Verdict: verdict,
}
}
// dispatchTurnRow is the row underneath every other loss here: whether the turn
// that opened the fan-out reached the transcript at all. A turn is appended
// when it ends, so a process that dies inside one takes the request with it —
// and a graph reconstruction has nothing to attach to even in principle.
func dispatchTurnRow(before, after Observation) row {
verdict := verdictPersisted
switch {
case !before.FanOutTurn:
verdict = verdictNotMeasured
case !after.FanOutTurn:
verdict = verdictLost
}
return row{
Semantic: "the turn that dispatched the fan-out", Authority: "session transcript + event log",
Artifact: "<stem>.jsonl, <stem>.events.jsonl", Reconstruction: "appended when the turn ends",
Before: recordedText(before), After: recordedText(after), Verdict: verdict,
}
}
func recordedText(o Observation) string {
if o.FanOutTurn {
return fmt.Sprintf("in the transcript (%d rows)", o.Transcript.Messages)
}
return fmt.Sprintf("absent (%d rows)", o.Transcript.Messages)
}
// settledOutcomeRow classifies the items that had already finished. Their
// answers are what a later turn reads back, so this is the row that decides
// whether a fan-out's history is durable at all.
func settledOutcomeRow(before, after Observation) row {
outcome, verdict := settledOutcome(before, after)
return row{
Semantic: "items that had settled before the death", Authority: "the graph fold and the sub-agent store",
Artifact: "subagents/<ref>.meta.json",
Reconstruction: "one durable child record per item that executed",
Before: fmt.Sprintf("%d executed: %s", len(executedWorkers(before)), graphSummary(before)),
After: outcome, Verdict: verdict,
}
}
func settledOutcome(before, after Observation) (string, string) {
executed := len(executedWorkers(before))
switch {
case len(before.Graph.Nodes) == 0:
return graphNotMeasured, verdictNotMeasured
case graphIdentical(before, after):
return graphExact, verdictExact
case executed == 0:
return graphNotMeasured, verdictNotMeasured
case len(after.Children.Facts) == 0:
return graphLostSilent, verdictViolated
case len(after.Children.Facts) < executed:
return fmt.Sprintf("%s (%d of %d children)", graphLossy, len(after.Children.Facts), executed), verdictLossy
default:
return graphLossy, verdictLossy
}
}
// interruptedOutcomeRow classifies the item that was still executing. This is
// the row the arm exists for: the graph is gone either way, and what separates
// a defensible design from a durable gap is whether anything the new process
// can read still says that work was opened and cut.
func interruptedOutcomeRow(before, after Observation) row {
outcome, verdict := interruptedOutcome(before, after)
return row{
Semantic: "the item still executing at the death", Authority: "the sub-agent store and the transcript",
Artifact: "subagents/<ref>.meta.json, <stem>.jsonl",
Reconstruction: "an interrupted child record, or an unsettled call the transcript still owes",
Before: orNone(join(runningWorkers(before))), After: outcome, Verdict: verdict,
}
}
func interruptedOutcome(before, after Observation) (string, string) {
switch {
case len(before.Graph.Nodes) == 0 || len(runningWorkers(before)) == 0:
return graphNotMeasured, verdictNotMeasured
case len(runningWorkers(after)) > 0 || len(childrenInState(after, string(sessionstore.SubagentRunning))) > 0:
return graphGhost, verdictViolated
case len(after.InterruptedExecutions) > 0:
return graphInterrupted, verdictLossy
case len(childrenInState(after, string(sessionstore.SubagentInterrupted))) > 0:
return graphInterrupted, verdictLossy
case len(after.Obligation.UnansweredCalls) > 0 || after.Obligation.InterruptionMarked > 0:
return graphInterrupted, verdictLossy
default:
return graphLostSilent, verdictViolated
}
}
// graphIdentical compares the whole graph, not its node count. Two graphs with
// the same nodes and different grants are not the same graph, and a comparison
// that only counted would have called that exact.
func graphIdentical(before, after Observation) bool {
return graphNodes(before) == graphNodes(after) &&
graphEdges(before) == graphEdges(after) &&
graphMap(before.Graph.Grants) == graphMap(after.Graph.Grants) &&
graphMap(before.Graph.Waits) == graphMap(after.Graph.Waits)
}
// graphStructureRow is the shape a reader asks for: which nodes, and the typed
// edges between them. Spawn, depends, context and adopt are separate facts, and
// a restart that kept the nodes while losing the edges kept a list, not a graph.
func graphStructureRow(before, after Observation) row {
return valueRow("run graph edges", "event sink (GraphDelta)",
"none observed", "fold GraphDelta events, if any survive",
graphEdges(before), graphEdges(after), false)
}
// graphSemanticsRow is the authority envelope and the reason a node waited.
// They are called out apart from the structure because nothing else in the
// system records them: a child's persisted metadata carries its model and its
// profile, and says nothing about what it was allowed to touch.
func graphSemanticsRow(before, after Observation) row {
return valueRow("run graph grants and waits", "event sink (GraphDelta)",
"none observed", "no durable source carries an authority envelope",
graphEnvelope(before), graphEnvelope(after), false)
}
func graphEnvelope(o Observation) string {
grants, waits := graphMap(o.Graph.Grants), graphMap(o.Graph.Waits)
if grants == "" && waits == "" {
return ""
}
return fmt.Sprintf("grants[%s] waits[%s]", orNone(grants), orNone(waits))
}
// childProvenanceRow is what the store still owns, which is the evidence behind
// an interrupted classification. It is reported as its own row so a reader can
// see the reconstruction the label stands on rather than trusting the label.
func childProvenanceRow(before, after Observation) row {
return valueRow("delegated runs the store owns", "SubagentStore, swept at boot",
"subagents/<ref>.meta.json", "ListSubagentsByParent; CleanupStaleRunning marks stale running children interrupted",
childSummary(before), childSummary(after), true)
}
// graphObligationRow is the last fallback a reader has: a fleet call the
// transcript records with no result is the host saying a fan-out was opened and
// never settled, even when nothing else remembers it.
func graphObligationRow(before, after Observation) row {
return row{
Semantic: "fan-out calls the transcript never settled", Authority: "the canonical transcript",
Artifact: "<stem>.jsonl", Reconstruction: "tool calls with no result",
Before: orNone(join(before.Obligation.UnansweredCalls)),
After: orNone(join(after.Obligation.UnansweredCalls)), Verdict: verdictStable,
}
}
// rebuiltGraph is what the production fold makes of this session's durable
// facts, join included: a record names its own execution, and an older one is
// aliased to its parent call only where the journal opened one by that id.
func rebuiltGraph(o Observation) execgraph.Result {
opened := make(map[string]bool, len(o.Executions))
for _, e := range o.Executions {
opened[e.ID] = true
}
children := make([]execgraph.ChildOutcome, 0, len(o.Children.Facts))
for _, f := range o.Children.Facts {
identity := delegation.ResolveExecutionIdentity(
sessionstore.SubagentMeta{ExecutionID: f.ExecutionID, ParentToolCallID: f.ParentToolCallID},
func(id string) bool { return opened[id] })
children = append(children, execgraph.ChildOutcome{
Execution: identity.Execution, Ref: f.Ref, Status: f.Status,
})
}
return execgraph.Rebuild(o.Executions, children, nil)
}
// rebuildStateRow compares the states the fold produces against the ones the
// dying process drew. Nodes whose owner disappeared are excluded and reported
// by their own row: putting those back as running is the one outcome this
// whole line of work exists to prevent.
func rebuildStateRow(before, after Observation) row {
rebuilt := rebuiltGraph(after)
cut := map[string]bool{}
for _, i := range rebuilt.Interrupted {
cut[i.Execution] = true
}
return valueRow("rebuilt node states", "execgraph.Rebuild over journal + store",
"<stem>.execution.jsonl + subagents/<ref>.meta.json",
"recomputed from durable facts, never restored",
statesExcept(graphOf(before), cut), statesExcept(rebuilt.Graph, cut), false)
}
// rebuildTopologyRow compares every typed edge. Spawn and depends are stated by
// the openings, adopt names its source, and context is derived — so a mismatch
// here says which of those four is not yet recoverable.
func rebuildTopologyRow(before, after Observation) row {
return valueRow("rebuilt topology", "execgraph.Rebuild over journal + store",
"<stem>.execution.jsonl", "spawn and depends stated, adopt named, context derived",
edgeSummary(graphOf(before)), edgeSummary(rebuiltGraph(after).Graph), false)
}
// rebuildEnvelopeRow is the authority and the wait provenance together: what
// each item was allowed to touch, and why admission was first refused.
func rebuildEnvelopeRow(before, after Observation) row {
return valueRow("rebuilt grants and wait causes", "execgraph.Rebuild over journal + store",
"<stem>.execution.jsonl", "recorded at the opening and at the first refusal",
envelopeSummary(graphOf(before)), envelopeSummary(rebuiltGraph(after).Graph), false)
}
// rebuildInterruptedRow accounts for the nodes the state row leaves out. They
// are not dropped: the fold names them as interruptions, split by whether they
// had reached a slot. A node the dying process showed live and the rebuild
// mentions nowhere would be a loss the state row could not see.
func rebuildInterruptedRow(before, after Observation) row {
rebuilt := rebuiltGraph(after)
var got []string
for _, i := range rebuilt.Interrupted {
kind := execjournal.InterruptedBeforeStart
if i.Started {
kind = execjournal.InterruptedDuringExecution
}
got = append(got, i.Execution+":"+kind)
}
sort.Strings(got)
return valueRow("nodes the rebuild will not call live", "execgraph.Rebuild over journal + store",
"<stem>.execution.jsonl", "an execution with no owner is named, never redrawn as running",
liveAtDeath(before), join(got), false)
}
// liveAtDeath is what the dying process still showed as someone's work.
func liveAtDeath(o Observation) string {
var out []string
for _, n := range o.Graph.Nodes {
if n.Kind == agentgraph.KindGroup {
continue
}
switch n.State {
case agentgraph.StateRunning:
out = append(out, n.ID+":"+execjournal.InterruptedDuringExecution)
case agentgraph.StateQueued, agentgraph.StatePending:
out = append(out, n.ID+":"+execjournal.InterruptedBeforeStart)
}
}
sort.Strings(out)
return join(out)
}
// rebuildIdentityRow is what a reader sees on a node beyond its state: the
// label it was dispatched under, and which worker ran it. Only one of those is
// recorded, so this row is where a remaining gap shows up rather than hiding
// inside an exact verdict about states.
func rebuildIdentityRow(before, after Observation) row {
return valueRow("rebuilt node identity", "execgraph.Rebuild over journal + store",
"<stem>.execution.jsonl", "label from the opening; the worker override is recorded nowhere",
identitySummary(graphOf(before)), identitySummary(rebuiltGraph(after).Graph), false)
}
func identitySummary(g agentgraph.Graph) string {
var out []string
for _, n := range g.Nodes {
if n.Kind != agentgraph.KindWorker {
continue
}
out = append(out, fmt.Sprintf("%s[label=%s profile=%s model=%s effort=%s]",
n.ID, orNone(n.Label), orNone(n.Profile), orNone(n.Model), orNone(n.Effort)))
}
sort.Strings(out)
return join(out)
}
// statesExcept renders worker states, skipping the executions whose owner is
// gone. Those are not a state the rebuild may assert, and comparing them would
// demand exactly the ghost this refuses to draw.
func statesExcept(g agentgraph.Graph, skip map[string]bool) string {
var out []string
for _, n := range g.Nodes {
if n.Kind == agentgraph.KindGroup || skip[n.ID] {
continue
}
out = append(out, n.ID+":"+string(n.State))
}
sort.Strings(out)
return join(out)
}
func edgeSummary(g agentgraph.Graph) string {
out := make([]string, 0, len(g.Edges))
for _, e := range g.Edges {
out = append(out, fmt.Sprintf("%s-%s->%s", e.From, e.Kind, e.To))
}
sort.Strings(out)
return join(out)
}
func envelopeSummary(g agentgraph.Graph) string {
var out []string
for _, n := range g.Nodes {
if n.Grant != "" {
out = append(out, n.ID+"="+string(n.Grant))
}
if n.Wait != "" {
out = append(out, n.ID+"@"+string(n.Wait))
}
}
sort.Strings(out)
return join(out)
}
// executionJournalRow is the record written before the work started, which is
// the only artifact here that can outlive an unfinished turn. It is what a
// restart reads to say a delegation existed at all.
func executionJournalRow(before, after Observation) row {
return valueRow("delegations the journal recorded", "execution journal, written before dispatch",
"<stem>.execution.jsonl", "folded from openings and settlings",
executionSummary(before), executionSummary(after), true)
}
// executionTopologyRow is the ordering a restart inherits. An item that never
// started is explained by what it was held behind, so the topology has to
// survive the process that knew it — and survive unchanged, or the explanation
// is a different graph's.
func executionTopologyRow(before, after Observation) row {
return valueRow("declared ordering between items", "the fan-out plan, recorded at the opening",
"<stem>.execution.jsonl", "read back with the openings, never re-derived",
topologySummary(before), topologySummary(after), true)
}
func topologySummary(o Observation) string {
var out []string
for _, e := range o.Executions {
if len(e.DependsOn) == 0 {
continue
}
up := append([]string(nil), e.DependsOn...)
sort.Strings(up)
out = append(out, e.ID+"<-"+strings.Join(up, "+"))
}
if len(out) == 0 {
return ""
}
sort.Strings(out)
return join(out)
}
// executionKindRow holds the two interruptions apart against what the dying
// process actually showed. The item the graph had running must come back as one
// that reached a slot; the one its dependency blocked must come back as one
// that did not. A classification agreeing with neither is a label, not a fact.
func executionKindRow(before, after Observation) row {
running := runningWorkers(before)
verdict := verdictHolds
if len(after.InterruptedExecutions) == 0 {
verdict = verdictNotMeasured
}
for _, e := range after.InterruptedExecutions {
want := execjournal.InterruptedBeforeStart
if slices.Contains(running, e.ID) {
want = execjournal.InterruptedDuringExecution
}
if e.Interruption() != want {
verdict = verdictViolated
}
}
return row{
Semantic: "how each interruption is classified", Authority: "the execution journal's start records",
Artifact: "<stem>.execution.jsonl", Reconstruction: "started at death -> during-execution, never started -> before-start",
Before: orNone(join(running)) + " running at death", After: interruptionKinds(after), Verdict: verdict,
}
}
func interruptionKinds(o Observation) string {
if len(o.InterruptedExecutions) == 0 {
return "none"
}
out := make([]string, 0, len(o.InterruptedExecutions))
for _, e := range o.InterruptedExecutions {
out = append(out, e.ID+":"+e.Interruption())
}
sort.Strings(out)
return join(out)
}
// executionContextRow is the other side of that record: what the next request
// is told. It must state provenance and never offer a continuation, so the
// count is checked against the interruptions rather than against a fixed
// number — a block with nothing behind it is a ghost in prose.
func executionContextRow(before, after Observation) row {
verdict := verdictHolds
if blocksOwed(before) != before.ModelSeesInterruptedExecution ||
blocksOwed(after) != after.ModelSeesInterruptedExecution {
verdict = verdictViolated
}
return row{
Semantic: "what the next request says about them", Authority: "control, projected per request",
Artifact: "none: derived, never stored", Reconstruction: "one block while an interrupted execution stands, none otherwise",
Before: executionBlocks(before), After: executionBlocks(after), Verdict: verdict,
}
}
func blocksOwed(o Observation) int {
if len(o.InterruptedExecutions) > 0 {
return 1
}
return 0
}
func executionBlocks(o Observation) string {
return fmt.Sprintf("%d block(s) for %d interrupted", o.ModelSeesInterruptedExecution, len(o.InterruptedExecutions))
}
func executionSummary(o Observation) string {
if len(o.Executions) == 0 {
return ""
}
open, started := 0, 0
for _, e := range o.Executions {
if e.Open() {
open++
}
if e.Started() {
started++
}
}
return fmt.Sprintf("%d opened, %d started, %d still open, %d with no owner here",
len(o.Executions), started, open, len(o.InterruptedExecutions))
}
func childrenInState(o Observation, status string) []string {
var out []string
for _, f := range o.Children.Facts {
if f.Status == status {
out = append(out, f.Ref)
}
}
return out
}
func childSummary(o Observation) string {
if len(o.Children.Facts) == 0 {
if o.Children.Err != "" {
return "error: " + o.Children.Err
}
return ""
}
counts := map[string]int{}
for _, f := range o.Children.Facts {
counts[f.Status]++
}
keys := make([]string, 0, len(counts))
for k := range counts {
keys = append(keys, k)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, k := range keys {
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
}
return fmt.Sprintf("%d child(ren): %s", len(o.Children.Facts), strings.Join(parts, " "))
}
// graphSummary is the fan-out as the dying process last saw it, which is the
// "before" every graph row is compared against.
func graphSummary(o Observation) string {
if len(o.Graph.Nodes) == 0 {
return "no fan-out"
}
counts := map[string]int{}
for _, n := range o.Graph.Nodes {
if n.Kind == agentgraph.KindWorker {
counts[string(n.State)]++
}
}
keys := make([]string, 0, len(counts))
for k := range counts {
keys = append(keys, k)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, k := range keys {
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
}
return fmt.Sprintf("%d node(s), %d edge(s): %s",
len(o.Graph.Nodes), len(o.Graph.Edges), strings.Join(parts, " "))
}
// graphArmInvalid refuses an arm that never reached the state it names. The
// running arms need a child in flight at death; every fan-out arm needs a
// fan-out at all. Reporting a smaller shape as the thing that was interrupted
// is how a probe answers a question nobody asked.
func graphArmInvalid(arm string, before Observation) string {
if len(before.Graph.Nodes) == 0 {
return "no fan-out was published before the process exited, so nothing was measured"
}
if arm == armGraphCompleted {
if len(runningWorkers(before)) > 0 {
return "the fan-out was still running at exit, so this is not the completed arm"
}
return ""
}
if len(runningWorkers(before)) == 0 {
return "no child was executing when the process died, so nothing was interrupted"
}
if arm == armGraphMixed {
return mixedArmInvalid(before)
}
return ""
}
// mixedArmInvalid holds the density arm to its own premise. Its whole claim is
// that one death carried every reachable state at once, and an arm that reached
// three of them measures a different scenario than the one it reports.
func mixedArmInvalid(before Observation) string {
var missing []string
for _, state := range wantedStates(armGraphMixed) {
if !holdsAll(agentgraph.Graph{Nodes: before.Graph.Nodes}, []agentgraph.NodeState{state}) {
missing = append(missing, string(state))
}
}
if len(missing) > 0 {
return "the fan-out never reached " + join(missing) + ", so the arm is not the mixed shape it reports"
}
if !carriesGrant(before, agentgraph.GrantRead) || !carriesGrant(before, agentgraph.GrantWrite) {
return "the fan-out carried only one kind of grant, so the envelope row measures one side of it"
}
return ""
}
func carriesGrant(o Observation, want agentgraph.Grant) bool {
for _, got := range o.Graph.Grants {
if got == string(want) {
return true
}
}
return false
}
// graphEdges renders the typed relations in a stable order. Sorting is not
// cosmetic: the fold keeps first-seen order, which two processes reach by
// different routes, and an unsorted comparison would report a reordering as a
// changed graph.
func graphEdges(o Observation) string {
if len(o.Graph.Edges) == 0 {
return ""
}
out := make([]string, 0, len(o.Graph.Edges))
for _, e := range o.Graph.Edges {
out = append(out, fmt.Sprintf("%s-%s->%s", e.From, e.Kind, e.To))
}
sort.Strings(out)
return join(out)
}
+602
View File
@@ -0,0 +1,602 @@
package main
import (
"context"
"fmt"
"os"
"sort"
"strings"
"sync/atomic"
"time"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/session/control"
)
// The interactive host-initiated arms, named for the one surface they measure: a
// person typing /<skill> into a live session. The headless counterpart is not
// measured here — it is caller-owned and its cancellation need not match, and
// reading one for the other is how a real number gets the wrong subject.
const (
armHostCompleted = "interactive-host-skill-completed"
armHostQueued = "interactive-host-skill-queued-crash"
armHostRunning = "interactive-host-skill-running-crash"
armHostCancel = "interactive-host-skill-cancel"
// The headless counterpart: the controller's own synchronous entry point,
// whose caller owns both the context and the cancellation. It is measured
// apart from the four above and compared to them, never read for them.
armHeadlessCompleted = "headless-host-skill-completed"
armHeadlessRunningCancel = "headless-host-skill-running-cancel"
armHeadlessQueuedCancel = "headless-host-skill-queued-cancel"
)
func headlessHostArm(name string) bool {
switch name {
case armHeadlessCompleted, armHeadlessRunningCancel, armHeadlessQueuedCancel:
return true
}
return false
}
// callerStopArm names the arms a caller stops through its own context. The
// controller never admitted these turns, so its own cancel holds nothing for
// them — reading one surface with the other's stop is a mistake this probe has
// already made once.
func callerStopArm(name string) bool {
return name == armHeadlessRunningCancel || name == armHeadlessQueuedCancel
}
func hostExecArm(name string) bool {
switch name {
case armHostCompleted, armHostQueued, armHostRunning, armHostCancel:
return true
}
return headlessHostArm(name)
}
// queuedHostArm names the arms whose ceiling is filled before the invocation.
func queuedHostArm(name string) bool {
return name == armHostQueued || name == armHeadlessQueuedCancel
}
// The four identities one host-started run can be known by. They are printed
// apart because this is the first surface where they may not coincide: for a
// model-delegated run the execution and the parent call are one string, and
// that has only ever been true because a tool call produced every durable
// execution there was.
const (
probeHostDispatch = "probe:host-dispatch-id"
probeHostChildRef = "probe:host-child-ref"
probeHostParentID = "probe:host-parent-lineage"
probeHostExecution = "probe:host-execution-id"
// probeHostStanding is what the call itself was doing at the reading:
// admitted or held back, its child in the provider or not.
probeHostStanding = "probe:host-standing"
// probeHostAuthority is the snapshot at the same instant: what a reader
// starting from the authority would hold, on either surface.
probeHostAuthority = "probe:host-authority"
)
// hostSlashInput is what a person types. The task carries the child sentinel so
// the scripted provider knows how this arm's child behaves.
func hostSlashInput(arm string) string {
return "/" + probeSkillName + " " + hostChildSentinel(arm) + " host-started work"
}
// runHostExecConstruct drives one interactive slash invocation and dies where
// this arm is about. The queued arm fills the ceiling first with a background
// fleet, which is dispatched through Run rather than Submit: Run is never
// admitted through the turn gate, so the slash the arm measures can still be.
func runHostExecConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
if queuedHostArm(arm) {
go func() { _ = ctrl.Run(ctx, fanOutTurn(turn+1, fleetHolderSentinel)) }()
if err := waitForCeiling(prov); err != nil {
return err
}
}
// Each surface is entered and stopped by the owner it has: the controller's
// gate for a turn it admitted, and the caller's own context for a
// synchronous call it never did.
callerCtx, stopCaller := context.WithCancel(ctx)
defer stopCaller()
var answered atomic.Bool
if headlessHostArm(arm) {
go func() {
_, _ = ctrl.RunSubagentProfile(callerCtx, probeSkillName, hostHeadlessTask(arm), false)
answered.Store(true)
}()
} else {
go ctrl.Submit(hostSlashInput(arm))
}
if err := waitForHostExec(ctrl, prov, arm, &answered); err != nil {
return err
}
stop := ""
switch {
case arm == armHostCancel:
stop = stopHostExec(ctrl, prov)
case callerStopArm(arm):
stop = stopHostCaller(ctrl, prov, arm, stopCaller)
}
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
recordHostIdentities(&obs, ctrl, sink, prov, arm)
if stop != "" {
obs.Progress[probeStopReached] = []string{stop}
}
if err := writeObservation(root, obs); err != nil {
return err
}
os.Exit(0)
return nil
}
// recordHostIdentities writes down all four names this run could be known by,
// at one instant. A single "identity: none" would hide which of them exists and
// which does not, and that difference is the whole question.
func recordHostIdentities(obs *Observation, ctrl *control.Controller, sink *graphSink, prov *scripted, arm string) {
obs.Progress[probeChildEntered] = []string{fmt.Sprint(prov.fleets.held(childEntered(hostChildSentinel(arm))))}
obs.Progress[probeHostDispatch] = []string{orNone(join(hostDispatchIDs(sink)))}
obs.Progress[probeHostChildRef] = []string{orNone(join(hostChildRefs(*obs)))}
obs.Progress[probeHostParentID] = []string{orNone(join(hostParentLineage(*obs)))}
obs.Progress[probeHostExecution] = []string{orNone(join(hostExecutions(*obs)))}
obs.Progress[probeHostStanding] = []string{hostStanding(ctrl, prov, arm)}
// The authority, read at the same instant. It is what a reader starts from
// on either surface, so it is where the two are compared — the delta stream
// exists only where a sink does, and a synchronous caller has none.
obs.Progress[probeHostAuthority] = []string{hostAuthorityNodes(ctrl)}
}
// hostHeadlessTask is what the controller's own entry point is asked to do.
func hostHeadlessTask(arm string) string {
return hostChildSentinel(arm) + " host-started work"
}
func hostAuthorityNodes(ctrl *control.Controller) string {
var out []string
for _, n := range ctrl.ExecutionGraph().Graph.Nodes {
if ofHostExecution(n.ID) {
out = append(out, n.ID+":"+orDash(string(n.State)))
}
}
sort.Strings(out)
return orNone(join(out))
}
// stopHostCaller stops work through the only owner a synchronous call has: the
// context its caller passed in. The controller's gate never admitted this turn,
// so asking it to stop one would be reading a surface with another's stop.
func stopHostCaller(ctrl *control.Controller, prov *scripted, arm string, stop func()) string {
stopped := time.Now()
stop()
child := hostChildSentinel(arm)
deadline := time.Now().Add(20 * time.Second)
for time.Now().Before(deadline) && !prov.fleets.held(childLeft(child)) {
time.Sleep(50 * time.Millisecond)
}
live := "gone"
if prov.fleets.held(childEntered(child)) && !prov.fleets.held(childLeft(child)) {
live = "still running"
}
return fmt.Sprintf("turn=%s work=%s ctx=%t after=%s", turnStanding(ctrl, arm), live,
prov.fleets.held(childCtxDone(child)), time.Since(stopped).Round(time.Second))
}
func hostChildSentinel(arm string) string {
if arm == armHostCompleted || arm == armHeadlessCompleted {
return childDone
}
return childHang
}
// hostDispatchIDs are the synthetic event ids the host minted so the child's
// activity could nest under something on screen. They are UI identity, not a
// call the model made, which is why no persisted lineage may carry one.
func hostDispatchIDs(sink *graphSink) []string {
var out []string
for id := range sink.toolIDs() {
if strings.HasPrefix(id, "slash-skill-") {
out = append(out, id)
}
}
sort.Strings(out)
return out
}
// hostChildRefs are the store's own artifact ids. They exist only once a child
// transcript has been prepared, which is why they cannot serve as the identity
// of work the scheduler is still holding back.
func hostChildRefs(o Observation) []string {
var out []string
for _, f := range o.Children.Facts {
if f.Kind == "skill" {
out = append(out, f.Ref+"="+f.Status)
}
}
sort.Strings(out)
return out
}
func hostParentLineage(o Observation) []string {
var out []string
for _, f := range o.Children.Facts {
if f.Kind == "skill" {
out = append(out, f.Ref+"→"+orDash(f.ParentToolCallID))
}
}
sort.Strings(out)
return out
}
func hostExecutions(o Observation) []string {
var out []string
for _, e := range o.Executions {
if ofHostExecution(e.ID) || strings.Contains(e.Name, probeSkillName) {
out = append(out, e.ID)
}
}
sort.Strings(out)
return out
}
// ofHostExecution matches both names this work can go by: the execution the host
// owns, and the synthetic event id it mints for the screen. Reading only the
// second is how an arm reports absence when what changed is the identity.
func ofHostExecution(id string) bool {
return strings.HasPrefix(id, "host-") || strings.HasPrefix(id, "slash-skill-")
}
func hostStanding(ctrl *control.Controller, prov *scripted, arm string) string {
return fmt.Sprintf("turn=%v ceiling=%t child=%t", ctrl.Running(),
prov.fleets.held(childEntered(childHold)),
prov.fleets.held(childEntered(hostChildSentinel(arm))))
}
func waitForCeiling(prov *scripted) error {
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
if prov.fleets.held(childEntered(childHold)) {
return nil
}
time.Sleep(50 * time.Millisecond)
}
return errUnexpected("a fan-out holding the ceiling", "none reached the provider")
}
// waitForHostExec blocks until the invocation stands where this arm dies. The
// queued arm asks a negative — the child never arrived while the ceiling was
// full — so it settles and asks again rather than reading a race.
func waitForHostExec(ctrl *control.Controller, prov *scripted, arm string, answered *atomic.Bool) error {
child := hostChildSentinel(arm)
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
switch {
case queuedHostArm(arm):
// In flight, read through whichever owner this surface has: a turn
// the controller admitted answers its gate, and a synchronous call
// it never admitted answers only by returning.
if hostInFlight(ctrl, arm, answered) && !prov.fleets.held(childEntered(child)) {
time.Sleep(2 * time.Second)
if prov.fleets.held(childEntered(child)) {
return errUnexpected("a host invocation the scheduler was still holding back", "its child ran")
}
return nil
}
case arm == armHostCompleted:
if prov.fleets.held(childLeft(child)) && !ctrl.Running() {
return nil
}
case arm == armHeadlessCompleted:
if prov.fleets.held(childLeft(child)) {
return nil
}
default:
if prov.fleets.held(childEntered(child)) {
return nil
}
}
time.Sleep(50 * time.Millisecond)
}
return errUnexpected("a host invocation "+hostWanted(arm), hostStanding(ctrl, prov, arm))
}
// hostInFlight reports that the invocation has not answered yet, asked of the
// owner the surface actually has.
func hostInFlight(ctrl *control.Controller, arm string, answered *atomic.Bool) bool {
if headlessHostArm(arm) {
return !answered.Load()
}
return ctrl.Running()
}
func hostWanted(arm string) string {
switch {
case queuedHostArm(arm):
return "held back by a full ceiling"
case arm == armHostCompleted || arm == armHeadlessCompleted:
return "that finished"
}
return "with its child inside the provider"
}
// stopHostExec stops the work the way a person does: the turn's own cancel. A
// slash invocation is admitted through the same gate a typed message is, so
// that gate is the owner of its stop.
func stopHostExec(ctrl *control.Controller, prov *scripted) string {
stopped := time.Now()
ctrl.Cancel()
deadline := time.Now().Add(20 * time.Second)
for time.Now().Before(deadline) && ctrl.Running() {
time.Sleep(50 * time.Millisecond)
}
live := "gone"
if prov.fleets.held(childEntered(childHang)) && !prov.fleets.held(childLeft(childHang)) {
live = "still running"
}
return fmt.Sprintf("turn=%s work=%s ctx=%t after=%s", turnStanding(ctrl, armHostCancel), live,
prov.fleets.held(childCtxDone(childHang)), time.Since(stopped).Round(time.Second))
}
// The verdicts this arm needs. They are separate words because the failure
// modes are separate facts, and one label over all of them would say less than
// any of them does.
const (
// verdictIdentityGap is work the host admitted that no identity outliving
// the process names. It is not about a lost record: none of the candidate
// identities can carry one.
verdictIdentityGap = "IDENTITY-GAP"
// verdictResultDurable is an invocation whose child record survives while
// the execution that produced it does not. A reader can see that a child of
// this name was cut; nothing says which invocation it belonged to.
verdictResultDurable = "record-only"
)
func hostExecRows(arm string, before, after Observation) []row {
return []row{
hostAdmissionRow(arm, before),
hostIdentityRow(before),
hostGraphRow(before),
hostJournalRow(before, after),
hostStoreRow(before, after),
hostLineageOracleRow(before, after),
hostPreStartIdentityRow(arm, before, after),
hostCancelRow(arm, before),
hostRestartRow(arm, before, after),
}
}
func hostAdmissionRow(arm string, before Observation) row {
entered := firstProgress(before, probeChildEntered) == "true"
answer, verdict := "admitted: the child reached the provider", verdictHolds
switch {
case queuedHostArm(arm) && !entered:
answer = "refused: the ceiling was full and the child never arrived"
case queuedHostArm(arm):
answer, verdict = "not held back: the child ran with the ceiling full", verdictViolated
case !entered:
answer, verdict = "the child never arrived", verdictNotMeasured
}
return row{
Semantic: "does it ask the session scheduler", Authority: skillAuthority,
Artifact: "none observed", Reconstruction: "in-process only",
Before: answer + " [" + firstProgress(before, probeHostStanding) + "]",
After: "—", Verdict: verdict,
}
}
// hostIdentityRow prints all four names at once. This is the row the arm exists
// for: for a model-delegated run the execution and the parent call are the same
// string, and that has only ever held because a tool call produced every durable
// execution there was.
func hostIdentityRow(before Observation) row {
return row{
Semantic: "the four names this run could be known by", Authority: crossAuthority,
Artifact: "the event stream, subagents/<ref>.meta.json, <stem>.execution.jsonl",
Reconstruction: "read apart, because they may not coincide here",
Before: fmt.Sprintf("dispatch=%s child=%s lineage=%s execution=%s",
firstProgress(before, probeHostDispatch), firstProgress(before, probeHostChildRef),
firstProgress(before, probeHostParentID), firstProgress(before, probeHostExecution)),
After: "—", Verdict: verdictHolds,
}
}
func hostGraphRow(before Observation) row {
var drawn []string
for _, n := range before.Graph.Nodes {
if ofHostExecution(n.ID) {
drawn = append(drawn, n.ID+":"+string(n.State))
}
}
sort.Strings(drawn)
// The stream and the authority are reported together because only one of
// them is surface-independent: a synchronous caller has no sink to draw on,
// and a reader on either surface starts from the snapshot regardless.
authority := firstProgress(before, probeHostAuthority)
return row{
Semantic: "the graph, drawn and reconstructable",
Authority: liveAuthority + " and " + authorityName,
Artifact: "<stem>.execution.jsonl", Reconstruction: "the stream where a sink exists; the authority always",
Before: fmt.Sprintf("stream=%s authority=%s", orNone(join(drawn)), authority),
After: "—", Verdict: presenceVerdict(authority != "none"),
}
}
func hostJournalRow(before, after Observation) row {
return presenceRow("does the journal record it", journalAuthority, "<stem>.execution.jsonl",
"ExecutionHistory over the whole session", hostExecutions(before), hostExecutions(after))
}
func hostStoreRow(before, after Observation) row {
return presenceRow("what the store holds for it", storeAuthority, "subagents/<ref>.meta.json",
"ListSubagentsByParent over the transcript stem", hostChildRefs(before), hostChildRefs(after))
}
// hostLineageOracleRow is the contract Step 8 respected and this must not
// weaken. A synthetic host event id is UI identity; persisting one as a parent
// would say the model issued a call it never made.
func hostLineageOracleRow(before, after Observation) row {
leaked := hostSyntheticLeaks(before)
leaked = append(leaked, hostSyntheticLeaks(after)...)
empty := hostLineageEmpty(before) && hostLineageEmpty(after)
answer, verdict := "no persisted lineage names a synthetic host id, and every parent is empty", verdictHolds
switch {
case len(leaked) > 0:
answer, verdict = "a synthetic host id was persisted as lineage: "+join(leaked), verdictViolated
case !empty:
answer, verdict = "a host-started run was filed under a parent call: "+
join(hostParentLineage(after)), verdictViolated
case len(hostChildRefs(before))+len(hostChildRefs(after)) == 0:
verdict = verdictNotMeasured
answer = "no child record exists to carry a lineage"
}
return row{
Semantic: "synthetic host id persisted as lineage", Authority: storeAuthority,
Artifact: "subagents/<ref>.meta.json", Reconstruction: "every persisted parent, read for host event ids",
Before: answer, After: "—", Verdict: verdict,
}
}
func hostSyntheticLeaks(o Observation) []string {
var out []string
for _, f := range o.Children.Facts {
if strings.HasPrefix(f.ParentToolCallID, "slash-skill-") {
out = append(out, f.Ref+"→"+f.ParentToolCallID)
}
}
return out
}
func hostLineageEmpty(o Observation) bool {
for _, f := range o.Children.Facts {
if f.Kind == "skill" && strings.TrimSpace(f.ParentToolCallID) != "" {
return false
}
}
return true
}
// hostPreStartIdentityRow is what decides where an identity would have to be
// created. Work the scheduler is holding back has no child artifact yet, so a
// store ref cannot name it; if nothing else survives the process either, no
// candidate identity exists at the moment the work first became real.
func hostPreStartIdentityRow(arm string, before, after Observation) row {
survives := len(hostExecutions(after)) > 0
answer := "nothing that outlives the process names this work"
verdict := verdictIdentityGap
if survives {
answer, verdict = "an execution the journal opened: "+join(hostExecutions(after)), verdictHolds
}
if !queuedHostArm(arm) && firstProgress(before, probeChildEntered) != "true" {
verdict = verdictNotMeasured
}
return row{
Semantic: "an identity that exists before the child does", Authority: crossAuthority,
Artifact: "<stem>.execution.jsonl",
Reconstruction: "the candidates, read against what the work needs",
Before: hostAtDeath(arm), After: answer, Verdict: verdict,
}
}
func hostAtDeath(arm string) string {
switch {
case queuedHostArm(arm):
return "admitted-domain work, held back, no child artifact yet"
case arm == armHostCompleted || arm == armHeadlessCompleted:
return "ran and returned"
}
return "executing inside the provider"
}
func hostCancelRow(arm string, before Observation) row {
if arm != armHostCancel && !callerStopArm(arm) {
return row{
Semantic: "what a stop reaches", Authority: "the turn's own cancel",
Artifact: "none observed", Reconstruction: "in-process only",
Before: "not stopped in this arm", After: "—", Verdict: verdictNotMeasured,
}
}
return row{
Semantic: "what a stop reaches", Authority: "the turn's own cancel",
Artifact: "subagents/<ref>.meta.json",
Reconstruction: "the live stop and the record it left, kept apart",
Before: firstProgress(before, probeStopReached) + " store=" + orNone(join(hostChildRefs(before))),
After: "—", Verdict: verdictHolds,
}
}
// hostRestartRow classifies what the next process inherits. An answer that
// survived while its execution did not is not a loss to the user, so it is not
// called one; work that was executing and left nothing is.
func hostRestartRow(arm string, before, after Observation) row {
exec, children := hostExecutions(after), hostChildRefs(after)
rebuilt := hostRebuilt(after)
answer := fmt.Sprintf("execution=%s store=%s rebuilt=%s",
orNone(join(exec)), orNone(join(children)), orNone(join(rebuilt)))
return row{
Semantic: "what a restart inherits", Authority: rebuildAuthority,
Artifact: "<stem>.execution.jsonl + subagents/<ref>.meta.json",
Reconstruction: "execgraph.Rebuild, then the store beside it",
Before: hostAtDeath(arm), After: answer,
Verdict: hostRestartVerdict(arm, before, exec, children),
}
}
// hostRestartVerdict is driven by what a next process can actually hold, not by
// what the arm expected. A child record with nothing to place it is not silence:
// a reader can still see that a child of this name was cut. What it cannot do is
// say which invocation that was, and the identity row answers that separately —
// folding the two would report a surviving record as a total loss.
func hostRestartVerdict(arm string, before Observation, exec, children []string) string {
switch {
case len(exec) > 0:
return verdictHolds
case len(children) > 0:
return verdictResultDurable
case firstProgress(before, probeChildEntered) == "true" || queuedHostArm(arm):
return graphLostSilent
default:
return verdictNotMeasured
}
}
// hostRebuilt is what a restart draws for this invocation. The whole rebuilt
// graph is wider than the arm — the queued arm fills its ceiling with a fan-out,
// and a fan-out records everything — so reading its nodes as the invocation's is
// how an arm reports a gap as closed by evidence belonging to something else.
func hostRebuilt(o Observation) []string {
var out []string
for _, n := range rebuiltGraph(o).Graph.Nodes {
if n.Kind == agentgraph.KindGroup {
continue
}
if ofHostExecution(n.ID) || strings.Contains(n.Label, probeSkillName) {
out = append(out, n.ID+":"+string(n.State))
}
}
sort.Strings(out)
return out
}
// hostExecArmInvalid reports the premise this arm could not establish.
func hostExecArmInvalid(arm string, before Observation) string {
entered := firstProgress(before, probeChildEntered) == "true"
switch {
case queuedHostArm(arm) && entered:
return "the invocation ran while the ceiling was meant to be full"
case !queuedHostArm(arm) && !entered:
return "the invocation's child never reached the provider"
case headlessHostArm(arm) && len(hostDispatchIDs2(before)) > 0:
return "a synchronous call minted a dispatch id, so this was not the headless surface"
case !headlessHostArm(arm) && len(hostDispatchIDs2(before)) == 0:
return "the host minted no dispatch id, so this was not the slash surface"
}
return ""
}
// hostDispatchIDs2 reads the dispatch id back off the observation, where the
// construct phase recorded what only a live sink could see.
func hostDispatchIDs2(o Observation) []string {
if got := firstProgress(o, probeHostDispatch); got != "none" && got != "unrecorded" {
return strings.Split(got, ", ")
}
return nil
}
+163
View File
@@ -0,0 +1,163 @@
package main
import (
"context"
"fmt"
"os"
"strings"
"time"
"reasonix/internal/session/control"
)
// Two trees, read at once. A stop aimed at a turn must close the turn's own
// tree and leave the job's alone; an explicit job stop does the reverse, and
// reading only one of them is how a probe calls a violated handoff correct. The
// manager's own root needs no separate question: a job still running with its
// child's context open is the root answering directly.
const (
// armLineageForeground is the positive control: it proves the reading can
// see an ordinary cancellation arrive.
armLineageForeground = "lineage-foreground-cancel"
// armLineageBackground is the one under suspicion.
armLineageBackground = "lineage-background-cancel"
// armLineageJobKill is the other positive control: without it, a reading
// that could never observe a job's own cancellation would make the arm
// above pass for the wrong reason.
armLineageJobKill = "lineage-job-kill"
)
func lineageArm(name string) bool {
switch name {
case armLineageForeground, armLineageBackground, armLineageJobKill:
return true
}
return false
}
// runLineageConstruct drives one delegation, stops it the way this arm's owner
// stops it, and reads both trees afterwards. It records six facts, kept apart
// on purpose: a context that closed and a call that returned are different
// events, and only their combination says whether a cancellation propagated or
// something else ended the work.
func runLineageConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
ctrl.Send(fanOutTurn(turn+1, taskSentinel))
if err := waitFor(60*time.Second, func() bool { return prov.fleets.held(childEntered(childHang)) }); err != nil {
return errUnexpected("a delegated child inside the provider", "none arrived")
}
before := lineageReading(ctrl, prov, "before")
switch arm {
case armLineageJobKill:
jobs := ctrl.Jobs()
if len(jobs) == 0 {
return errUnexpected("a background job to stop", "none running")
}
for _, job := range jobs {
ctrl.CancelJob(job.ID)
}
default:
ctrl.Cancel()
}
// Long enough for a cancellation that was going to arrive to have arrived.
time.Sleep(3 * time.Second)
after := lineageReading(ctrl, prov, "after")
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
obs.Progress[probeLineage] = []string{before, after}
if err := writeObservation(root, obs); err != nil {
return err
}
os.Exit(0)
return nil
}
const probeLineage = "probe:lineage"
// lineageReading is the state of both trees at one instant, in the terms the
// rows judge: the turn's gate, the job's own liveness, the child's context, the
// child's call, and the durable record the orchestration keeps.
func lineageReading(ctrl *control.Controller, prov *scripted, at string) string {
return fmt.Sprintf("%s turn=%s job=%s childCtx=%s childCall=%s record=%s",
at, turnGateStanding(ctrl), jobStanding(ctrl),
yesNo(prov.fleets.held(childCtxDone(childHang)), "cancelled", "open"),
yesNo(prov.fleets.held(childLeft(childHang)), "returned", "inside"),
loneRecordStanding(ctrl))
}
func turnGateStanding(ctrl *control.Controller) string {
if ctrl.Running() {
return "active"
}
return "ended"
}
func jobStanding(ctrl *control.Controller) string {
if len(ctrl.Jobs()) > 0 {
return "running"
}
return "gone"
}
func loneRecordStanding(ctrl *control.Controller) string {
for _, e := range control.ExecutionHistory(ctrl.SessionPath()) {
if e.ID == parallelLoneNodeID {
return lifecycleOf(e)[1:]
}
}
return "absent"
}
func yesNo(ok bool, yes, no string) string {
if ok {
return yes
}
return no
}
func waitFor(within time.Duration, ready func() bool) error {
deadline := time.Now().Add(within)
for time.Now().Before(deadline) {
if ready() {
return nil
}
time.Sleep(50 * time.Millisecond)
}
return fmt.Errorf("timed out")
}
// lineageRows read the two trees against the ownership each arm is about. The
// three sets of expectations are deliberately different: a fix that closes
// everything on a stop satisfies the foreground arm and violates both others.
func lineageRows(arm string, before Observation) []row {
at := reading(before.Progress[probeLineage], 1)
turnEnded := strings.Contains(at, "turn=ended")
jobLive := strings.Contains(at, "job=running")
childCancelled := strings.Contains(at, "childCtx=cancelled")
recordOpen := !strings.Contains(at, "settled") && !strings.Contains(at, "record=absent")
rows := []row{lineageRow("the turn the stop was aimed at", at, "ends", turnEnded)}
if arm == armLineageForeground {
return append(rows,
lineageRow("the child's own context", at, "closes with the turn that owned it", childCancelled),
lineageRow("the delegation record", at, "is let go of", !recordOpen))
}
if arm == armLineageJobKill {
return append(rows[:0],
lineageRow("the job the stop was aimed at", at, "ends", !jobLive),
lineageRow("the child's own context", at, "closes with the job that owned it", childCancelled),
lineageRow("the delegation record", at, "is let go of", !recordOpen))
}
return append(rows,
lineageRow("the job the turn no longer owns", at, "keeps running", jobLive),
lineageRow("the child's own context", at, "is not closed by another owner's stop", !childCancelled),
lineageRow("the delegation record", at, "stays open while its child runs", recordOpen))
}
func lineageRow(semantic, got, want string, ok bool) row {
return row{
Semantic: semantic, Authority: "both trees, read at the same instant",
Artifact: "none: in-process state", Reconstruction: want,
Before: want, After: orNone(got), Verdict: held(ok),
}
}
+356
View File
@@ -0,0 +1,356 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"slices"
"time"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/contract/event"
"reasonix/internal/contract/provider"
"reasonix/internal/session/control"
)
// The four single-delegation arms. A lone task is not another executor: it
// reaches the same RunProfileSpec a fleet item does and asks the same scheduler
// for a slot. What these ask is whether that shared chain is wired to the
// durable authority the fan-out arms established — and where, exactly, it stops
// being wired, which is a different question from "does it have a graph".
const (
// armTaskCompleted lets the delegation finish and the turn close. Nothing is
// interrupted, so a loss here is not about interruption: it says the run's
// provenance was never durable at all.
armTaskCompleted = "task-completed"
// armTaskRunning kills the process with the child mid-execution, the same
// death the fan-out arms take.
armTaskRunning = "task-running"
// armTaskFgQueued is the live-truth arm: the ceiling is full when the
// delegation asks, so the scheduler is holding it back. What the parent and
// the graph say while that is true is the whole question — a picture that
// says running while nothing has started is wrong before any restart.
armTaskFgQueued = "task-foreground-queued"
// armTaskBgQueued is the one the foreground arms cannot stand in for: a
// background run hands its execution to a job and asks for a slot inside it,
// so the parent tool call has already returned when the refusal happens.
armTaskBgQueued = "task-background-queued"
// armTaskBgRunning is the same handoff with the slot granted, which is where
// ownership of the terminal moves from the caller to the job.
armTaskBgRunning = "task-background-running"
// The cancellation arms. They ask the one question the five before them
// cannot: who owns the ending when work is stopped, on each side of the
// boundary that decides whether it ever ran. A child the scheduler never
// admitted produced nothing, so the store owes it no terminal at all.
armTaskFgQueuedCancel = "task-foreground-queued-cancel"
armTaskBgQueuedCancel = "task-background-queued-cancel"
armTaskFgRunningCancel = "task-foreground-running-cancel"
armTaskBgRunningCancel = "task-background-running-cancel"
)
func cancelTaskArm(name string) bool {
switch name {
case armTaskFgQueuedCancel, armTaskBgQueuedCancel, armTaskFgRunningCancel, armTaskBgRunningCancel:
return true
}
return false
}
// beforeStartArm names the arms cancelled while the scheduler still held the
// delegation, which is the half of the boundary the store must stay out of.
func beforeStartArm(name string) bool {
return name == armTaskFgQueuedCancel || name == armTaskBgQueuedCancel
}
func loneTaskArm(name string) bool {
switch name {
case armTaskCompleted, armTaskRunning, armTaskFgQueued, armTaskBgQueued, armTaskBgRunning:
return true
}
return cancelTaskArm(name)
}
// queuedTaskArm names the arms that fill the ceiling before delegating, so the
// scheduler has to refuse the delegation admission.
func queuedTaskArm(name string) bool {
return name == armTaskFgQueued || name == armTaskBgQueued || beforeStartArm(name)
}
// childEntered names the record a child leaves when it reaches the provider.
// The queued arms ask a negative — that one never did — which nothing else in
// the probe can answer.
func childEntered(sentinel string) string { return "child-entered:" + sentinel }
// childLeft names the record a child leaves when it returns. Entered without
// left is a child still inside the provider — the state a cancellation that
// never arrived leaves behind.
func childLeft(sentinel string) string { return "child-left:" + sentinel }
// childCtxDone names the record a child leaves when its own context closes,
// which is a different fact from having returned: a run whose caller stopped
// waiting returns to nobody with its context still open.
func childCtxDone(sentinel string) string { return "child-ctxdone:" + sentinel }
// probeChildEntered is where that answer rides into the observation. It is not
// a status the kernel emits: the key names the probe so no row reads it as one.
const probeChildEntered = "probe:delegated-child-entered"
func backgroundTaskArm(name string) bool {
switch name {
case armTaskBgQueued, armTaskBgRunning, armTaskBgQueuedCancel, armTaskBgRunningCancel,
armLineageBackground, armLineageJobKill, armCancelBackground, armFleetActiveStore:
return true
}
return false
}
// The sentinels one turn carries. The queued arm names the holder as well: its
// ceiling has to be occupied before the delegation asks for a slot, or the
// refusal it is built to record never happens.
const (
taskSentinel = "PROBE-TASK"
taskCallID = "probe_task"
taskHoldsFirst = fleetHolderSentinel + " " + taskSentinel
)
// loneTaskCall is the delegation itself, dispatched through the tool a model
// calls. Nothing here reaches past the tool's own arguments: the probe measures
// the production path, so it must enter it the way a model does.
func loneTaskCall(arm string) []provider.Chunk {
prompt, description := childDone+" lone task", "completes"
switch {
case arm == armFleetActiveStore:
// This delegation is the other occupant of the ceiling its arm shares
// with a fan-out, and its own child is the only thing that can say the
// slot is taken. Holding is how it hangs and says so at once.
prompt, description = childHold+" lone task", "holds a slot to the death"
case arm != armTaskCompleted:
prompt, description = childHang+" lone task", "still executing at death"
}
args, _ := json.Marshal(map[string]any{
"prompt": prompt, "description": description,
"run_in_background": backgroundTaskArm(arm),
})
return []provider.Chunk{{
Type: provider.ChunkToolCall,
ToolCall: &provider.ToolCall{ID: taskCallID, Name: "task", Arguments: string(args)},
}}
}
// probeStepIDs are the host's open steps, in the order they have to be signed.
// The arm whose turn ends has to close them: a delivery is refused while the
// list is open, and a todo_write that marks an item completed on its own is
// refused too — the sign-off is the only door.
func probeStepIDs() []string { return []string{"probe_step_01", "probe_step_02", "probe_step_03"} }
// signStep signs off one step per round, which is what the host accepts: the
// sign-off promotes the next item, so two in one round sign against a list that
// no longer stands.
func signStep(id string) []provider.Chunk {
args, _ := json.Marshal(map[string]any{
"step_id": id,
"result": "the probe drove this step to its end",
"evidence": []map[string]any{
{"kind": "manual", "summary": "the probe established this state directly"},
},
})
return []provider.Chunk{{
Type: provider.ChunkToolCall,
ToolCall: &provider.ToolCall{ID: "probe_sign_" + id, Name: "complete_step", Arguments: string(args)},
}}
}
// closesItsTurn names the arms whose turn has to reach its end. A turn that
// called a tool cannot deliver while the host's step list is open, so these
// sign it off one step per round — the sign-off promotes the next item, and two
// in one round would sign against a list that no longer stands.
func closesItsTurn(arm string) bool {
return arm == armTaskCompleted || arm == armSkillCompleted || arm == armReadOnlySkillCompleted
}
// loneTaskSentinels is what this arm's one prompt carries.
func loneTaskSentinels(arm string) string {
if queuedTaskArm(arm) {
return taskHoldsFirst
}
return taskSentinel
}
// runLoneTaskConstruct drives one delegation and dies at the moment this arm is
// about. The completed arm is the exception that proves the rest: it lets the
// turn close, so what it loses was never about dying inside one.
func runLoneTaskConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
// One turn, whatever the arm needs in it: a session admits no second one. The
// queued arm's ceiling is filled by a background fleet named in the same
// prompt, whose own child says when the slot is taken.
prompt := fanOutTurn(turn+1, loneTaskSentinels(arm))
switch {
case arm == armTaskCompleted:
if err := ctrl.Run(ctx, prompt); err != nil {
return fmt.Errorf("delegating turn: %w", err)
}
case guardedArm(arm):
// A stop belongs to whoever admitted the turn, so an arm that is going
// to be stopped is admitted the way a person's input is.
ctrl.Send(prompt)
if err := waitForLoneTask(sink, prov, arm); err != nil {
return err
}
default:
go func() { _ = ctrl.Run(ctx, prompt) }()
if err := waitForLoneTask(sink, prov, arm); err != nil {
return err
}
}
stop := ""
if cancelTaskArm(arm) {
stop = stopDelegation(ctrl, prov, arm)
}
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
if stop != "" {
obs.Progress[probeStopReached] = []string{stop}
}
if queuedTaskArm(arm) {
// Whether the delegation's own child ever ran is the fact the live rows
// are read against, and only the scripted provider knows it.
obs.Progress[probeChildEntered] = []string{fmt.Sprint(prov.fleets.held(childEntered(childHang)))}
}
if err := writeObservation(root, obs); err != nil {
return err
}
os.Exit(0)
return nil
}
// stopDelegation stops the work the way a person does — the turn's own cancel
// for a foreground run, the job list and a kill for a backgrounded one — and
// waits for the orchestration to let go of it. What each layer holds then is
// this arm's whole question, so nothing is read until the record closes.
func stopDelegation(ctrl *control.Controller, prov *scripted, arm string) string {
stopped := time.Now()
if backgroundTaskArm(arm) {
for _, job := range ctrl.Jobs() {
ctrl.CancelJob(job.ID)
}
} else {
ctrl.Cancel()
}
// Wait for the record to close, but do not demand it: whether a stop reaches
// the work it was aimed at is what this arm measures, and an arm that failed
// when it did not would report the finding as its own premise breaking.
deadline := time.Now().Add(20 * time.Second)
for time.Now().Before(deadline) && !settledLone(ctrl) {
time.Sleep(50 * time.Millisecond)
}
live := "gone"
if prov.fleets.held(childEntered(childHang)) && !prov.fleets.held(childLeft(childHang)) {
live = "still running"
}
return fmt.Sprintf("turn=%s work=%s after=%s", turnStanding(ctrl, arm), live,
time.Since(stopped).Round(time.Second))
}
func settledLone(ctrl *control.Controller) bool {
for _, e := range control.ExecutionHistory(ctrl.SessionPath()) {
if e.ID == parallelLoneNodeID && !e.SettledAt.IsZero() {
return true
}
}
return false
}
// waitForLoneTask blocks until the delegation stands where this arm dies. The
// foreground arms read the live graph; the background ones cannot — a handed-off
// run publishes no node — so they read the progress the parent was told.
func waitForLoneTask(sink *graphSink, prov *scripted, arm string) error {
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
if loneTaskReady(sink, prov, arm) {
// A refusal is a state nothing announces, so the arm holds still and
// asks again: a child that was merely slow to start would have
// arrived by now, and the death would be measuring a race.
if beforeStartArm(arm) || arm == armTaskFgQueued {
time.Sleep(2 * time.Second)
return refusalHolding(prov, arm)
}
return nil
}
time.Sleep(50 * time.Millisecond)
}
graph, _ := sink.snapshot()
return errUnexpected("a lone delegation "+loneTaskWanted(arm),
nodeStates(graph)+" progress="+fmt.Sprint(sink.phaseSeries()))
}
// refusalHolding reports the arm's premise once more, after the settle: the
// delegation's child must still not have run.
func refusalHolding(prov *scripted, arm string) error {
if prov.fleets.held(childEntered(childHang)) {
return errUnexpected("a delegation the scheduler was still holding back", "its child ran")
}
return nil
}
func loneTaskReady(sink *graphSink, prov *scripted, arm string) bool {
graph, _ := sink.snapshot()
switch arm {
case armTaskFgQueued, armTaskFgQueuedCancel:
// The node is drawn — the delegation reached the graph — and its child
// has not started, which with the ceiling full is the refusal.
return loneNodeDrawn(graph) && !prov.fleets.held(childEntered(childHang))
case armTaskRunning, armTaskFgRunningCancel, armTaskBgRunningCancel:
return holdsAll(graph, []agentgraph.NodeState{agentgraph.StateRunning})
case armTaskBgQueued, armTaskBgQueuedCancel:
phases := sink.phaseSeries()[taskCallID]
return len(phases) > 0 && phases[len(phases)-1] == string(subagentQueued)
case armTaskBgRunning:
return slices.Contains(sink.phaseSeries()[taskCallID], subagentRunning)
}
return false
}
// loneNodeDrawn reports whether the delegation's own node is on the graph.
func loneNodeDrawn(g agentgraph.Graph) bool {
for _, n := range g.Nodes {
if n.Kind == agentgraph.KindWorker && n.ID == parallelLoneNodeID {
return true
}
}
return false
}
// parallelLoneNodeID is what a lone delegation's node is called: the parent
// call, and the first slot of the fan-out numbering it shares.
const parallelLoneNodeID = taskCallID + "/sub-1"
func loneTaskWanted(arm string) string {
switch arm {
case armTaskRunning, armTaskFgRunningCancel, armTaskBgRunningCancel:
return "running"
case armTaskFgQueued, armTaskFgQueuedCancel, armTaskBgQueuedCancel:
return "drawn while the scheduler holds it back"
case armTaskBgQueued:
return "queued for a slot and not started"
}
return "running in its job"
}
// The two phases the probe waits on. They are the parent-facing statuses a
// background delegation emits, and for a handed-off run they are the only live
// evidence there is.
const (
subagentQueued = "queued"
subagentRunning = "running"
)
// progressPhase reads one subagent status event. The tool id is the parent call
// the delegation was made from, which is also what the graph node hangs under
// and what the sub-agent store records as its parent — the join this arm exists
// to check rather than assume.
func progressPhase(e event.Event) (id, phase string, ok bool) {
if e.Kind != event.ToolProgress || e.Tool.Name != event.SubagentProgressStatusName {
return "", "", false
}
return e.Tool.ID, e.Tool.Output, e.Tool.ID != ""
}
+536
View File
@@ -0,0 +1,536 @@
package main
import (
"fmt"
"sort"
"strings"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/state/execgraph"
"reasonix/internal/state/execjournal"
)
// What a lone delegation leaves behind, layer by layer. The columns are the
// ordinary ones: what the dying process held, and what the next one can still
// say. Nothing here is a fix — the arm exists to find which transition stops
// being recorded, because "single task has no graph" was the roadmap's guess
// and the live chain has since been unified.
func loneTaskRows(arm string, before, after Observation) []row {
return []row{
liveDelegationRow(before),
parentStatusRow(before),
durableOpeningRow(before, after),
durableQueueRow(before, after),
durableStartRow(before, after),
durableSettleRow(before, after),
delegationStoreRow(before, after),
rebuiltDelegationRow(before, after),
rebuiltDelegationIdentityRow(after),
delegationInterruptionRow(before, after),
identityJoinRow(before),
heldBackRow(arm, before),
stoppedOwnershipRow(arm, before),
startedTerminalRow(before, after),
fanOutControlRow(before),
}
}
// ofLoneTask reports whether an id belongs to the delegation under measurement.
// The queued arm fills its ceiling with a fan-out, and a fan-out records
// everything — so without this the rows would read that record as the lone
// task's own and report a gap as closed.
func ofLoneTask(id string) bool {
return id == taskCallID || strings.HasPrefix(id, taskCallID+"/")
}
const (
crossAuthority = "the journal and the store, read against each other"
liveAuthority = "event sink (GraphDelta), in-process"
journalAuthority = "execjournal, written by the orchestration"
storeAuthority = "sub-agent store"
rebuildAuthority = "execgraph.Rebuild over journal + store"
)
// liveDelegationRow is what the dying process actually drew. A backgrounded run
// draws nothing at all — openDelegationGraph returns early for it — so this row
// separates "the graph was lost" from "there was never a node".
func liveDelegationRow(before Observation) row {
var drawn []string
for _, n := range before.Graph.Nodes {
if n.Kind == agentgraph.KindWorker && ofLoneTask(n.ID) {
drawn = append(drawn, n.ID+":"+string(n.State))
}
}
sort.Strings(drawn)
return row{
Semantic: "the delegation the live graph drew", Authority: liveAuthority,
Artifact: "none observed", Reconstruction: "in-process only",
Before: orNone(join(drawn)), After: "—", Verdict: liveVerdict(len(drawn) > 0),
}
}
func liveVerdict(drawn bool) string {
if drawn {
return verdictStable
}
return verdictNotMeasured
}
// parentStatusRow is the other live surface, and the only one a handed-off run
// has: the statuses the parent tool call was told.
func parentStatusRow(before Observation) row {
var told []string
for id, phases := range before.Progress {
if ofLoneTask(id) {
told = append(told, id+":"+strings.Join(phases, "→"))
}
}
sort.Strings(told)
return row{
Semantic: "what the parent call was told", Authority: "subagent progress (ToolProgress)",
Artifact: "none observed", Reconstruction: "in-process only",
Before: orNone(join(told)), After: "—", Verdict: liveVerdict(len(told) > 0),
}
}
func durableOpeningRow(before, after Observation) row {
return valueRow("durable opening", journalAuthority, "<stem>.execution.jsonl",
"read back by ExecutionHistory", executionIDs(before), executionIDs(after), true)
}
func durableQueueRow(before, after Observation) row {
return valueRow("durable queue refusal", journalAuthority, "<stem>.execution.jsonl",
"the scheduler's first refusal, with its cause",
queuedCauses(before), queuedCauses(after), true)
}
func durableStartRow(before, after Observation) row {
return valueRow("durable slot grant", journalAuthority, "<stem>.execution.jsonl",
"written before the child can act", startedIDs(before), startedIDs(after), true)
}
func durableSettleRow(before, after Observation) row {
return valueRow("durable settlement", journalAuthority, "<stem>.execution.jsonl",
"the orchestration letting go of the execution",
settledIDs(before), settledIDs(after), true)
}
// delegationStoreRow is the layer that already survives for a fan-out item. It
// is the reason this arm cannot be read off the graph alone: a store record with
// no opening leaves the run's grant, worker and scheduling unaccounted for.
func delegationStoreRow(before, after Observation) row {
return valueRow("child the store kept", storeAuthority, "subagents/<ref>.meta.json",
"ListSubagentsByParent over the transcript stem",
childFacts(before), childFacts(after), true)
}
func rebuiltDelegationRow(before, after Observation) row {
rebuilt := rebuiltGraph(after)
var nodes []string
for _, n := range rebuilt.Graph.Nodes {
if n.Kind != agentgraph.KindGroup && ofLoneTask(n.ID) {
nodes = append(nodes, n.ID+":"+string(n.State))
}
}
sort.Strings(nodes)
return valueRow("the node a restart rebuilds", rebuildAuthority, "<stem>.execution.jsonl",
"execgraph.Rebuild", loneDelegationAtDeath(before), join(nodes), false)
}
// rebuiltDelegationIdentityRow is what a reader sees on the rebuilt node beyond
// its state. Grant and worker are opening facts: the store cannot answer them,
// so an arm that only compared states would call a hollow node exact.
func rebuiltDelegationIdentityRow(after Observation) row {
var got []string
for _, n := range rebuiltGraph(after).Graph.Nodes {
if n.Kind == agentgraph.KindGroup || !ofLoneTask(n.ID) {
continue
}
got = append(got, fmt.Sprintf("%s grant=%s model=%s effort=%s wait=%s",
n.ID, orDash(string(n.Grant)), orDash(n.Model), orDash(n.Effort), orDash(string(n.Wait))))
}
sort.Strings(got)
return row{
Semantic: "grant, worker and wait on the rebuilt node", Authority: rebuildAuthority,
Artifact: "<stem>.execution.jsonl", Reconstruction: "opening facts, which no store record carries",
Before: "declared at the opening", After: orNone(join(got)),
Verdict: rebuiltIdentityVerdict(got),
}
}
func rebuiltIdentityVerdict(got []string) string {
if len(got) == 0 {
return verdictLost
}
return verdictExact
}
func delegationInterruptionRow(before, after Observation) row {
var got []string
for _, i := range rebuiltGraph(after).Interrupted {
if ofLoneTask(i.Execution) {
got = append(got, i.Execution+":"+interruptionKind(i))
}
}
sort.Strings(got)
return valueRow("the interruption a restart names", rebuildAuthority, "<stem>.execution.jsonl",
"an execution with no owner is named, never redrawn as running",
loneDelegationAtDeath(before), join(got), false)
}
// loneDelegationAtDeath is the delegation as the dying process last knew it,
// from whichever surface knew: a foreground run has a node, a handed-off one
// has only the store's record, and an arm whose rows read the wrong surface
// would report a loss the other one never had.
func loneDelegationAtDeath(o Observation) string {
if live := liveWorkers(o); live != "" {
return live
}
return childFacts(o)
}
func interruptionKind(i execgraph.Interruption) string {
if i.Started {
return execjournal.InterruptedDuringExecution
}
return execjournal.InterruptedBeforeStart
}
// identityJoinRow is the question that decides whether Step 8 needs an identity
// at all: the live node, the call it hung under, the parent the store recorded,
// and the execution the journal opened. Four spellings, or one identity.
func identityJoinRow(before Observation) row {
live, call, stored, opened := identitiesAtDeath(before)
verdict := verdictViolated
switch {
case stored != "" && (stored == opened || stored == call || strings.HasPrefix(live, stored)):
verdict = verdictHolds
case stored == "" && !startedAtDeath(before):
// Nothing executed, so the store owes no record and there is no join to
// make. A missing artifact here is the arm's own premise, not a defect.
verdict = verdictNotMeasured
}
return row{
Semantic: "one identity or four", Authority: "the graph, the call, the store and the journal",
Artifact: "subagents/<ref>.meta.json, <stem>.execution.jsonl",
Reconstruction: "the store's parent call is what a rebuild joins on",
Before: fmt.Sprintf("node=%s call=%s", orDash(live), orDash(call)),
After: fmt.Sprintf("store parent=%s journal=%s", orDash(stored), orDash(opened)),
Verdict: verdict,
}
}
// identitiesAtDeath reads the four spellings out of the dying process's own
// observation, so a join that only holds after a restart cannot be claimed.
func identitiesAtDeath(o Observation) (live, call, stored, opened string) {
for _, n := range o.Graph.Nodes {
if n.Kind == agentgraph.KindWorker && ofLoneTask(n.ID) {
live, call = n.ID, n.ParentID
break
}
}
for id := range o.Progress {
if call == "" && ofLoneTask(id) {
call = id
}
}
for _, f := range o.Children.Facts {
if ofLoneTask(f.ParentToolCallID) {
stored = f.ParentToolCallID
break
}
}
for _, e := range o.Executions {
if ofLoneTask(e.ID) {
opened = e.ID
break
}
}
return live, call, stored, opened
}
// startedAtDeath reports whether the delegation was ever granted a slot.
func startedAtDeath(o Observation) bool {
for _, e := range o.Executions {
if ofLoneTask(e.ID) && e.Started() {
return true
}
}
return false
}
func executionIDs(o Observation) string {
var out []string
for _, e := range o.Executions {
if ofLoneTask(e.ID) {
out = append(out, e.ID)
}
}
sort.Strings(out)
return join(out)
}
func queuedCauses(o Observation) string {
var out []string
for _, e := range o.Executions {
if e.Queued() && ofLoneTask(e.ID) {
out = append(out, e.ID+":"+orDash(e.Cause))
}
}
sort.Strings(out)
return join(out)
}
func startedIDs(o Observation) string {
var out []string
for _, e := range o.Executions {
if e.Started() && ofLoneTask(e.ID) {
out = append(out, e.ID)
}
}
sort.Strings(out)
return join(out)
}
func settledIDs(o Observation) string {
var out []string
for _, e := range o.Executions {
if !e.SettledAt.IsZero() && ofLoneTask(e.ID) {
out = append(out, e.ID)
}
}
sort.Strings(out)
return join(out)
}
func childFacts(o Observation) string {
var out []string
for _, f := range o.Children.Facts {
if ofLoneTask(f.ParentToolCallID) {
out = append(out, f.ParentToolCallID+":"+f.Status)
}
}
sort.Strings(out)
return join(out)
}
func liveWorkers(o Observation) string {
var out []string
for _, n := range o.Graph.Nodes {
if n.Kind == agentgraph.KindWorker && ofLoneTask(n.ID) {
out = append(out, n.ID+":"+string(n.State))
}
}
sort.Strings(out)
return join(out)
}
func orDash(s string) string {
if strings.TrimSpace(s) == "" {
return "-"
}
return s
}
// heldBackRow is the live-truth row: with the ceiling full and the delegation's
// own child never reaching the provider, the scheduler is holding it back — and
// what the picture says while that is true is a claim about now, which no
// restart can correct. Running is the answer that is wrong.
func heldBackRow(arm string, before Observation) row {
if !queuedTaskArm(arm) {
return row{
Semantic: "what the picture says while the scheduler holds it back", Authority: liveAuthority,
Artifact: "none observed", Reconstruction: "this arm never fills the ceiling",
Before: "—", After: "—", Verdict: verdictNotMeasured,
}
}
drawn := orNone(liveWorkers(before))
told := orNone(loneStatus(before))
verdict := verdictHolds
if strings.Contains(drawn, ":running") || told == subagentRunning {
verdict = verdictViolated
}
return row{
Semantic: "what the picture says while the scheduler holds it back", Authority: liveAuthority,
Artifact: "none observed",
Reconstruction: "queued with its cause, or at least not started",
Before: "held back: ceiling full, child never entered",
After: "graph=" + drawn + " parent=" + told, Verdict: verdict,
}
}
func loneStatus(o Observation) string {
for id, phases := range o.Progress {
if ofLoneTask(id) && len(phases) > 0 {
return phases[len(phases)-1]
}
}
return ""
}
// stoppedOwnershipRow is the invariant a stop has to leave behind: once the
// turn is over, nothing it owned may still be open with a live owner. A record
// left open is what a restart reads as an interruption, so a turn that merely
// stopped waiting looks, afterwards, exactly like a process that died.
func stoppedOwnershipRow(arm string, before Observation) row {
if !cancelTaskArm(arm) {
return row{
Semantic: "work the stopped turn still owns", Authority: crossAuthority,
Artifact: "<stem>.execution.jsonl", Reconstruction: "this arm stops nothing",
Before: "—", After: "—", Verdict: verdictNotMeasured,
}
}
reading := ""
if seen := before.Progress[probeStopReached]; len(seen) > 0 {
reading = seen[0]
}
entry, _ := loneEntry(before)
open := entry.SettledAt.IsZero()
turnOver := strings.Contains(reading, "turn=ended")
// An open record implies a live owner: nothing else closes one. The child's
// liveness rides in the reading as evidence, not as the test — a delegation
// still waiting for a slot has no child and is just as much still owned.
return row{
Semantic: "work the stopped turn still owns", Authority: crossAuthority,
Artifact: "<stem>.execution.jsonl",
Reconstruction: "a turn that has ended owns nothing still open",
Before: "the turn owned it when the stop was issued",
After: orNone(reading) + " record=" + entryStanding(entry), Verdict: held(!(turnOver && open)),
}
}
func entryStanding(e execjournal.Entry) string {
if e.SettledAt.IsZero() {
return "still open"
}
return "settled"
}
// startedTerminalRow refuses the combination neither layer may produce: a child
// record for work the journal proves was never admitted. A state comparison
// shows that only as "cancelled came back failed"; this names the cause, which
// is a store asked to speak for an execution that never happened.
func startedTerminalRow(before, after Observation) row {
if startedAtDeath(before) {
return row{
Semantic: "a child record for work that never started", Authority: crossAuthority,
Artifact: "subagents/<ref>.meta.json, <stem>.execution.jsonl",
Reconstruction: "this delegation was granted a slot, so the store owes a record",
Before: "—", After: "—", Verdict: verdictNotMeasured,
}
}
kept := orNone(join(compact(childFacts(before), childFacts(after))))
return row{
Semantic: "a child record for work that never started", Authority: crossAuthority,
Artifact: "subagents/<ref>.meta.json, <stem>.execution.jsonl",
Reconstruction: "no slot was ever granted, so nothing ran and nothing owes a terminal",
Before: "none", After: kept, Verdict: held(kept == "none"),
}
}
func compact(parts ...string) []string {
var out []string
for _, p := range parts {
if strings.TrimSpace(p) != "" {
out = append(out, p)
}
}
return out
}
// fanOutControlRow is the positive control the queued arm carries with it: the
// fleet that fills its ceiling runs through the same scheduler in the same
// session, and records everything. What this arm reports missing is missing at
// the entry point, not in the machinery.
func fanOutControlRow(before Observation) row {
var recorded []string
for _, e := range before.Executions {
if !ofLoneTask(e.ID) {
recorded = append(recorded, e.ID+lifecycleOf(e))
}
}
sort.Strings(recorded)
verdict := verdictNotMeasured
if len(recorded) > 0 {
verdict = verdictPersisted
}
return row{
Semantic: "a fan-out in the same session, same scheduler", Authority: journalAuthority,
Artifact: "<stem>.execution.jsonl", Reconstruction: "control: the machinery this arm reports missing",
Before: orNone(join(recorded)), After: "—", Verdict: verdict,
}
}
func lifecycleOf(e execjournal.Entry) string {
marks := ""
if e.Queued() {
marks += " queued=" + orDash(e.Cause)
}
if e.Started() {
marks += " started"
}
if !e.SettledAt.IsZero() {
marks += " settled"
}
return marks
}
// loneTaskArmInvalid guards each arm's premise: the state it is built to die in
// has to have been reached, or its rows describe a different death.
func loneTaskArmInvalid(arm string, before Observation) string {
phases := strings.Join(before.Progress[taskCallID], "→")
switch arm {
case armTaskCompleted:
if !strings.Contains(phases, "completed") && len(before.Children.Facts) == 0 {
return "the delegation never completed, so this arm measures a different death"
}
case armTaskRunning:
if !holdsState(before, string(agentgraph.StateRunning)) {
return "nothing was executing when the process died"
}
case armTaskFgQueued:
if entered := before.Progress[probeChildEntered]; len(entered) == 0 || entered[0] != "false" {
return "the delegation's child ran, so the scheduler was not holding it back"
}
if !strings.Contains(fanOutControlRow(before).Before, "queued=slots") {
return "no fan-out filled the ceiling, so nothing refused the delegation"
}
case armTaskBgQueued:
if !strings.HasSuffix(phases, subagentQueued) {
return "the delegation was not left queued for a slot: " + orNone(phases)
}
case armTaskBgRunning:
if !strings.Contains(phases, subagentRunning) {
return "the delegation never entered its job: " + orNone(phases)
}
}
if cancelTaskArm(arm) {
return stoppedArmInvalid(arm, before)
}
return ""
}
// stoppedArmInvalid holds a cancellation arm to the side of the boundary it was
// built for. An arm that stopped the work somewhere else measures a different
// ownership question and must say so rather than reporting the answer.
func stoppedArmInvalid(arm string, before Observation) string {
entry, ok := loneEntry(before)
switch {
case !ok:
return "the delegation was never recorded, so there is no ownership to read"
case beforeStartArm(arm) && entry.Started():
return "the delegation was granted a slot before the stop, so this is not a pre-start cancellation"
case !beforeStartArm(arm) && !entry.Started():
return "the delegation never held a slot, so this is not a cancellation of work that ran"
}
return ""
}
func loneEntry(o Observation) (execjournal.Entry, bool) {
for _, e := range o.Executions {
if ofLoneTask(e.ID) {
return e, true
}
}
return execjournal.Entry{}, false
}
+48
View File
@@ -0,0 +1,48 @@
// Process-boundary resurrection probe. It answers one question and modifies
// nothing: after the process that built a session exits, what can a new process
// still prove about that session's runtime semantics without asking a model?
package main
import (
"flag"
"fmt"
"os"
)
func main() {
phase := flag.String("phase", "", "internal: construct|successor|resume, spawned by the orchestrator")
root := flag.String("root", "", "internal: this arm's root directory")
armName := flag.String("arm", "", "internal: this arm's name")
work := flag.String("work", "", "directory to hold arm roots; default is a temp dir")
only := flag.String("only", "", "run a single arm by name")
jsonOut := flag.String("json", "", "also write the machine-readable matrix here")
keep := flag.Bool("keep", false, "keep arm roots on disk after the run")
sabotage := flag.String("sabotage", "", "run under a deliberate defect the arm must fail on (ui-graph-mixed: publish|trajectory)")
flag.Parse()
if *sabotage != "" {
// Carried to the children through the environment they already inherit,
// so no phase has to learn about a flag it does not read.
if err := os.Setenv(envProbeSabotage, *sabotage); err != nil {
fmt.Fprintln(os.Stderr, "runtime-resume:", err)
os.Exit(1)
}
}
var err error
switch *phase {
case "construct":
err = runConstruct(*root, *armName)
case "successor":
err = runSuccessor(*root, *armName)
case "resume":
err = runResume(*root, *armName)
case "":
err = orchestrate(*work, *only, *jsonOut, *keep)
default:
err = fmt.Errorf("unknown phase %q", *phase)
}
if err != nil {
fmt.Fprintln(os.Stderr, "runtime-resume:", err)
os.Exit(1)
}
}
+584
View File
@@ -0,0 +1,584 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"reasonix/internal/state/sessionstore"
"regexp"
"sort"
"strings"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/contract/provider"
"reasonix/internal/runtime/delegation"
"reasonix/internal/safety/evidence"
"reasonix/internal/session/control"
"reasonix/internal/state/execjournal"
"reasonix/internal/state/store"
)
// Observation is everything one phase can read about a session without asking a
// model. Both phases capture the same shape so the matrix is a comparison and
// not two differently-shaped reports.
type Observation struct {
Phase string `json:"phase"`
Arm string `json:"arm"`
SessionPath string `json:"session_path"`
SystemHash string `json:"system_hash"`
SystemBytes int `json:"system_bytes"`
// BootSystemHash is what this process's assembly composed, read before any
// resume; SystemHash is the session's own leading row. Different questions.
BootSystemHash string `json:"boot_system_hash"`
BootSystemBytes int `json:"boot_system_bytes"`
Transcript TranscriptObs `json:"transcript"`
Goal GoalObs `json:"goal"`
Todos []evidence.TodoItem `json:"todos"`
Decisions []control.Decision `json:"decisions"`
Context ContextObs `json:"context"`
Sidecar SidecarObs `json:"sidecar"`
View ViewObs `json:"view"`
TodoNotes TodoNoteObs `json:"todo_notes"`
Deferred DeferredObs `json:"deferred"`
Obligation ObligationObs `json:"obligation"`
// Interrupted is what the host derives from a barrier it found open and did
// not open itself: evidence the wait happened, not a question to answer.
Interrupted []control.InterruptedAdjudication `json:"interrupted"`
// ModelSeesInterruption counts the interruption blocks the request carries,
// which is a different surface from the derived fact behind them.
ModelSeesInterruption int `json:"model_sees_interruption"`
// Journal is how each barrier ended. A superseded edge is the durable
// evidence that a turn received the interruption: nothing else writes one.
Journal []control.AdjudicationEntry `json:"journal,omitempty"`
Graph GraphObs `json:"graph"`
// Children is the durable side of the same fan-out the graph draws. The
// graph has no read surface after a restart, so without this a lost graph
// and a lost delegation would be one indistinguishable row.
Children ChildrenObs `json:"children"`
// Listing is the same children through the read list_subagents performs.
// It filters by workspace and walks lineage, so it can answer differently
// from the one above — and it is the answer a model asking would be shown.
Listing ListingObs `json:"listing"`
// FanOutTurn is whether the dispatching turn is in this phase's transcript
// at all. A turn is appended when it ends, so a process that dies inside one
// leaves no record of the request, and a lost graph is the smaller half.
FanOutTurn bool `json:"fan_out_turn"`
// Executions is the durable record of delegations this turn opened, and
// InterruptedExecutions the host's judgement over it: open, with no owner
// here. Written before the work starts, so it outlives an unfinished turn.
Executions []execjournal.Entry `json:"executions,omitempty"`
InterruptedExecutions []execjournal.Entry `json:"interrupted_executions,omitempty"`
// ModelSeesInterruptedExecution counts the blocks the next request carries
// about them, which is a different surface from the fact behind them.
ModelSeesInterruptedExecution int `json:"model_sees_interrupted_execution"`
// WaitCauses counts the refusals the graph carries, and WaitSeries keeps
// each node's causes in publication order. The fold keeps only the latest,
// which cannot say whether a cause was reported once or replaced.
WaitCauses map[string]int `json:"wait_causes,omitempty"`
WaitSeries map[string][]string `json:"wait_series,omitempty"`
// Progress is every parent-facing subagent status, per call. A background
// delegation draws no node, so this is where its live evidence is.
Progress map[string][]string `json:"progress,omitempty"`
// SettledWorkers and QueuedWorkers are the transition arm's evidence that
// capacity was actually freed while a refusal still stood.
SettledWorkers int `json:"settled_workers"`
QueuedWorkers []string `json:"queued_workers,omitempty"`
Artifacts []ArtifactObs `json:"artifacts"`
// UI is what a frontend attached to this process saw, in order. Only the one
// arm that runs a client fills it; a value comparison cannot answer what it
// asks, which is through which door each fact arrived.
UI *UIObs `json:"ui,omitempty"`
}
type TranscriptObs struct {
Messages int `json:"messages"`
Digest string `json:"digest"`
}
type GoalObs struct {
Goal string `json:"goal"`
Status string `json:"status"`
TurnsUsed int `json:"turns_used"`
}
// ContextObs is the live judgement: CheckpointState is "none" whenever the
// projection did not survive validation, which is what the compaction arms are
// actually asking about.
type ContextObs struct {
ProjectionVersion uint64 `json:"projection_version"`
CheckpointState string `json:"checkpoint_state"`
CanonicalTokens int `json:"canonical_tokens"`
ProjectedTokens int `json:"projected_tokens"`
Blocked bool `json:"blocked"`
}
// SidecarObs is the stored claim, read off disk rather than through the agent,
// so a projection the host refuses to use is still visible as present.
type SidecarObs struct {
Present bool `json:"present"`
Messages int `json:"messages"`
CoveredCount int `json:"covered_count"`
CoveredPrefixHash string `json:"covered_prefix_hash"`
ProjectionVersion uint64 `json:"projection_version"`
TranscriptVersion uint64 `json:"transcript_version"`
Generation uint64 `json:"generation"`
Err string `json:"err,omitempty"`
}
// ViewObs is the model-visible context, spliced the way ContextProjection
// declares it: the stored body followed by canonical[CoveredCount:]. Markers
// are what a fold can be held to — the scripted digest carries none, so a
// marker that is gone was dropped, not summarised.
type ViewObs struct {
Messages int `json:"messages"`
// Markers are the user turns still shown, Echoes the assistant replies. A
// fold takes the reply first, so the two answer different questions.
Markers []string `json:"markers"`
Echoes []string `json:"echoes"`
BodyMarkers []string `json:"body_markers"`
BodyEchoes []string `json:"body_echoes"`
SplicedFromTail int `json:"spliced_from_tail"`
}
// TodoNoteObs locates the host's step-identity note inside the model-visible
// view. BodyLen is where the frozen body ends, so an index below it says the
// note was frozen ahead of the live tail rather than riding it.
type TodoNoteObs struct {
Count int `json:"count"`
Indexes []int `json:"indexes"`
IDs []string `json:"ids"`
BodyLen int `json:"body_len"`
ViewLen int `json:"view_len"`
HostIDs []string `json:"host_ids"`
// ReadableInHistory says the conversation still shows every current id, so
// no note is owed. Without it a count of zero cannot be told apart from a
// lost identity, and the gate would demand a note nobody needs.
ReadableInHistory bool `json:"readable_in_history"`
}
// todoNoteMarker is the opening of todoIdentityNote. The probe reads the view
// the way a model would, so it matches the text rather than a type.
const todoNoteMarker = "Host task state."
var stepIDPattern = regexp.MustCompile(`probe_step_\d+`)
func todoNoteObs(view, history []provider.Message, todos []evidence.TodoItem) TodoNoteObs {
out := TodoNoteObs{BodyLen: len(history), ViewLen: len(view)}
for _, t := range todos {
if t.StepID != "" {
out.HostIDs = append(out.HostIDs, t.StepID)
}
}
out.ReadableInHistory = len(out.HostIDs) > 0
for _, id := range out.HostIDs {
if !mentionsID(history, id) {
out.ReadableInHistory = false
}
}
for i, m := range view {
if !strings.Contains(m.Content, todoNoteMarker) {
continue
}
out.Count++
out.Indexes = append(out.Indexes, i)
out.IDs = append(out.IDs, stepIDPattern.FindAllString(m.Content, -1)...)
}
return out
}
// DeferredObs is the effect a pending decision is holding back. Whether the
// decision itself survives is one question; whether what it blocked stayed
// blocked is the one that cannot be answered by a design choice.
type DeferredObs struct {
MarkerPath string `json:"marker_path"`
Executed bool `json:"executed"`
}
func deferredObs(workspace string) DeferredObs {
path := filepath.Join(workspace, deferredEffect)
_, err := os.Stat(path)
return DeferredObs{MarkerPath: deferredEffect, Executed: err == nil}
}
// ObligationObs is what the transcript itself still says about a turn that
// never finished: a tool call with no result is a claim the host made and did
// not settle, and an interruption record is the host saying so outright. Either
// tells a lost decision apart from a silently dropped one.
type ObligationObs struct {
UnansweredCalls []string `json:"unanswered_calls"`
InterruptionMarked int `json:"interruption_marked"`
}
func obligationObs(msgs []provider.Message) ObligationObs {
answered := map[string]bool{}
var out ObligationObs
for _, m := range msgs {
if m.ToolCallID != "" {
answered[m.ToolCallID] = true
}
if m.InterruptedTurn != nil {
out.InterruptionMarked++
}
}
for _, m := range msgs {
for _, call := range m.ToolCalls {
if call.ID != "" && call.ID != provider.LocalOnlyToolID && !answered[call.ID] {
out.UnansweredCalls = append(out.UnansweredCalls, call.Name+":"+call.ID)
}
}
}
return out
}
type GraphObs struct {
Deltas int `json:"deltas"`
Nodes []agentgraph.Node `json:"nodes,omitempty"`
Edges []agentgraph.Edge `json:"edges,omitempty"`
Grants map[string]string `json:"grants,omitempty"`
Waits map[string]string `json:"waits,omitempty"`
}
// ChildFact is one delegated run as the durable store records it, which is a
// different set of facts from the node the graph drew for it. Whatever the
// graph carried and this does not is a semantic no reconstruction can recover.
type ChildFact struct {
Ref string `json:"ref"`
Status string `json:"status"`
Kind string `json:"kind,omitempty"`
Name string `json:"name,omitempty"`
// ExecutionID is which execution the record belongs to; ParentToolCallID is
// the provider-visible call it descends from. They are separate answers, and
// a host-started run has only the first.
ExecutionID string `json:"executionId,omitempty"`
ParentToolCallID string `json:"parentToolCallId,omitempty"`
Model string `json:"model,omitempty"`
Effort string `json:"effort,omitempty"`
}
// ChildrenObs is every child the store still owns for this parent. It is the
// only durable execution evidence a restart has been shown to read, so a graph
// classification that ignores it would report a loss the host can in fact
// still speak to.
type ChildrenObs struct {
Facts []ChildFact `json:"facts,omitempty"`
Err string `json:"err,omitempty"`
}
// childrenObs reads the store the way a restart does: by parent session id,
// which is the transcript's stem. A session with no path owns nothing.
func childrenObs(root armRoot, sessionPath string) ChildrenObs {
if strings.TrimSpace(sessionPath) == "" {
return ChildrenObs{}
}
parent := strings.TrimSuffix(filepath.Base(sessionPath), ".jsonl")
artifacts, err := sessionstore.ListSubagentsByParent(root.Sessions, parent)
if err != nil {
return ChildrenObs{Err: err.Error()}
}
out := ChildrenObs{}
for _, a := range artifacts {
out.Facts = append(out.Facts, ChildFact{
Ref: a.Ref, Status: string(a.Meta.Status), Kind: a.Meta.Kind, Name: a.Meta.Name,
ExecutionID: a.Meta.ExecutionID,
ParentToolCallID: a.Meta.ParentToolCallID, Model: a.Meta.Model, Effort: a.Meta.Effort,
})
}
sort.Slice(out.Facts, func(i, j int) bool { return out.Facts[i].Ref < out.Facts[j].Ref })
return out
}
// ListingRow is one row list_subagents would print. The tool call itself needs
// a registry this probe has no way to build, so the read behind it is what is
// exercised; the formatting adds no fact the row does not already carry.
type ListingRow struct {
Ref string `json:"ref"`
Status string `json:"status"`
From string `json:"from,omitempty"`
}
// ListingObs is what the recovery surface would show for this conversation.
type ListingObs struct {
Rows []ListingRow `json:"rows,omitempty"`
Err string `json:"err,omitempty"`
}
// listingObs reads the store the way the tool does: by the parent session the
// child recorded, filtered to this workspace. The workspace filter is why this
// cannot be folded into childrenObs — a root that does not match drops every
// row, and a reader must be able to see that happen rather than read it as an
// empty store.
func listingObs(root armRoot, sessionPath string) ListingObs {
if strings.TrimSpace(sessionPath) == "" {
return ListingObs{}
}
store := delegation.NewSubagentStore(filepath.Join(root.Sessions, "subagents"))
parent := strings.TrimSuffix(filepath.Base(sessionPath), ".jsonl")
artifacts, err := store.ListForParent(parent, root.Workspace)
if err != nil {
return ListingObs{Err: err.Error()}
}
out := ListingObs{}
for _, a := range artifacts {
out.Rows = append(out.Rows, ListingRow{
Ref: a.Ref, Status: string(a.Meta.Status), From: a.Meta.ParentToolCallID,
})
}
return out
}
type ArtifactObs struct {
Name string `json:"name"`
Bytes int64 `json:"bytes"`
}
func capture(phase, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, root armRoot) Observation {
path := ctrl.SessionPath()
history := ctrl.History()
system := systemText(history)
snap := ctrl.ContextMaintenanceSnapshot()
st, loaded, loadErr := sessionstore.LoadCompactionState(path)
view, viewMsgs := viewObs(st, loaded, history)
graph, deltas := sink.snapshot()
return Observation{
Phase: phase,
Arm: arm,
SessionPath: path,
SystemHash: shortSum(system),
SystemBytes: len(system),
BootSystemHash: shortSum(bootSystem),
BootSystemBytes: len(bootSystem),
Transcript: TranscriptObs{Messages: len(history), Digest: transcriptDigest(history)},
Goal: GoalObs{Goal: ctrl.Goal(), Status: ctrl.GoalStatus(), TurnsUsed: ctrl.GoalRuntime().TurnsUsed},
Todos: ctrl.Todos(),
Decisions: ctrl.Decisions(),
Context: ContextObs{
ProjectionVersion: snap.ProjectionVersion,
CheckpointState: snap.CheckpointState,
CanonicalTokens: snap.CanonicalTokens,
ProjectedTokens: snap.ProjectedTokens,
Blocked: snap.Blocked,
},
Sidecar: sidecarObs(st, loaded, loadErr),
View: view,
// The note is derived for a request and stored nowhere, so the contracts
// about it are read off the request rather than off the stored view.
TodoNotes: todoNoteObs(ctrl.ModelVisibleMessages(), viewMsgs, ctrl.Todos()),
Graph: graphObs(graph, deltas),
WaitCauses: waitCauseCounts(graph),
WaitSeries: sink.waitSeries(),
Progress: sink.phaseSeries(),
SettledWorkers: settledWorkers(graph),
QueuedWorkers: queuedWorkers(graph),
Children: childrenObs(root, path),
Listing: listingObs(root, path),
FanOutTurn: fanOutTurnRecorded(history),
Executions: control.ExecutionHistory(path),
InterruptedExecutions: ctrl.InterruptedExecutions(),
ModelSeesInterruptedExecution: countBlocks(
ctrl.ModelVisibleMessages(), "<interrupted-execution>"),
Deferred: deferredObs(root.Workspace),
Obligation: obligationObs(history),
Interrupted: ctrl.InterruptedAdjudications(),
Journal: control.AdjudicationHistory(path),
ModelSeesInterruption: countBlocks(ctrl.ModelVisibleMessages(), "<interrupted-adjudication>"),
Artifacts: readArtifacts(path),
}
}
// bootSystemText is what the freshly built assembly composed, read before a
// resume replaces the session under it.
func bootSystemText(ctrl *control.Controller) string { return systemText(ctrl.History()) }
func systemText(msgs []provider.Message) string {
var b strings.Builder
for _, m := range msgs {
if m.Role == provider.RoleSystem {
b.WriteString(m.Content)
b.WriteByte('\n')
}
}
return b.String()
}
// transcriptDigest covers only what reaches a provider, so a local-only marker
// appended by either process cannot read as a changed conversation.
func transcriptDigest(msgs []provider.Message) string {
h := sha256.New()
for _, m := range provider.ModelMessages(msgs) {
b, _ := json.Marshal(struct {
R, C, N string
}{string(m.Role), m.Content, m.Name})
h.Write(b)
h.Write([]byte{'\n'})
}
return hex.EncodeToString(h.Sum(nil))[:16]
}
func shortSum(s string) string {
sum := sha256.Sum256([]byte(s))
return hex.EncodeToString(sum[:8])
}
func sidecarObs(st sessionstore.CompactionState, ok bool, err error) SidecarObs {
out := SidecarObs{Present: ok}
if err != nil {
out.Err = err.Error()
return out
}
if !ok {
return out
}
out.Messages = len(st.Projection.Messages)
out.CoveredCount = st.Projection.CoveredCount
out.CoveredPrefixHash = st.Projection.CoveredPrefixHash
out.ProjectionVersion = st.Projection.ProjectionVersion
out.TranscriptVersion = st.TranscriptVersion
out.Generation = st.Generation
return out
}
// viewObs splices the model-visible context. With no usable projection the
// view is the canonical transcript, which is what the host would send.
func viewObs(st sessionstore.CompactionState, ok bool, canonical []provider.Message) (ViewObs, []provider.Message) {
var out ViewObs
if !ok || len(st.Projection.Messages) == 0 {
out.Messages = len(canonical)
out.Markers = markersIn(contents(canonical), markerPattern)
out.Echoes = markersIn(contents(canonical), echoPattern)
return out, canonical
}
body := st.Projection.Messages
out.BodyMarkers = markersIn(contents(body), markerPattern)
out.BodyEchoes = markersIn(contents(body), echoPattern)
view := append([]provider.Message(nil), body...)
if n := st.Projection.CoveredCount; n >= 0 && n < len(canonical) {
view = append(view, canonical[n:]...)
out.SplicedFromTail = len(canonical) - n
}
out.Messages = len(view)
out.Markers = markersIn(contents(view), markerPattern)
out.Echoes = markersIn(contents(view), echoPattern)
return out, view
}
// mentionsID reads an id the way the host's own visibility check does: text or
// tool arguments, anywhere in the messages the request carries.
func mentionsID(msgs []provider.Message, id string) bool {
for _, m := range msgs {
if strings.Contains(m.Content, id) {
return true
}
for _, call := range m.ToolCalls {
if strings.Contains(call.Arguments, id) {
return true
}
}
}
return false
}
func countBlocks(msgs []provider.Message, marker string) int {
n := 0
for _, m := range msgs {
if strings.Contains(m.Content, marker) {
n++
}
}
return n
}
func contents(msgs []provider.Message) []string {
out := make([]string, 0, len(msgs))
for _, m := range msgs {
out = append(out, m.Content)
}
return out
}
// waitCauseCounts is how many nodes the graph shows under each refusal. It
// counts nodes rather than reports: a node keeps its cause after it starts, so
// this answers what was refused, not what is refused now.
func waitCauseCounts(g agentgraph.Graph) map[string]int {
out := map[string]int{}
for _, n := range g.Nodes {
if n.Wait != "" {
out[string(n.Wait)]++
}
}
if len(out) == 0 {
return nil
}
return out
}
// queuedWorkers are the items still refused admission, in a stable order.
func queuedWorkers(g agentgraph.Graph) []string {
var out []string
for _, n := range g.Nodes {
if n.Kind != agentgraph.KindGroup && n.State == agentgraph.StateQueued {
out = append(out, n.ID)
}
}
sort.Strings(out)
return out
}
func graphObs(g agentgraph.Graph, deltas int) GraphObs {
out := GraphObs{Deltas: deltas}
if len(g.Nodes) == 0 {
return out
}
out.Nodes = g.Nodes
out.Edges = g.Edges
out.Grants = map[string]string{}
out.Waits = map[string]string{}
for _, n := range g.Nodes {
if n.Grant != "" {
out.Grants[n.ID] = string(n.Grant)
}
if n.Wait != "" {
out.Waits[n.ID] = string(n.Wait)
}
}
return out
}
// readArtifacts lists the session sidecars that exist, which is the evidence
// behind every "persisted" claim in the matrix.
func readArtifacts(sessionPath string) []ArtifactObs {
if sessionPath == "" {
return nil
}
paths := append([]string{sessionPath}, store.SessionSidecarFiles(sessionPath)...)
var out []ArtifactObs
for _, p := range paths {
if p == "" {
continue
}
info, err := os.Stat(p)
if err != nil {
continue
}
out = append(out, ArtifactObs{Name: filepath.Base(p), Bytes: info.Size()})
}
return out
}
func writeObservation(root armRoot, obs Observation) error {
b, err := json.MarshalIndent(obs, "", " ")
if err != nil {
return err
}
return os.WriteFile(filepath.Join(root.Dir, "obs-"+obs.Phase+".json"), b, 0o644)
}
func readObservation(root armRoot, phase string) (Observation, error) {
var obs Observation
b, err := os.ReadFile(filepath.Join(root.Dir, "obs-"+phase+".json"))
if err != nil {
return obs, err
}
return obs, json.Unmarshal(b, &obs)
}
+338
View File
@@ -0,0 +1,338 @@
package main
import (
"context"
"fmt"
"os"
"reasonix/internal/state/sessionstore"
"time"
"reasonix/internal/runtime/agent"
"reasonix/internal/session/control"
"reasonix/internal/state/checkpoint"
)
// probeTurns is enough conversation for a fold to have a region to work on:
// the system row and the first user turn are pinned, and the recent tail is
// kept verbatim, so a short session has nothing foldable between them.
const probeTurns = 8
// refoldTurns grows the transcript past the first fold by less than the fold
// keeps verbatim, so the second fold has to reach into the stored body.
const refoldTurns = 3
// runTurns drives count turns, each carrying its own marker, and returns the
// next turn number.
func runTurns(ctx context.Context, ctrl *control.Controller, from, count int) (int, error) {
for i := range count {
n := from + i + 1
prompt := fmt.Sprintf("%s Probe turn %d: record the task list and keep working.", marker(n), n)
if err := ctrl.Run(ctx, prompt); err != nil {
return n, fmt.Errorf("turn %d: %w", n, err)
}
}
return from + count, nil
}
// openDecisionAndDie leaves a question open and kills the process while the
// turn is still parked on it. It exits rather than returning: a clean shutdown
// would resolve or cancel the question, which is the one thing a process that
// dies mid-decision does not do.
func openDecisionAndDie(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink) error {
// The gate a person answers through, wired the way chat and desktop wire
// it. Without it the ask tool decides for itself and nothing ever blocks.
ctrl.EnableInteractiveApproval()
go func() { _ = ctrl.Run(ctx, marker(99)+" "+askSentinel+": ask before writing.") }()
if err := waitForAsk(ctrl); err != nil {
return err
}
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
if obs.Deferred.Executed {
return errUnexpected("a write still held back by the open question", obs.Deferred.MarkerPath)
}
if err := writeObservation(root, obs); err != nil {
return err
}
os.Exit(0)
return nil
}
// waitForAsk blocks until the host reports a question waiting on a person.
func waitForAsk(ctrl *control.Controller) error {
deadline := time.Now().Add(60 * time.Second)
for time.Now().Before(deadline) {
for _, d := range ctrl.Decisions() {
if d.Kind == control.DecisionAsk {
return nil
}
}
time.Sleep(50 * time.Millisecond)
}
return errUnexpected("a question waiting on a person", ctrl.Decisions())
}
// rewindTurn drives the rewind a person drives, recording the projection
// before it: the tail arm takes back the turn appended after the fold, the
// covered arm lands below the fold boundary and must lose the projection
// whatever coverage says.
func rewindTurn(root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink) error {
points := ctrl.Checkpoints()
if len(points) == 0 {
return errUnexpected("a checkpoint to rewind to", 0)
}
target := points[len(points)-1].Turn
if arm == armCoveredRewind {
var err error
if target, err = turnBelowFold(ctrl, points); err != nil {
return err
}
}
if err := writeObservation(root, capture(extraPhase(arm), arm, bootSystem, ctrl, sink, root)); err != nil {
return err
}
if err := ctrl.Rewind(target, control.RewindConversation); err != nil {
return fmt.Errorf("rewind: %w", err)
}
return nil
}
// turnBelowFold picks the checkpoint whose boundary sits just below the fold,
// read from the sidecar rather than assumed: a control aiming at a fixed turn
// stops removing folded history the moment coverage moves, and says nothing.
// A boundary at the first message is skipped — a session left holding only a
// system row is never persisted, so the restart would read the arm's work back.
func turnBelowFold(ctrl *control.Controller, points []checkpoint.Meta) (int, error) {
covered := 0
if st, ok, _ := sessionstore.LoadCompactionState(ctrl.SessionPath()); ok {
covered = st.Projection.CoveredCount
}
target := -1
for _, p := range points {
plan, err := ctrl.PrepareRewind(p.Turn, control.RewindConversation)
if err != nil || !plan.HasBoundary {
continue
}
if plan.BoundaryIndex > 1 && plan.BoundaryIndex < covered {
target = p.Turn
}
}
if target < 0 {
return 0, errUnexpected("a checkpoint below the fold boundary", covered)
}
return target, nil
}
// extraPhase names the in-process observation an arm records before its
// process exits, empty when it records none.
func extraPhase(arm string) string {
switch arm {
case armRefoldIntoBody:
return "prefold"
case armTailRewind, armCoveredRewind:
return "prerewind"
}
return ""
}
const probeGoal = "Prove which runtime semantics survive an OS process boundary."
// armConstruct hands the arm to the construct path it needs, reporting whether
// one took over. Each of them records its own observation and exits inside the
// turn, so a caller that sees taken has nothing left to do but report the error.
func armConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) (bool, error) {
switch {
case arm == armOpenDecision || successorArm(arm):
return true, openDecisionAndDie(ctx, root, arm, bootSystem, ctrl, sink)
case deriveArm(arm):
return true, runDeriveConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
case terminalArm(arm):
return true, runTerminalConstruct(ctx, root, arm, bootSystem, ctrl, sink, turn)
case schedulerWaitArm(arm):
return true, runSchedulerWaitConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
case lineageArm(arm):
return true, runLineageConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
case cancelRoutingArm(arm):
return true, runCancelRoutingConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
case loneTaskArm(arm):
return true, runLoneTaskConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
case activeStoreArm(arm):
return true, runActiveStoreConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
case skillArm(arm):
return true, runSkillClassConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
case ephemeralArm(arm):
return true, runEphemeralConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
case hostExecArm(arm):
return true, runHostExecConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
case graphArm(arm) || uiArm(arm):
return true, runFanOutConstruct(ctx, root, arm, bootSystem, ctrl, sink, turn)
}
return false, nil
}
// runConstruct establishes host state through the same calls a frontend makes,
// records what this process can see, and returns so the process can exit. It
// never asserts: an arm that failed to establish something is reported as not
// measured, which is the one reading a missing "before" supports.
func runConstruct(dir, arm string) error {
root := rootFor(dir)
if err := root.create(); err != nil {
return err
}
if skillArm(arm) || arm == armEphemeralSkill || hostExecArm(arm) {
if err := writeProbeSkill(root.Workspace); err != nil {
return fmt.Errorf("probe skill: %w", err)
}
}
if total, writers := schedulerLimits(arm); total > 0 {
if err := root.writeProjectConfig(total, writers); err != nil {
return fmt.Errorf("project config: %w", err)
}
}
ctx := context.Background()
sink := &graphSink{}
bus, into := uiBus(arm, sink)
ctrl, prov, err := buildRuntime(ctx, root, arm, into)
if err != nil {
return fmt.Errorf("build: %w", err)
}
defer ctrl.Close()
// The process that dies is a window in production, and a window records what
// it served. Without that the trajectory this session leaves behind holds
// none of its own frames.
recordLikeAWindow(ctrl, bus)
bootSystem := bootSystemText(ctrl)
ctrl.EnsureSessionPath()
if ctrl.SessionPath() == "" {
return errUnexpected("a session path", "")
}
turn, err := runTurns(ctx, ctrl, 0, probeTurns)
if err != nil {
return err
}
if err := ctrl.SetGoalDurable(probeGoal); err != nil {
return fmt.Errorf("goal: %w", err)
}
if _, err := ctrl.Compact(ctx, agent.CompactRequest{Instructions: "Fold the probe turn."}); err != nil {
return fmt.Errorf("compact: %w", err)
}
if taken, err := armConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn); taken {
return err
}
if arm == armTodoIdentity {
// Identity A is already written and folded away. Grow, replace the list
// with identity B, then fold again: the second fold is what decides
// whether the model is left holding one current identity or two.
if turn, err = runTurns(ctx, ctrl, turn, refoldTurns); err != nil {
return err
}
if err := ctrl.Run(ctx, fmt.Sprintf("%s %s: replace the task list.", marker(turn+1), retodoSentinel)); err != nil {
return fmt.Errorf("retodo turn: %w", err)
}
// Grow well past the rewrite so the fold takes the new ids out of view.
// A scenario that leaves them readable asks nothing of the note.
if _, err = runTurns(ctx, ctrl, turn+1, probeTurns); err != nil {
return err
}
if _, err := ctrl.Compact(ctx, agent.CompactRequest{Instructions: "Fold again, after the identity moved."}); err != nil {
return fmt.Errorf("refold: %w", err)
}
} else if arm == armRefoldIntoBody {
// Grow past the fold, record the view a second fold will operate on,
// then fold again: its region reaches back into the stored body, which
// has no canonical counterpart to map a new boundary onto.
if _, err = runTurns(ctx, ctrl, turn, refoldTurns); err != nil {
return err
}
if err := writeObservation(root, capture(extraPhase(arm), arm, bootSystem, ctrl, sink, root)); err != nil {
return err
}
if _, err := ctrl.Compact(ctx, agent.CompactRequest{Instructions: "Fold again, into the stored body."}); err != nil {
return fmt.Errorf("refold: %w", err)
}
} else if appendsAfterFold(arm) {
if _, err = runTurns(ctx, ctrl, turn, 1); err != nil {
return err
}
}
if arm == armTailRewind || arm == armCoveredRewind {
if err := rewindTurn(root, arm, bootSystem, ctrl, sink); err != nil {
return err
}
}
return writeObservation(root, capture("construct", arm, bootSystem, ctrl, sink, root))
}
// successorArm reports whether this arm starts from an interrupted barrier.
func successorArm(arm string) bool {
_, ok := successorTurn(arm)
return ok
}
// runSuccessor is the middle process: it inherits the interruption and does
// what a person would do next — nothing, unrelated work, or something that
// reads like an answer to the dead question. It runs a model on purpose; the
// phase that judges what the host knows still does not.
func runSuccessor(dir, arm string) error {
root := rootFor(dir)
before, err := readObservation(root, "construct")
if err != nil {
return err
}
ctx := context.Background()
sink := &graphSink{}
ctrl, _, err := buildRuntime(ctx, root, arm, sink)
if err != nil {
return fmt.Errorf("build: %w", err)
}
defer ctrl.Close()
bootSystem := bootSystemText(ctrl)
session, err := sessionstore.LoadSession(before.SessionPath)
if err != nil {
return fmt.Errorf("load session: %w", err)
}
if err := ctrl.Resume(session, before.SessionPath); err != nil {
return fmt.Errorf("resume: %w", err)
}
if turn, _ := successorTurn(arm); turn != "" {
if err := ctrl.Run(ctx, turn); err != nil {
return fmt.Errorf("successor turn: %w", err)
}
}
return writeObservation(root, capture("successor", arm, bootSystem, ctrl, sink, root))
}
// runResume boots a second time against the same roots and reads host state.
// It deliberately never calls Run: the question is what the host can prove
// before a provider call, and a model that reconstructs a lost identity from
// the transcript would turn a real loss into a passing row.
func runResume(dir, arm string) error {
root := rootFor(dir)
before, err := readObservation(root, "construct")
if err != nil {
return fmt.Errorf("read construct observation: %w", err)
}
ctx := context.Background()
sink := &graphSink{}
bus, into := uiBus(arm, sink)
ctrl, _, err := buildRuntime(ctx, root, arm, into)
if err != nil {
return fmt.Errorf("build: %w", err)
}
defer ctrl.Close()
bootSystem := bootSystemText(ctrl)
session, err := sessionstore.LoadSession(before.SessionPath)
if err != nil {
return fmt.Errorf("load session: %w", err)
}
if err := ctrl.Resume(session, before.SessionPath); err != nil {
return fmt.Errorf("resume: %w", err)
}
obs := capture("resume", arm, bootSystem, ctrl, sink, root)
if uiArm(arm) {
ui := observeUI(root, ctrl, bus, obs)
obs.UI = &ui
}
return writeObservation(root, obs)
}
+133
View File
@@ -0,0 +1,133 @@
package main
import (
"fmt"
"sort"
"strconv"
"strings"
)
// A fold may only remove a run of markers from the middle: the pinned head
// stays, the retained tail stays, and everything between them becomes the
// digest. Any other shape means the view lost something no fold folded.
const (
verdictContiguous = "contiguous"
verdictHole = "HOLE"
markPrefix = "PROBE-MARK-"
echoPrefix = "PROBE-ECHO-"
)
// refoldRows report what the second fold did. They are their own set because
// the comparison is inside one process — the view before the refold against
// the view after it — not across the process boundary.
func refoldRows(prefold, postfold, resumed Observation, total int) []row {
return []row{
coverageRow("fold 1: stored body and coverage", prefold),
coverageRow("fold 2: stored body and coverage", postfold),
continuityRow("assistant work visible after fold 1", prefold, total),
continuityRow("assistant work visible after fold 2", postfold, total),
continuityRow("assistant work visible after restart", resumed, total),
turnsRow("user turns visible after fold 2", postfold),
}
}
func coverageRow(semantic string, o Observation) row {
return row{
Semantic: semantic, Authority: "compaction sidecar", Artifact: "<stem>.context.json",
Reconstruction: "projection.Messages + canonical[CoveredCount:]",
Before: fmt.Sprintf("canonical=%d", o.Transcript.Messages),
After: fmt.Sprintf("body=%d covered=%d spliced=+%d view=%d",
o.Sidecar.Messages, o.Sidecar.CoveredCount, o.View.SplicedFromTail, o.View.Messages),
Verdict: verdictStable,
}
}
// continuityRow judges the assistant side: a fold replaces one run of replies
// with a digest, so anything but a single surviving run is a loss no fold made.
func continuityRow(semantic string, o Observation, total int) row {
shape := markerShape(o.View.Echoes, echoPrefix)
verdict := verdictContiguous
if reason := markerHole(o.View.Echoes, echoPrefix, total); reason != "" {
verdict, shape = verdictHole, shape+" — "+reason
}
return row{
Semantic: semantic, Authority: "the spliced model-visible view",
Artifact: "none: derived", Reconstruction: "assistant replies surviving in the view",
Before: fmt.Sprintf("emitted 1-%d", total), After: shape, Verdict: verdict,
}
}
// turnsRow reports the user side beside it: the retention budget keeps those
// verbatim, which is why a one-sided tag could not see a dropped reply.
func turnsRow(semantic string, o Observation) row {
return row{
Semantic: semantic, Authority: "the spliced model-visible view",
Artifact: "none: derived", Reconstruction: "user turns surviving in the view",
Before: "kept by the retention budget",
After: markerShape(o.View.Markers, markPrefix), Verdict: verdictStable,
}
}
// pad renders a tag number the way the scripted turns emit it.
func pad(n int) string { return fmt.Sprintf("%03d", n) }
func markerNumbers(markers []string, prefix string) []int {
seen := map[int]bool{}
var out []int
for _, m := range markers {
n, err := strconv.Atoi(strings.TrimPrefix(m, prefix))
if err != nil || seen[n] {
continue
}
seen[n] = true
out = append(out, n)
}
sort.Ints(out)
return out
}
// markerShape renders the surviving markers as runs, which is the form the
// judgement is about: "1, 9-12" says a fold took 2-8.
func markerShape(markers []string, prefix string) string {
nums := markerNumbers(markers, prefix)
if len(nums) == 0 {
return "none"
}
var runs []string
start := nums[0]
for i := 1; i <= len(nums); i++ {
if i < len(nums) && nums[i] == nums[i-1]+1 {
continue
}
if end := nums[i-1]; end == start {
runs = append(runs, strconv.Itoa(start))
} else {
runs = append(runs, fmt.Sprintf("%d-%d", start, end))
}
if i < len(nums) {
start = nums[i]
}
}
return strings.Join(runs, ", ")
}
// markerHole names the violation, or returns "" when the shape is one a fold
// can produce: an optional pinned head starting at 1, then one run reaching the
// last turn. Two gaps, or a tail that stops short of it, are losses.
func markerHole(markers []string, prefix string, total int) string {
nums := markerNumbers(markers, prefix)
if len(nums) == 0 {
return "the view shows no turn at all"
}
runs := strings.Split(markerShape(markers, prefix), ", ")
if last := nums[len(nums)-1]; last != total {
return fmt.Sprintf("the newest turn %d is missing", total)
}
if len(runs) > 2 {
return fmt.Sprintf("%d separate runs; a fold removes one", len(runs))
}
if len(runs) == 2 && nums[0] != 1 {
return fmt.Sprintf("the kept head starts at %d, not the pinned first turn", nums[0])
}
return ""
}
+45
View File
@@ -0,0 +1,45 @@
package main
import "testing"
// The oracle has to be able to fail, or "no hole" says nothing. These are the
// shapes a fold can produce and the shapes only a loss can.
func TestMarkerHoleDiscriminates(t *testing.T) {
echo := func(ns ...int) []string {
out := make([]string, 0, len(ns))
for _, n := range ns {
out = append(out, echoPrefix+pad(n))
}
return out
}
cases := []struct {
name string
in []string
total int
holed bool
}{
{"nothing folded", echo(1, 2, 3, 4, 5), 5, false},
{"middle folded away", echo(4, 5), 5, false},
{"pinned head kept beside the tail", echo(1, 4, 5), 5, false},
{"a reply dropped inside the kept tail", echo(3, 5), 5, true},
{"two gaps", echo(1, 3, 5), 5, true},
{"newest turn missing", echo(3, 4), 5, true},
{"head kept but not the first turn", echo(2, 4, 5), 5, true},
{"empty view", nil, 5, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
reason := markerHole(tc.in, echoPrefix, tc.total)
if holed := reason != ""; holed != tc.holed {
t.Fatalf("markerHole(%v) = %q, want holed=%v", tc.in, reason, tc.holed)
}
})
}
}
func TestMarkerShapeRendersRuns(t *testing.T) {
in := []string{echoPrefix + pad(1), echoPrefix + pad(9), echoPrefix + pad(10), echoPrefix + pad(11)}
if got := markerShape(in, echoPrefix); got != "1, 9-11" {
t.Fatalf("markerShape = %q, want %q", got, "1, 9-11")
}
}
+60
View File
@@ -0,0 +1,60 @@
package main
import (
"fmt"
"strings"
)
// renderMatrix prints one table per arm. It reports what each row's value was
// on both sides of the process boundary, because a verdict without the two
// readings behind it is an assertion.
func renderMatrix(results []armResult, work string) string {
var b strings.Builder
b.WriteString("# Runtime resurrection matrix\n\n")
b.WriteString("Measurement boundary: OS process. Phase B loads and inspects; it never calls a model.\n\n")
fmt.Fprintf(&b, "Arm roots: %s\n\n", work)
for _, res := range results {
fmt.Fprintf(&b, "## arm: %s\n\n%s\n\n", res.Arm, res.Asks)
if res.Invalid != "" {
fmt.Fprintf(&b, "**ARM INVALID — %s.** Its rows below establish nothing.\n\n", res.Invalid)
}
b.WriteString("| Semantic | Before death | After new process | Verdict | Authority | Artifact |\n")
b.WriteString("| --- | --- | --- | --- | --- | --- |\n")
for _, r := range res.Rows {
fmt.Fprintf(&b, "| %s | %s | %s | %s | %s | %s |\n",
r.Semantic, cell(r.Before), cell(r.After), r.Verdict, r.Authority, r.Artifact)
}
b.WriteString("\n")
b.WriteString(summarize(res))
b.WriteString("\n")
}
return b.String()
}
func summarize(res armResult) string {
counts := map[string]int{}
for _, r := range res.Rows {
counts[r.Verdict]++
}
var parts []string
// Every verdict the matrix can produce. A summary that silently omits one
// is how a VIOLATED row reads as a clean arm.
for _, v := range []string{verdictPersisted, verdictExact, verdictLossy, verdictLost, verdictNotMeasured,
verdictChanged, verdictStable, verdictContiguous, verdictHole, verdictHolds, verdictViolated} {
if counts[v] > 0 {
parts = append(parts, fmt.Sprintf("%s=%d", v, counts[v]))
}
}
return strings.Join(parts, " ") + "\n"
}
func cell(s string) string {
if strings.TrimSpace(s) == "" {
return "—"
}
s = strings.ReplaceAll(s, "|", "\\|")
if len(s) > 64 {
return s[:61] + "..."
}
return s
}
+323
View File
@@ -0,0 +1,323 @@
package main
import (
"encoding/json"
"fmt"
"sort"
"strings"
)
// Verdicts. They are deliberately not booleans: "the host holds it verbatim"
// and "a fold over canonical artifacts happens to agree" are different claims
// about durability, and a row that was never established supports neither.
const (
verdictPersisted = "persisted-direct"
verdictExact = "reconstructed-exact"
verdictLossy = "reconstructed-lossy"
verdictLost = "lost"
verdictNotMeasured = "not-measured"
verdictChanged = "changed"
verdictStable = "stable"
)
type row struct {
Semantic string `json:"semantic"`
Authority string `json:"authority"`
Artifact string `json:"artifact"`
Reconstruction string `json:"reconstruction"`
Before string `json:"before"`
After string `json:"after"`
Verdict string `json:"verdict"`
}
type armResult struct {
Arm string `json:"arm"`
Asks string `json:"asks"`
Invalid string `json:"invalid,omitempty"`
Rows []row `json:"rows"`
Extra *Observation `json:"extra,omitempty"`
Before Observation `json:"before"`
After Observation `json:"after"`
}
func classify(a arm, extra *Observation, before, after Observation) armResult {
res := armResult{Arm: a.name, Asks: a.asks, Extra: extra, Before: before, After: after}
prefix := prefixRow(before, after)
res.Rows = append(res.Rows, prefix)
if a.name == "system-swap" && prefix.Verdict != verdictChanged {
res.Invalid = "the lever did not move the stable prefix, so this arm measures nothing"
}
if a.name == "exact" && prefix.Verdict != verdictStable {
res.Invalid = "the stable prefix moved without a lever, so this arm is not the control it claims"
}
res.Rows = append(res.Rows,
valueRow("system row in the session", "the transcript's own leading system message",
"<stem>.jsonl", "loaded with the transcript",
before.SystemHash, after.SystemHash, true),
valueRow("canonical transcript", "session transcript + event log",
"<stem>.jsonl, <stem>.events.jsonl", "LoadSession replays the append log",
before.Transcript.Digest, after.Transcript.Digest, true),
valueRow("active goal identity", "goalMachine (control)",
"<stem>.goal-state.json", "restoreFromState on Resume",
before.Goal.Goal, after.Goal.Goal, true),
valueRow("goal status", "goalMachine (control)",
"<stem>.goal-state.json", "restoreFromState on Resume",
before.Goal.Status, after.Goal.Status, true),
valueRow("todo host identity", "agent todoState (host-owned, never in the prompt)",
"none verbatim; goal-state.json snapshots a terminal goal's list",
"rebuilt from the transcript's todo_write / complete_step receipts",
todoIDs(before), todoIDs(after), false),
valueRow("open decisions", "approvalManager (control, in-memory)",
"none", "none: derived from live pending approvals/asks",
decisionIDs(before), decisionIDs(after), false),
valueRow("compaction projection (stored)", "compaction sidecar",
"<stem>.context.json", "LoadProjectionSidecar on bind",
sidecarClaim(before), sidecarClaim(after), true),
useRow("compaction projection (in use)", before, after),
valueRow("run graph nodes", "event sink (GraphDelta)",
"none observed", "fold GraphDelta events, if any survive",
graphNodes(before), graphNodes(after), false),
valueRow("run graph grants", "event sink (GraphDelta)",
"none observed", "fold GraphDelta events, if any survive",
graphMap(before.Graph.Grants), graphMap(after.Graph.Grants), false),
valueRow("run graph wait causes", "event sink (GraphDelta)",
"none observed", "fold GraphDelta events, if any survive",
graphMap(before.Graph.Waits), graphMap(after.Graph.Waits), false),
)
switch {
case successorArm(a.name) && extra != nil:
res.Rows = append(res.Rows, successorRows(a.name, *extra, after)...)
// The premise is a barrier open at death. The construct process is its
// own waiter, so it reports no interruption of its own.
if len(before.Decisions) == 0 {
res.Invalid = "no decision was open when the first process died"
}
case a.name == armOpenDecision:
res.Rows = append(res.Rows, decisionRows(before, after)...)
if len(before.Decisions) == 0 {
res.Invalid = "no decision was open when the process died, so nothing was interrupted"
}
case deriveArm(a.name):
res.Rows = append(res.Rows, deriveRows(a.name, before, after)...)
res.Invalid = deriveArmInvalid(a.name, before)
case terminalArm(a.name):
res.Rows = append(res.Rows, terminalRows(a.name, before, after)...)
res.Invalid = terminalArmInvalid(a.name, before)
case schedulerWaitArm(a.name):
res.Rows = append(res.Rows, schedulerWaitRows(a.name, before, after)...)
res.Invalid = schedulerArmInvalid(a.name, before)
case graphArm(a.name):
res.Rows = append(res.Rows, graphRows(before, after)...)
res.Invalid = graphArmInvalid(a.name, before)
case uiArm(a.name):
res.Rows = append(res.Rows, uiRows(before, after)...)
res.Invalid = uiArmInvalid(before, after)
case lineageArm(a.name):
res.Rows = append(res.Rows, lineageRows(a.name, before)...)
case cancelRoutingArm(a.name):
res.Rows = append(res.Rows, cancelRoutingRows(a.name, before)...)
case loneTaskArm(a.name):
res.Rows = append(res.Rows, loneTaskRows(a.name, before, after)...)
res.Invalid = loneTaskArmInvalid(a.name, before)
case activeStoreArm(a.name):
res.Rows = append(res.Rows, activeStoreRows(before, after)...)
res.Invalid = activeStoreInvalid(before)
case skillArm(a.name):
res.Rows = append(res.Rows, skillClassRows(a.name, before, after)...)
res.Invalid = skillArmInvalid(a.name, before)
case ephemeralArm(a.name):
res.Rows = append(res.Rows, ephemeralRows(a.name, before, after)...)
res.Invalid = ephemeralArmInvalid(before)
case hostExecArm(a.name):
res.Rows = append(res.Rows, hostExecRows(a.name, before, after)...)
res.Invalid = hostExecArmInvalid(a.name, before)
case a.name == armTodoIdentity:
res.Rows = append(res.Rows, todoRows(before, after)...)
case a.name == armRefoldIntoBody && extra != nil:
res.Rows = append(res.Rows, refoldRows(*extra, before, after, probeTurns+refoldTurns)...)
if extra.Sidecar.Messages == 0 {
res.Invalid = "the first fold stored no body, so the second had none to reach into"
}
case (a.name == armTailRewind || a.name == armCoveredRewind) && extra != nil:
res.Rows = append(res.Rows, useRow("projection across the rewind, same process", *extra, before))
res.Rows = append(res.Rows, rewindLandingRow(a.name, *extra, before))
res.Invalid = rewindLandedWrong(a.name, *extra, before)
}
return res
}
// prefixRow is the arm's own control rather than a durability claim: it reports
// whether the stable prefix moved, which is the thing each arm sets up.
func prefixRow(before, after Observation) row {
verdict := verdictStable
if before.BootSystemHash != after.BootSystemHash {
verdict = verdictChanged
}
return row{
Semantic: "stable prefix identity (composed)", Authority: "boot assembly (recomposed per process)",
Artifact: "none (composed from config, memory and instructions)", Reconstruction: "recomposed at boot",
Before: before.BootSystemHash, After: after.BootSystemHash, Verdict: verdict,
}
}
func valueRow(semantic, authority, artifact, reconstruction, before, after string, persisted bool) row {
r := row{Semantic: semantic, Authority: authority, Artifact: artifact,
Reconstruction: reconstruction, Before: before, After: after}
switch {
case before == "" || before == "-":
r.Verdict = verdictNotMeasured
case after == "" || after == "-":
r.Verdict = verdictLost
case before != after:
r.Verdict = verdictLossy
case persisted:
r.Verdict = verdictPersisted
default:
r.Verdict = verdictExact
}
return r
}
func todoIDs(o Observation) string {
var ids []string
for _, t := range o.Todos {
id := t.StepID
if id == "" {
id = "(no step_id)"
}
ids = append(ids, id+":"+t.Status)
}
return join(ids)
}
func decisionIDs(o Observation) string {
var ids []string
for _, d := range o.Decisions {
ids = append(ids, string(d.Kind)+":"+d.ID)
}
return join(ids)
}
func sidecarClaim(o Observation) string {
if !o.Sidecar.Present {
return ""
}
return fmt.Sprintf("v%d msgs=%d covered=%d hash=%s",
o.Sidecar.ProjectionVersion, o.Sidecar.Messages, o.Sidecar.CoveredCount, short(o.Sidecar.CoveredPrefixHash))
}
// rewindLandingRow says where the rewind actually landed relative to the fold.
// An arm that names a side and does not reach it is not measuring that side.
func rewindLandingRow(arm string, prerewind, after Observation) row {
side := "above the fold boundary"
if after.Transcript.Messages < prerewind.Sidecar.CoveredCount {
side = "below the fold boundary"
}
return row{
Semantic: "where the rewind landed", Authority: "checkpoint boundary against the sidecar",
Artifact: "none: derived", Reconstruction: "canonical length after the rewind against CoveredCount before it",
Before: fmt.Sprintf("covered=%d", prerewind.Sidecar.CoveredCount),
After: fmt.Sprintf("canonical=%d, %s", after.Transcript.Messages, side), Verdict: verdictStable,
}
}
// rewindLandedWrong invalidates an arm that aimed at one side of the fold and
// reached the other. covered-rewind stopped removing folded history the moment
// coverage became a real boundary, and nothing in the matrix said so.
func rewindLandedWrong(arm string, prerewind, after Observation) string {
below := after.Transcript.Messages < prerewind.Sidecar.CoveredCount
switch {
case arm == armCoveredRewind && !below:
return "the rewind landed above the fold boundary, so it removed no folded history"
case arm == armTailRewind && below:
return "the rewind landed below the fold boundary, so it is not a tail-only truncation"
}
return ""
}
// useRow turns on whether the next request would ride the projection, not on
// the token counts beside it: a tail edit legitimately moves those, and letting
// them decide the verdict reported a surviving projection as a loss.
func useRow(semantic string, before, after Observation) row {
r := valueRow(semantic, "projectionValid over the covered prefix",
"none: a judgement, not a stored value",
"revalidated against the canonical prefix on every read",
projectionState(before), projectionState(after), false)
r.Before, r.After = projectionUse(before), projectionUse(after)
return r
}
// projectionState is the compared value: in use, not in use, or never measured.
func projectionState(o Observation) string {
switch o.Context.CheckpointState {
case "":
return ""
case "none":
return "NOT-in-use"
}
return "in-use"
}
// projectionUse reads the host's own judgement rather than the stored claim.
// CheckpointState is "none" whenever validation failed, which is the difference
// between a sidecar that loaded and one the next request will actually ride.
// The label itself is not compared — "applied" and "restored" are the same
// answer reached from different sides — so the row turns on use and size.
func projectionUse(o Observation) string {
if o.Context.CheckpointState == "" {
return ""
}
use := "in-use"
if o.Context.CheckpointState == "none" {
use = "NOT-in-use"
}
return fmt.Sprintf("%s projected=%d canonical=%d", use,
o.Context.ProjectedTokens, o.Context.CanonicalTokens)
}
// graphNodes renders the fold's nodes in a stable order. The fold keeps
// first-seen order, which two processes reach by different routes, so an
// unsorted rendering would report a reordering as a changed graph.
func graphNodes(o Observation) string {
if len(o.Graph.Nodes) == 0 {
return ""
}
ids := make([]string, 0, len(o.Graph.Nodes))
for _, n := range o.Graph.Nodes {
ids = append(ids, n.ID+":"+string(n.State))
}
sort.Strings(ids)
return join(ids)
}
// graphMap renders one per-node attribute set. Map iteration is unordered, so
// without the sort two identical envelopes would compare as different.
func graphMap(m map[string]string) string {
if len(m) == 0 {
return ""
}
out := make([]string, 0, len(m))
for k, v := range m {
out = append(out, k+"="+v)
}
sort.Strings(out)
return join(out)
}
func join(in []string) string {
if len(in) == 0 {
return ""
}
return strings.Join(in, ", ")
}
func short(s string) string {
if len(s) <= 12 {
return s
}
return s[:12]
}
func marshalResults(results []armResult) ([]byte, error) {
return json.MarshalIndent(results, "", " ")
}
+265
View File
@@ -0,0 +1,265 @@
package main
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"time"
"reasonix/internal/assembly/boot"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/contract/event"
"reasonix/internal/session/control"
)
// armRoot names the four directories one arm owns. Home and Workspace are
// separate so a lever can change what the prefix folds without touching the
// transcript, and Sessions is explicit so the probe never writes to the
// developer's own session directory.
type armRoot struct{ Home, Workspace, Sessions, Dir string }
func rootFor(dir string) armRoot {
return armRoot{
Dir: dir,
Home: filepath.Join(dir, "home"),
Workspace: filepath.Join(dir, "workspace"),
Sessions: filepath.Join(dir, "sessions"),
}
}
func (r armRoot) create() error {
for _, d := range []string{r.Home, r.Workspace, r.Sessions} {
if err := os.MkdirAll(d, 0o755); err != nil {
return err
}
}
return nil
}
// writeProjectConfig puts a project config beside the workspace, which is where
// a person configures these ceilings. The scheduler arms need them small enough
// to reach a specific refusal, and a poked field would prove nothing about the
// judgement production makes.
func (r armRoot) writeProjectConfig(total, writers int) error {
body := fmt.Sprintf("[agent]\nmax_subagent_concurrency = %d\nmax_parallel_writers = %d\n", total, writers)
return os.WriteFile(filepath.Join(r.Workspace, "reasonix.toml"), []byte(body), 0o644)
}
// requireFresh refuses a root a previous run left behind. A reused root is how
// a lever's effect ends up already present in the construct phase, which reads
// as "the lever changed nothing" — a false negative that looks like a result.
func (r armRoot) requireFresh() error {
entries, err := os.ReadDir(r.Dir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
if len(entries) > 0 {
return fmt.Errorf("arm root %s is not empty; each run needs a clean root", r.Dir)
}
return nil
}
// graphSink records what the run graph published while this process was alive.
// It is the only "before" the graph row has: nothing else in a session's
// artifacts is known to carry it, and a row with no before is not measured.
type graphSink struct {
mu sync.Mutex
graph agentgraph.Graph
deltas int
// waits is every wait cause published per node, in order. The fold keeps
// only the latest, which cannot say whether a cause was reported once or
// replaced — and that is exactly what the transition arm asks.
waits map[string][]string
// phases is every subagent status the parent was told, per call. A run
// handed to a job publishes no graph node at all, so for those this is the
// only live evidence that anything happened.
phases map[string][]string
// tools is when each call was dispatched and when it answered. A stop that
// reached a running tool is visible as the second arriving early.
tools map[string][2]time.Time
}
func (s *graphSink) Emit(e event.Event) {
if e.Kind == event.ToolDispatch || e.Kind == event.ToolResult {
s.noteTool(e)
return
}
if id, phase, ok := progressPhase(e); ok {
s.mu.Lock()
if s.phases == nil {
s.phases = map[string][]string{}
}
s.phases[id] = append(s.phases[id], phase)
s.mu.Unlock()
return
}
if e.Kind != event.GraphDelta || e.Graph == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
s.deltas++
for _, n := range e.Graph.Nodes {
if n.Wait == "" {
continue
}
if s.waits == nil {
s.waits = map[string][]string{}
}
s.waits[n.ID] = append(s.waits[n.ID], string(n.Wait))
}
s.graph.Apply(*e.Graph)
}
func (s *graphSink) snapshot() (agentgraph.Graph, int) {
s.mu.Lock()
defer s.mu.Unlock()
return s.graph, s.deltas
}
// noteTool records the two edges of a tool call this probe may be asked about.
func (s *graphSink) noteTool(e event.Event) {
if e.Tool.ID == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
if s.tools == nil {
s.tools = map[string][2]time.Time{}
}
at := s.tools[e.Tool.ID]
if e.Kind == event.ToolDispatch {
at[0] = time.Now()
} else {
at[1] = time.Now()
}
s.tools[e.Tool.ID] = at
}
func (s *graphSink) toolDispatched(id string) (time.Time, bool) {
s.mu.Lock()
defer s.mu.Unlock()
at, ok := s.tools[id]
return at[0], ok && !at[0].IsZero()
}
func (s *graphSink) toolResult(id string) (time.Time, bool) {
s.mu.Lock()
defer s.mu.Unlock()
at, ok := s.tools[id]
return at[1], ok && !at[1].IsZero()
}
// phaseSeries is every status published for one call, in publication order.
func (s *graphSink) phaseSeries() map[string][]string {
s.mu.Lock()
defer s.mu.Unlock()
out := map[string][]string{}
for id, seq := range s.phases {
out[id] = append([]string(nil), seq...)
}
return out
}
// waitSeries is every cause published for one node, in publication order.
func (s *graphSink) waitSeries() map[string][]string {
s.mu.Lock()
defer s.mu.Unlock()
out := map[string][]string{}
for id, seq := range s.waits {
out[id] = append([]string(nil), seq...)
}
return out
}
// buildRuntime assembles a real runtime bound to this arm's roots. Everything a
// frontend would supply is supplied; only the provider is scripted, so the
// probe never depends on a network or a key.
func buildRuntime(ctx context.Context, root armRoot, arm string, sink event.Sink) (*control.Controller, *scripted, error) {
prov := newScripted(arm)
ctrl, err := boot.Build(ctx, boot.Options{
Version: "runtime-resume-probe",
Model: probeModelRef,
Sink: sink,
WorkspaceRoot: root.Workspace,
Home: root.Home,
SessionDir: root.Sessions,
ProviderResolver: resolverFor(prov),
Stderr: io.Discard,
HeadlessApprovalMode: approvalMode(arm),
})
if err != nil {
return nil, nil, err
}
ctrl.ApplyHeadlessApprovalMode(approvalMode(arm))
return ctrl, prov, nil
}
// approvalMode is the gate this arm needs. Everything else runs denied, the
// strictest a headless frontend can be. A fleet is not read-only, so under a
// denied gate the dispatch is refused before any child starts and the arm
// measures a permission decision instead of a process boundary.
func approvalMode(arm string) string {
if graphArm(arm) || uiArm(arm) || loneTaskArm(arm) || cancelRoutingArm(arm) || lineageArm(arm) ||
schedulerWaitArm(arm) || terminalArm(arm) || deriveArm(arm) || activeStoreArm(arm) ||
skillArm(arm) || ephemeralArm(arm) || hostExecArm(arm) {
return control.ToolApprovalAuto
}
return "deny"
}
// childEnv binds the child process to this arm's home through the environment
// as well as through Options.Home: a Controller's later re-reads and the shared
// history index still read the process, not the assembly.
func childEnv(root armRoot) []string {
env := append(os.Environ(),
"REASONIX_HOME="+root.Home,
"REASONIX_STATE_HOME="+root.Home,
)
// A child runs in its own workspace, so the one arm that has to build the
// frontend is told where the checkout is rather than searching for it.
if repo := repoRoot(); repo != "" {
env = append(env, envProbeRepo+"="+repo)
}
return env
}
// repoRoot walks up from the orchestrator's working directory for the module
// this probe lives in. Empty when it is not run from a checkout, which the arm
// reports as not measured rather than as a failure.
func repoRoot() string {
dir, err := os.Getwd()
if err != nil {
return ""
}
for {
if b, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil &&
strings.HasPrefix(string(b), "module reasonix") {
return dir
}
parent := filepath.Dir(dir)
if parent == dir {
return ""
}
dir = parent
}
}
// toolIDs is every call this sink saw dispatched, which is where a host-minted
// event id first appears: it is not in the graph and not in the transcript.
func (s *graphSink) toolIDs() map[string]bool {
s.mu.Lock()
defer s.mu.Unlock()
out := make(map[string]bool, len(s.tools))
for id := range s.tools {
out[id] = true
}
return out
}
+376
View File
@@ -0,0 +1,376 @@
package main
import (
"context"
"fmt"
"os"
"sort"
"strings"
"time"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/session/control"
)
// The scheduler-wait arms. Each dies with one item ready and refused admission,
// and they are separate because the scheduler decides in a fixed order —
// slots, then writers, then claim. An arm that let an earlier cause stand would
// report it as the later one, so each has to clear every check above the one it
// is measuring.
const (
// armWaitSlots fills the session's total ceiling with runs that never end.
// The refused item is read-only: it clears every writer check by not being
// a writer at all.
armWaitSlots = "wait-slots"
// armWaitWriters leaves total capacity free and fills the writer ceiling.
// Its two writers declare disjoint paths, so a claim conflict cannot be
// what refuses the second one.
armWaitWriters = "wait-writers"
// armWaitClaim leaves both ceilings free and overlaps two write paths. A
// single fan-out cannot reach this: preflight refuses concurrent writers
// whose claims overlap, so the holder is dispatched as a background fleet
// and the refused writer comes from a later one.
armWaitClaim = "wait-claim"
// armWaitTransition asks what a wait cause means once the thing it named
// stops being what holds the item back. The scheduler reports the cause
// once, when the request queues; the blocker can change afterwards.
armWaitTransition = "wait-transition"
)
func schedulerWaitArm(name string) bool {
switch name {
case armWaitSlots, armWaitWriters, armWaitClaim, armWaitTransition:
return true
}
return false
}
// schedulerLimits are the ceilings an arm needs to reach its refusal. They are
// written to the project config, so the run makes the same judgement a person
// configuring the session would get.
func schedulerLimits(arm string) (total, writers int) {
switch arm {
case armTerminalCancelled:
// One slot, so the group has an item running and items that never got
// one — the split a cancellation is reported against.
return 1, 1
case armLineageBackground, armLineageJobKill:
// Room for a second run: the alive check asks whether the manager can
// still start anything, and a ceiling of one would answer for the
// hanging child instead.
return 4, 4
case armTaskFgQueued, armTaskBgQueued, armTaskFgQueuedCancel, armTaskBgQueuedCancel:
// One slot, held by the fleet dispatched ahead of the delegation, so the
// refusal the arm records is the scheduler's and not a race.
return 1, 1
case armUIGraphMixed:
// One slot, so the item after the one that hangs is refused: the death
// then holds a node that was queued and never admitted.
return 1, 1
case armHostQueued, armHeadlessQueuedCancel:
// One slot, held by the fan-out dispatched ahead of the invocation.
return 1, 1
case armSkillQueued, armReadOnlySkillQueued:
// One slot, held by the fleet dispatched ahead of the skill, so the
// refusal the arm reads is the scheduler's and not a race.
return 1, 1
case armFleetActiveStore:
// Two slots: the delegation holds one to the death, and the fan-out's
// first item frees the other for exactly one of the two behind it.
return 2, 2
case armWaitSlots:
return 2, 1
case armWaitWriters:
return 6, 1
case armWaitClaim:
return 6, 3
case armWaitTransition:
return 3, 1
}
return 0, 0
}
// wantedCause is the refusal this arm exists to produce. The transition arm
// starts at slots and asks what happens to that answer afterwards.
func wantedCause(arm string) agentgraph.WaitCause {
switch arm {
case armWaitWriters:
return agentgraph.WaitWriters
case armWaitClaim:
return agentgraph.WaitClaim
}
return agentgraph.WaitSlots
}
// waitingWorkers are the nodes the scheduler refused and has not admitted: the
// arm's premise. State is checked as well as the cause, because a node that
// went on to run still carries the cause it waited under.
func waitingWorkers(g agentgraph.Graph, cause agentgraph.WaitCause) []string {
var out []string
for _, n := range g.Nodes {
if n.Kind == agentgraph.KindGroup || n.State != agentgraph.StateQueued {
continue
}
if n.Wait == cause {
out = append(out, n.ID)
}
}
sort.Strings(out)
return out
}
// waitForRefusal blocks until one item is queued under this arm's cause.
func waitForRefusal(sink *graphSink, arm string) error {
cause := wantedCause(arm)
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
graph, _ := sink.snapshot()
if len(waitingWorkers(graph, cause)) > 0 {
return nil
}
time.Sleep(50 * time.Millisecond)
}
graph, _ := sink.snapshot()
return errUnexpected("an item queued under "+string(cause), waitStates(graph))
}
func waitStates(g agentgraph.Graph) string {
out := make([]string, 0, len(g.Nodes))
for _, n := range g.Nodes {
out = append(out, fmt.Sprintf("%s:%s/%s", n.ID, n.State, orNone(string(n.Wait))))
}
sort.Strings(out)
return orNone(join(out))
}
// schedTurn is the prompt that opens this arm's fan-out. The slots arm needs
// only one fleet; the others name both, and the latch order dispatches the
// holder first while the refused fleet waits on the holder's own signal.
func schedTurn(n int, arm string) string {
if arm == armWaitSlots {
return fmt.Sprintf("%s %s: dispatch the fan-out.", marker(n), fleetPairSentinel)
}
return fmt.Sprintf("%s %s %s: dispatch the fan-out.", marker(n), fleetHolderSentinel, fleetRefusedSentinel)
}
// runSchedulerWaitConstruct drives the arm to its refusal and exits while the
// item is still queued. The transition arm goes further: it frees the capacity
// the refusal named and asks what the reported cause does about it.
func runSchedulerWaitConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
go func() { _ = ctrl.Run(ctx, schedTurn(turn+1, arm)) }()
if err := waitForRefusal(sink, arm); err != nil {
return err
}
if arm == armWaitTransition {
if err := releaseAndSettle(sink, prov); err != nil {
return err
}
}
if err := writeObservation(root, capture("construct", arm, bootSystem, ctrl, sink, root)); err != nil {
return err
}
os.Exit(0)
return nil
}
// releaseAndSettle frees the run that filled total capacity, then waits for the
// graph to show it gone. What the refused item's cause does after that is the
// arm's question: the ceiling it named has room, and the writer ceiling that
// now holds it back was never reported.
func releaseAndSettle(sink *graphSink, prov *scripted) error {
prov.releaseOnce()
deadline := time.Now().Add(60 * time.Second)
for time.Now().Before(deadline) {
graph, _ := sink.snapshot()
if settledWorkers(graph) > 0 && len(waitingWorkers(graph, agentgraph.WaitSlots)) > 0 {
// Give any second report a chance to arrive before reading.
time.Sleep(500 * time.Millisecond)
return nil
}
time.Sleep(50 * time.Millisecond)
}
graph, _ := sink.snapshot()
return errUnexpected("capacity freed while a refusal still stands", waitStates(graph))
}
func settledWorkers(g agentgraph.Graph) int {
n := 0
for _, node := range g.Nodes {
if node.Kind != agentgraph.KindGroup && node.State != "" && node.State.Terminal() {
n++
}
}
return n
}
// schedulerWaitRows report what a restart inherits about an item the scheduler
// refused. The classification is already right — STARTED settled that — so the
// question here is narrower: can the host still say why it never started?
func schedulerWaitRows(arm string, before, after Observation) []row {
rows := []row{refusalRow(arm, before, after)}
if arm == armWaitTransition {
rows = append(rows, capacityFreedRow(before))
}
return append(rows,
waitSeriesRow(arm, before),
executionKindRow(before, after),
waitProvenanceRow(before, after),
schedulerGateRow(before, after),
)
}
// capacityFreedRow is the transition arm's evidence. It freed the ceiling the
// refusal named and the item stayed queued, held by a different one — so what
// the reported cause does next is a question about the contract, not a race.
func capacityFreedRow(before Observation) row {
return row{
Semantic: "the ceiling the refusal named, once freed", Authority: "the arm, through a child it releases",
Artifact: "none", Reconstruction: "an item settles while the refused one stays queued",
Before: fmt.Sprintf("%d settled, still queued: %s", before.SettledWorkers, orNone(join(before.QueuedWorkers))),
After: "not applicable: the scheduler is gone", Verdict: verdictStable,
}
}
// refusalRow is the arm's own control: it reached the refusal it names, and no
// earlier one. Reaching a different cause means an upper check intercepted it,
// and the arm measured someone else's question.
func refusalRow(arm string, before, after Observation) row {
want := string(wantedCause(arm))
verdict := verdictHolds
if before.WaitCauses[want] == 0 {
verdict = verdictNotMeasured
}
return row{
Semantic: "the refusal this arm reached", Authority: "SubagentScheduler.canStartLocked",
Artifact: "none: reported once through OnQueued", Reconstruction: "slots, then writers, then claim",
Before: causeSummary(before), After: causeSummary(after), Verdict: verdict,
}
}
// waitSeriesRow is what the transition arm exists for: how many times a cause
// was published for the item that never started. Once means the answer is the
// cause that queued it, not the one that held it last.
func waitSeriesRow(arm string, before Observation) row {
return row{
Semantic: "how often a wait cause was reported", Authority: "OnQueued, which fires once per request",
Artifact: "none", Reconstruction: causeContract(before),
Before: seriesSummary(before), After: "not applicable: the scheduler is gone",
Verdict: verdictStable,
}
}
// causeContract states what the observed reporting says the cause means. It is
// read off the series rather than assumed: one report can only be the cause
// that queued the request, and more than one would track the current blocker.
func causeContract(o Observation) string {
for _, seq := range o.WaitSeries {
if len(seq) > 1 {
return "reported again as the blocker changes: the cause is the current one"
}
}
return "reported once: the cause is the one that queued the request, not the one holding it now"
}
// waitProvenanceRow is what the batch was built to expose and then close. The
// refusal exists only at the moment the scheduler makes it, so it is recorded
// there; this compares what the graph showed against what a later process can
// still read back.
func waitProvenanceRow(before, after Observation) row {
return valueRow("why the item never started", "the execution journal's queue records",
"<stem>.execution.jsonl", "the cause the scheduler gave when it first refused admission",
causeSummary(before), journalCauses(after), true)
}
// journalCauses is the refusal a restart reads back, counted the way the graph
// counts it so the two sides of the row are the same measurement.
func journalCauses(o Observation) string {
counts := map[string]int{}
for _, e := range o.Executions {
if e.Queued() {
counts[e.Cause]++
}
}
if len(counts) == 0 {
return ""
}
keys := make([]string, 0, len(counts))
for k := range counts {
keys = append(keys, k)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, k := range keys {
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
}
return strings.Join(parts, " ")
}
// schedulerGateRow is the other half of what a queue record proves. An item
// only reaches the scheduler once its dependencies are answered, so a queued
// entry is durable evidence it crossed that gate — the one thing a dependency
// edge alone could never say.
func schedulerGateRow(before, after Observation) row {
return valueRow("items proven to have reached the scheduler", "the execution journal's queue records",
"<stem>.execution.jsonl", "a queue record implies the dependency gate was crossed",
queuedIDs(before), queuedIDs(after), true)
}
func queuedIDs(o Observation) string {
var out []string
for _, e := range o.Executions {
if e.Queued() {
out = append(out, e.ID)
}
}
sort.Strings(out)
return join(out)
}
func causeSummary(o Observation) string {
if len(o.WaitCauses) == 0 {
return ""
}
keys := make([]string, 0, len(o.WaitCauses))
for k := range o.WaitCauses {
keys = append(keys, k)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, k := range keys {
parts = append(parts, fmt.Sprintf("%s=%d", k, o.WaitCauses[k]))
}
return strings.Join(parts, " ")
}
func seriesSummary(o Observation) string {
if len(o.WaitSeries) == 0 {
return "none"
}
ids := make([]string, 0, len(o.WaitSeries))
for id := range o.WaitSeries {
ids = append(ids, id)
}
sort.Strings(ids)
out := make([]string, 0, len(ids))
for _, id := range ids {
out = append(out, id+":["+strings.Join(o.WaitSeries[id], "->")+"]")
}
return join(out)
}
// schedulerArmInvalid refuses an arm that did not reach its own refusal. Each
// arm exists to isolate one cause, and one that reached a different cause was
// intercepted by a check above it — reporting that as this arm's answer is how
// a probe measures the wrong ceiling and calls it evidence.
func schedulerArmInvalid(arm string, before Observation) string {
want := string(wantedCause(arm))
if before.WaitCauses[want] == 0 {
return "no item was refused under " + want + "; the arm reached " + orNone(causeSummary(before))
}
for cause := range before.WaitCauses {
if cause != want {
return "an item was also refused under " + cause + ", so this arm does not isolate " + want
}
}
return ""
}
+327
View File
@@ -0,0 +1,327 @@
package main
import (
"context"
"encoding/json"
"fmt"
"regexp"
"slices"
"strconv"
"strings"
"sync"
"sync/atomic"
"reasonix/internal/contract/provider"
)
// newScripted builds the provider with its synchronisation channels open. The
// scheduler arms coordinate through them; every other arm never reads them.
func newScripted(arm string) *scripted {
return &scripted{arm: arm, held: make(chan struct{}), release: make(chan struct{})}
}
// probeModelRef is both the catalog ref and the model name boot is given, so
// syntheticEntryFromResolver matches it without touching any config file.
const probeModelRef = "probe/scripted"
// probeAltModelRef is a second identity the same scripted provider answers to.
// Without one, every layer of the model resolution collapses onto the same
// string and a probe cannot tell which layer produced it.
const probeAltModelRef = "probe/alt"
// scripted answers completions from a fixed script so the construct phase can
// establish host state without a network call. A request carrying no tools is a
// host-internal completion (the compaction summary); everything else is a turn.
type scripted struct {
// arm is which run this provider is scripting. Only the fan-out arms read
// it: the pair fleet is the same dispatch either way, and the arm decides
// whether its second item finishes or is still executing at death.
arm string
turnCalls atomic.Int64
// retodo latches the one reply that replaces the task list. The sentinel
// stays in history after the call, so without it every later round would
// rewrite the list again.
retodo atomic.Bool
// asked latches the one reply that opens a question and, in the same round,
// tries to write. Calling ask ends the round, so the write is the effect a
// decision is holding back.
asked atomic.Bool
// Which fleets have been opened. A fleet's aggregate returns into the same
// round that dispatched it, so without a latch the reply that follows reads
// its own sentinel off the still-current user turn and dispatches again.
fleets dispatched
// opened says the arm's own fan-out has gone out, whichever sentinel opened
// it: the terminal and derive arms then hold the turn rather than replying,
// so the process still dies inside one.
opened atomic.Bool
// The scheduler arms dispatch two fleets and must not race them. held is
// closed by the holder's own child, which only runs once its slot is
// granted; release lets the arm free capacity while a refusal stands.
holding sync.Once
released sync.Once
held chan struct{}
release chan struct{}
}
// dispatched is the set of fleets already opened, keyed by the sentinel that
// opened each. One named state rather than a latch per fleet: they share a
// lifetime and a meaning, and every new fan-out was adding another flag.
type dispatched struct {
mu sync.Mutex
seen map[string]bool
}
// held reports whether something already recorded this name, without claiming
// it. The lone-delegation arms read it to ask a negative: that a child the
// scheduler should be holding back never reached the provider at all.
func (d *dispatched) held(name string) bool {
d.mu.Lock()
defer d.mu.Unlock()
return d.seen[name]
}
// first reports whether this is the sentinel's first dispatch, and records it.
func (d *dispatched) first(sentinel string) bool {
d.mu.Lock()
defer d.mu.Unlock()
if d.seen[sentinel] {
return false
}
if d.seen == nil {
d.seen = map[string]bool{}
}
d.seen[sentinel] = true
return true
}
// releaseOnce frees the child waiting on release. Idempotent: an arm may reach
// the moment to release through more than one path.
func (s *scripted) releaseOnce() { s.released.Do(func() { close(s.release) }) }
func (s *scripted) Name() string { return "probe" }
func (s *scripted) Stream(ctx context.Context, req provider.Request) (<-chan provider.Chunk, error) {
ch := make(chan provider.Chunk, 8)
go func() {
defer close(ch)
for _, c := range s.script(ctx, req) {
select {
case ch <- c:
case <-ctx.Done():
return
}
}
}()
return ch, nil
}
func (s *scripted) script(ctx context.Context, req provider.Request) []provider.Chunk {
if len(req.Tools) == 0 {
return append(text(summaryBody), done())
}
// A delegated run answers first: the parent's transcript quotes every fleet
// argument, so any later branch that scanned the conversation would make
// the parent answer as one of its own children.
if sentinel := childSentinel(req); sentinel != "" {
return s.childScript(ctx, sentinel)
}
if chunks, ok := s.fanOut(req); ok {
return chunks
}
if s.opens(req, sleepSentinel) {
return append(sleepCall(), done())
}
if s.opensRun(req, taskSentinel) {
if queuedTaskArm(s.arm) {
// The holder's own child says when the ceiling is occupied.
// Dispatching before that lets the delegation win the race and start.
<-s.held
}
return append(loneTaskCall(s.arm), done())
}
if s.opensRun(req, ephemeralSentinel) {
return append(ephemeralCall(s.arm), done())
}
if s.opensRun(req, skillSentinel) {
if queuedSkillArm(s.arm) {
// The holder's own child says when the ceiling is occupied.
// Dispatching before that lets the skill win the race and start.
<-s.held
}
return append(skillCall(s.arm), done())
}
// The one arm whose turn has to close signs the host's steps off first, one
// per round, because a turn that called a tool cannot deliver while the list
// is open. The latch is read last: a round that could not answer keeps it.
if closesItsTurn(s.arm) && s.opened.Load() && hasTool(req.Tools, "complete_step") {
for _, id := range probeStepIDs() {
if s.fleets.first("sign-" + id) {
return append(signStep(id), done())
}
}
}
// A terminal arm needs its fan-out to finish and its turn to stay open: a
// closed turn would settle the marker, and the process would no longer be
// dying inside one.
if (terminalArm(s.arm) || deriveArm(s.arm)) && s.opened.Load() {
<-ctx.Done()
return nil
}
if s.turnCalls.Add(1) == 1 && hasTool(req.Tools, "todo_write") {
return append(todoCall(firstTodos()), done())
}
if hasTool(req.Tools, "todo_write") && requestMentions(req, retodoSentinel) && !s.retodo.Swap(true) {
return append(todoCall(secondTodos()), done())
}
if requestMentions(req, askSentinel) && !s.asked.Swap(true) {
return append(askAndWrite(), done())
}
return append(text(echoMarker(req)+turnBody), done())
}
// A turn is tagged twice: the user side by markerPattern, the assistant reply
// by echoPattern. A fold folds assistant work first, so a one-sided tag would
// let the surviving user turn hide the reply that was dropped beside it.
var (
markerPattern = regexp.MustCompile(`PROBE-MARK-\d+`)
echoPattern = regexp.MustCompile(`PROBE-ECHO-\d+`)
)
func marker(n int) string { return fmt.Sprintf("PROBE-MARK-%03d", n) }
// markersIn returns the tags a message set still shows, in order.
func markersIn(texts []string, pat *regexp.Regexp) []string {
var out []string
for _, t := range texts {
out = append(out, pat.FindAllString(t, -1)...)
}
return out
}
// echoMarker answers a turn with the assistant-side tag for the same number.
func echoMarker(req provider.Request) string {
for _, m := range slices.Backward(req.Messages) {
found := markerPattern.FindAllString(m.Content, -1)
if len(found) == 0 {
continue
}
return "PROBE-ECHO-" + found[len(found)-1][len("PROBE-MARK-"):] + " reply. "
}
return ""
}
func hasTool(schemas []provider.ToolSchema, name string) bool {
for _, t := range schemas {
if t.Name == name {
return true
}
}
return false
}
func text(s string) []provider.Chunk {
return []provider.Chunk{{Type: provider.ChunkText, Text: s}}
}
func done() provider.Chunk { return provider.Chunk{Type: provider.ChunkDone} }
// todoCall writes the list the resume phase looks for. The step ids are the
// point: they are the host identity a completion has to cite. A first write
// may not open with a completed item — the host rejects that shape, and a
// rejected call would leave the row unmeasured rather than answered.
func todoCall(todos []map[string]any) []provider.Chunk {
args, _ := json.Marshal(map[string]any{"todos": todos})
return []provider.Chunk{{
Type: provider.ChunkToolCall,
ToolCall: &provider.ToolCall{ID: "probe_todo_" + strconv.Itoa(len(todos)), Name: "todo_write", Arguments: string(args)},
}}
}
// retodoSentinel asks the scripted reply to replace the task list, so a probe
// can move host step identity from one set of ids to another.
const retodoSentinel = "PROBE-RETODO"
// askSentinel asks for a question the host must put to a person, alongside the
// write that question is holding back.
const (
askSentinel = "PROBE-ASK"
deferredEffect = "probe-deferred-effect.txt"
deferredContents = "this write was not held back by the decision"
)
// askAndWrite opens a question and asks to write in the same round. The round
// ends at the question, so the write must not run — not while the answer is
// pending, and not because a later process reloaded the session.
func askAndWrite() []provider.Chunk {
ask, _ := json.Marshal(map[string]any{"questions": []map[string]any{{
"header": "Boundary",
"question": "Which side of the fold should the probe measure?",
"reason": "user_decision",
"options": []map[string]any{
{"label": "Below the fold", "description": "removes folded history"},
{"label": "Above the fold", "description": "leaves coverage intact"},
},
}}})
write, _ := json.Marshal(map[string]string{"path": deferredEffect, "content": deferredContents})
return []provider.Chunk{
{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{ID: "probe_ask_1", Name: "ask", Arguments: string(ask)}},
{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{ID: "probe_write_1", Name: "write_file", Arguments: string(write)}},
}
}
func requestMentions(req provider.Request, needle string) bool {
for _, m := range slices.Backward(req.Messages) {
if strings.Contains(m.Content, needle) {
return true
}
}
return false
}
func firstTodos() []map[string]any {
return []map[string]any{
{"content": "Establish probe state", "status": "in_progress", "activeForm": "Establishing probe state", "step_id": "probe_step_01"},
{"content": "Cross the process boundary", "status": "pending", "step_id": "probe_step_02"},
{"content": "Report the resurrection matrix", "status": "pending", "step_id": "probe_step_03"},
}
}
// secondTodos replaces the pending work while the in_progress item stays: the
// host refuses to drop an item that is in flight, so this is what a list
// rewrite actually looks like. The two new ids are what a stale note misses.
func secondTodos() []map[string]any {
return []map[string]any{
{"content": "Establish probe state", "status": "in_progress", "activeForm": "Establishing probe state", "step_id": "probe_step_01"},
{"content": "Split the frozen body", "status": "pending", "step_id": "probe_step_21"},
{"content": "Hold the ordering contract", "status": "pending", "step_id": "probe_step_22"},
}
}
// turnBody is fat on purpose: the fold keeps a recent verbatim tail, so thin
// replies leave no region between the pinned head and that tail.
var turnBody = "Probe state established. " + strings.Repeat("The probe records deterministic filler so the fold has material to work on. ", 60)
var summaryBody = strings.Join([]string{
"## Standing facts & constraints",
"The runtime-resume probe is measuring what survives a process boundary.",
"",
"## Goal",
"Establish host state, exit, and observe what a new process can still prove.",
}, "\n")
func resolverFor(prov *scripted) *provider.StaticResolver {
return &provider.StaticResolver{
Descriptors: []provider.Descriptor{{
Ref: probeModelRef, DisplayName: "probe", Model: "scripted",
ContextWindow: 128_000, Tools: true,
}, {
Ref: probeAltModelRef, DisplayName: "probe-alt", Model: "scripted-alt",
ContextWindow: 128_000, Tools: true,
}},
Providers: map[string]provider.Provider{probeModelRef: prov, probeAltModelRef: prov},
}
}
func errUnexpected(what string, got any) error {
return fmt.Errorf("probe could not establish %s (got %v); the arm measures nothing and is reported invalid", what, got)
}
@@ -0,0 +1,38 @@
package main
import (
"strings"
"testing"
)
// Sentinels are matched by substring, so one contained in another dispatches on
// the same prompt: the identity arm silently ran the cancellation fan-out for as
// long as PROBE-PARALLEL sat inside PROBE-PARALLEL-IDENTITY, and its rows read a
// scenario nobody wrote. Care did not catch it — the child sentinels carry a
// comment about exactly this hazard and the parent ones had no guard at all.
func TestNoSentinelContainsAnother(t *testing.T) {
sentinels := map[string]string{
"fleetWarm": fleetWarmSentinel, "fleetMixed": fleetMixedSentinel,
"fleetPair": fleetPairSentinel, "fleetHolder": fleetHolderSentinel,
"fleetRefused": fleetRefusedSentinel, "fleetTerminal": fleetTerminalSentinel,
"fleetOutcomes": fleetOutcomesSentinel, "fleetDerive": fleetDeriveSentinel,
"fleetIdentity": fleetIdentitySentinel, "fleetActive": fleetActiveSentinel,
"fleetSettled": fleetSettledSentinel, "fleetLive": fleetLiveSentinel,
"parallel": parallelSentinel, "parallelIdentity": parallelIdentity,
"task": taskSentinel, "sleep": sleepSentinel,
"ask": askSentinel, "retodo": retodoSentinel,
"childDone": childDone, "childHang": childHang, "childFail": childFail,
"childHold": childHold, "childRelease": childRelease, "childFailLate": childFailLate,
}
for outerName, outer := range sentinels {
for innerName, inner := range sentinels {
if outerName == innerName {
continue
}
if strings.Contains(outer, inner) {
t.Errorf("%s (%q) contains %s (%q): a prompt naming the first dispatches the second too",
outerName, outer, innerName, inner)
}
}
}
}
+694
View File
@@ -0,0 +1,694 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"reasonix/internal/state/sessionstore"
"sort"
"strings"
"time"
"reasonix/internal/contract/provider"
"reasonix/internal/ext/skill"
"reasonix/internal/session/control"
)
// The skill-runner classification arms. They ask what kind of execution a
// runAs=subagent skill is, because the answer decides whether it belongs to the
// delegation class the task and fan-out arms established or is an execution
// class of its own. Nothing here is a fix: a classification that guessed would
// be answered by whichever plumbing happened to exist.
const (
// armSkillCompleted lets a skill finish and the turn close, then asks what
// each layer kept. It runs both entry points — the tool a model calls and
// the controller's own — because their lineage differs by design and a row
// that read one for the other would report intent as a defect.
armSkillCompleted = "skill-completed"
// armSkillRunning kills the process with the child mid-execution: the death
// every other delegation arm takes.
armSkillRunning = "skill-running"
// armSkillQueued fills the ceiling first. It is the arm that says whether a
// skill belongs to the same admission domain as everything else, which is a
// different question from whether it leaves the same records.
armSkillQueued = "skill-queued"
// armSkillCancel stops a running skill through the turn that admitted it, so
// the stop has the owner a person's Stop has.
armSkillCancel = "skill-cancel"
// The read-only entry point is a separate surface, not a lighter version of
// the one above: it prepares no transcript and refuses continuation, so
// asking it for durability would be demanding it break its own contract.
// These three ask what it is instead.
armReadOnlySkillCompleted = "readonly-skill-completed"
armReadOnlySkillRunning = "readonly-skill-running"
armReadOnlySkillQueued = "readonly-skill-queued"
)
func skillArm(name string) bool {
switch name {
case armSkillCompleted, armSkillRunning, armSkillQueued, armSkillCancel:
return true
}
return readOnlySkillArm(name)
}
func readOnlySkillArm(name string) bool {
switch name {
case armReadOnlySkillCompleted, armReadOnlySkillRunning, armReadOnlySkillQueued:
return true
}
return false
}
func queuedSkillArm(name string) bool {
return name == armSkillQueued || name == armReadOnlySkillQueued
}
// runningSkillArm names the arms that die or are stopped with the child inside
// the provider, which is the only state a durable record could be missing from.
func runningSkillArm(name string) bool {
return name == armSkillRunning || name == armSkillCancel || name == armReadOnlySkillRunning
}
const (
skillSentinel = "PROBE-SKILL"
// skillCallID is the tool call the model makes. It is the identity the
// store would join to, so the arm names it rather than letting the round
// pick one.
skillCallID = "probe_skill"
// probeSkillName is the skill on disk. Not a built-in: those carry their own
// read-only promises and review contracts, and the arm needs a plain writer
// so the scheduler sees a claim rather than a reader.
probeSkillName = "probe-worker"
// hostSkillTask is what the controller's own entry point is asked to do. It
// completes, because that arm compares two lineages rather than two deaths.
hostSkillTask = childDone + " host-initiated skill"
)
// writeProbeSkill puts a runAs=subagent skill in the workspace, where a project
// skill lives. It is a writer: a read-only one would clear the writer and claim
// checks by not being subject to them, and the arm could not say whether a skill
// is admitted the way every other writer is.
func writeProbeSkill(workspace string) error {
dir := filepath.Join(workspace, ".reasonix", skill.SkillsDirname, probeSkillName)
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
body := "---\nname: " + probeSkillName + "\n" +
"description: The probe's own subagent, which answers whatever task it is handed.\n" +
"runAs: subagent\n---\n\nAnswer the task you are given and stop.\n"
return os.WriteFile(filepath.Join(dir, skill.SkillFile), []byte(body), 0o644)
}
// skillCall is the dispatch under measurement, made through the tool a model
// calls. The read-only arms enter through the other tool, which is a different
// runner rather than a flag on this one.
func skillCall(arm string) []provider.Chunk {
name := "run_skill"
if readOnlySkillArm(arm) {
name = "read_only_skill"
}
task := skillChildSentinel(arm) + " skill body"
args, _ := json.Marshal(map[string]any{"name": probeSkillName, "arguments": task})
return []provider.Chunk{{
Type: provider.ChunkToolCall,
ToolCall: &provider.ToolCall{ID: skillCallID, Name: name, Arguments: string(args)},
}}
}
// skillChildSentinel is how this arm's child behaves. The completed arms need one
// that finishes; every other arm needs one that holds until the process dies.
func skillChildSentinel(arm string) string {
if arm == armSkillCompleted || arm == armReadOnlySkillCompleted {
return childDone
}
return childHang
}
// skillSentinels is what this arm's one prompt carries. The queued arms name the
// holder as well: its ceiling has to be occupied before the skill asks for a
// slot, or the refusal they are built to record never happens.
func skillSentinels(arm string) string {
if queuedSkillArm(arm) {
return fleetHolderSentinel + " " + skillSentinel
}
return skillSentinel
}
// runSkillClassConstruct drives one skill and dies where this arm is about. The
// completed arms let the turn close and then run the controller's own entry
// point, so one death holds both lineages.
func runSkillClassConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
prompt := fanOutTurn(turn+1, skillSentinels(arm))
completed := arm == armSkillCompleted || arm == armReadOnlySkillCompleted
switch {
case completed:
if err := ctrl.Run(ctx, prompt); err != nil {
return fmt.Errorf("skill turn: %w", err)
}
if err := runHostInitiatedSkill(ctx, ctrl, arm); err != nil {
return err
}
case arm == armSkillCancel:
// A stop belongs to whoever admitted the turn, so the arm that is going
// to be stopped is admitted the way a person's input is.
ctrl.Send(prompt)
if err := waitForSkill(sink, prov, arm); err != nil {
return err
}
default:
go func() { _ = ctrl.Run(ctx, prompt) }()
if err := waitForSkill(sink, prov, arm); err != nil {
return err
}
}
stop := ""
if arm == armSkillCancel {
stop = stopSkill(ctrl, prov, arm)
}
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
child := skillChildSentinel(arm)
obs.Progress[probeChildEntered] = []string{fmt.Sprint(prov.fleets.held(childEntered(child)))}
obs.Progress[probeSkillChildLeft] = []string{fmt.Sprint(prov.fleets.held(childLeft(child)))}
obs.Progress[probeSkillChildCtxDone] = []string{fmt.Sprint(prov.fleets.held(childCtxDone(child)))}
obs.Progress[probeSkillStanding] = []string{skillStanding(sink, prov)}
if stop != "" {
obs.Progress[probeStopReached] = []string{stop}
}
if err := writeObservation(root, obs); err != nil {
return err
}
os.Exit(0)
return nil
}
// runHostInitiatedSkill runs the same skill through the controller's own entry
// point, which is what a slash invocation reaches. Its lineage is deliberately
// different — the runner drops the call id for a host-initiated run — so the
// identity row has both to compare rather than one to judge.
func runHostInitiatedSkill(ctx context.Context, ctrl *control.Controller, arm string) error {
if _, err := ctrl.RunSubagentProfile(ctx, probeSkillName, hostSkillTask, readOnlySkillArm(arm)); err != nil {
return fmt.Errorf("host-initiated skill: %w", err)
}
return nil
}
// The records a skill's child leaves. They ride the observation because only the
// scripted provider knows them, and a store row read without them would be
// asking about work nothing proves ran.
const (
probeSkillChildLeft = "probe:skill-child-left"
probeSkillChildCtxDone = "probe:skill-child-ctxdone"
// probeSkillStanding is where the call itself stood: dispatched, answered,
// and whether the ceiling was occupied. A refusal has no event of its own,
// so this is the only positive evidence the admission row has.
probeSkillStanding = "probe:skill-standing"
)
// waitForSkill blocks until the skill stands where this arm dies. A refusal is a
// state nothing announces, so the queued arms establish it from both sides — the
// call went out and has not answered, the ceiling is occupied, and the skill's
// own child never arrived — then hold still and ask again, because a child that
// was merely slow to start would have reached the provider by then.
func waitForSkill(sink *graphSink, prov *scripted, arm string) error {
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
if !queuedSkillArm(arm) {
if prov.fleets.held(childEntered(childHang)) {
return nil
}
} else if skillHeldBack(sink, prov) {
time.Sleep(2 * time.Second)
if !skillHeldBack(sink, prov) {
return errUnexpected("a skill the scheduler was still holding back",
"its child ran or its call answered")
}
return nil
}
time.Sleep(50 * time.Millisecond)
}
return errUnexpected("a skill child "+skillWanted(arm), skillStanding(sink, prov))
}
// skillHeldBack is the refusal expressed as facts rather than as an absence: the
// ceiling is occupied, the call is out, it has not answered, and no child of its
// own has reached the provider.
func skillHeldBack(sink *graphSink, prov *scripted) bool {
_, dispatched := sink.toolDispatched(skillCallID)
_, answered := sink.toolResult(skillCallID)
return prov.fleets.held(childEntered(childHold)) && dispatched && !answered &&
!prov.fleets.held(childEntered(childHang))
}
func skillStanding(sink *graphSink, prov *scripted) string {
_, dispatched := sink.toolDispatched(skillCallID)
_, answered := sink.toolResult(skillCallID)
return fmt.Sprintf("ceiling=%t dispatched=%t answered=%t child=%t",
prov.fleets.held(childEntered(childHold)), dispatched, answered,
prov.fleets.held(childEntered(childHang)))
}
func skillWanted(arm string) string {
if queuedSkillArm(arm) {
return "held back by a full ceiling"
}
return "inside the provider"
}
// stopSkill stops the work the way a person does — the turn's own cancel — and
// waits to see what that reaches. Nothing is demanded: whether a stop arrives is
// what the arm measures, and an arm that failed when it did not would report its
// own finding as a broken premise.
func stopSkill(ctrl *control.Controller, prov *scripted, arm string) string {
stopped := time.Now()
ctrl.Cancel()
deadline := time.Now().Add(20 * time.Second)
for time.Now().Before(deadline) && ctrl.Running() {
time.Sleep(50 * time.Millisecond)
}
live := "gone"
if prov.fleets.held(childEntered(childHang)) && !prov.fleets.held(childLeft(childHang)) {
live = "still running"
}
return fmt.Sprintf("turn=%s work=%s ctx=%t after=%s", turnStanding(ctrl, arm), live,
prov.fleets.held(childCtxDone(childHang)), time.Since(stopped).Round(time.Second))
}
// skillClassRows are the seven dimensions, one row each. They are separate
// because a skill can share one domain and not another — the whole reason this
// is a classification and not a comparison.
func skillClassRows(arm string, before, after Observation) []row {
return []row{
skillAdmissionRow(arm, before),
skillGraphRow(before),
skillJournalRow(before, after),
skillStoreRow(before, after),
skillCancelRow(arm, before),
skillRestartRow(arm, before, after),
skillIdentityRow(arm, before, after),
skillClassVerdictRow(arm, before, after),
}
}
const skillAuthority = "the scheduler, read through what the child did"
// skillAdmissionRow is the scheduler dimension. There is no refusal hook on a
// skill's acquire, so the refusal is read the only way it can be: the ceiling was
// proven occupied and the child never arrived, against the sibling arm where the
// ceiling was free and it did.
func skillAdmissionRow(arm string, before Observation) row {
entered := firstProgress(before, probeChildEntered) == "true"
answer := "admitted: the child reached the provider"
verdict := verdictHolds
switch {
case queuedSkillArm(arm) && !entered:
answer = "refused: the ceiling was full and the child never arrived"
case queuedSkillArm(arm):
answer = "not held back: the child ran with the ceiling full"
verdict = verdictViolated
case !entered:
answer = "the child never arrived with capacity free"
verdict = verdictNotMeasured
}
return row{
Semantic: "does it ask the session scheduler", Authority: skillAuthority,
Artifact: "none observed", Reconstruction: "in-process only",
Before: answer + " [" + firstProgress(before, probeSkillStanding) + "]",
After: "—", Verdict: verdict,
}
}
// skillGraphRow is the live picture. A skill that never draws one is invisible
// to every frontend while it runs, which is a product fact rather than a
// durability one and is why it is its own row.
func skillGraphRow(before Observation) row {
var drawn []string
for _, n := range before.Graph.Nodes {
if strings.HasPrefix(n.ID, skillCallID) {
drawn = append(drawn, n.ID+":"+string(n.State))
}
}
sort.Strings(drawn)
// The whole graph's size rides along: "the filter found nothing" and "there
// was nothing to find" are different readings, and only the second is what
// this row means to report.
return row{
Semantic: "does the live graph hold it", Authority: liveAuthority,
Artifact: "none observed", Reconstruction: "in-process only",
Before: fmt.Sprintf("%s (graph held %d node(s))", orNone(join(drawn)), len(before.Graph.Nodes)),
After: "—", Verdict: presenceVerdict(len(drawn) > 0),
}
}
// presenceVerdict reports what was found without calling absence a defect. What
// a skill owes each layer is the question; a row that answered it here would be
// asserting the classification it exists to establish.
func presenceVerdict(present bool) string {
if present {
return verdictHolds
}
return verdictAbsent
}
const verdictAbsent = "absent"
// skillJournalRow reports what the journal holds. It does not go through the
// ordinary value comparison: two empty columns are equal, and calling that
// persisted would report an execution nothing recorded as durably recorded —
// which is the substitution this whole probe exists to refuse.
func skillJournalRow(before, after Observation) row {
return presenceRow("does the journal record it", journalAuthority, "<stem>.execution.jsonl",
"ExecutionHistory over the whole session", skillExecutions(before), skillExecutions(after))
}
// presenceRow answers "is anything here" before it answers "is it the same".
// Absence on both sides is absence, never agreement.
func presenceRow(semantic, authority, artifact, reconstruction string, before, after []string) row {
r := row{Semantic: semantic, Authority: authority, Artifact: artifact,
Reconstruction: reconstruction, Before: orNone(join(before)), After: orNone(join(after))}
switch {
case len(before) == 0 && len(after) == 0:
r.Verdict = verdictAbsent
case len(after) == 0:
r.Verdict = verdictLost
case r.Before == r.After:
r.Verdict = verdictPersisted
default:
r.Verdict = verdictLossy
}
return r
}
// skillExecutions are the journal entries this arm's skill could be recorded
// under. The whole journal is read rather than a prefix: a skill records nothing
// today, so a filter written around an id it never wrote would report absence by
// construction.
func skillExecutions(o Observation) []string {
var out []string
for _, e := range o.Executions {
if strings.HasPrefix(e.ID, skillCallID) || strings.Contains(e.Name, probeSkillName) {
out = append(out, e.ID)
}
}
sort.Strings(out)
return out
}
// skillStoreRow is the durable half. The two entry points differ here by design:
// one prepares a transcript, the other refuses to.
func skillStoreRow(before, after Observation) row {
return presenceRow("what the store holds for it", storeAuthority, "subagents/<ref>.meta.json",
"ListSubagentsByParent over the transcript stem",
skillChildren(before), skillChildren(after))
}
// skillChildren are the store's records of skill children, by the kind the
// runner declared rather than by a name match: kind is what the store was told
// this execution is.
func skillChildren(o Observation) []string {
var out []string
for _, f := range o.Children.Facts {
if f.Kind == "skill" {
out = append(out, f.Name+"/"+orDash(f.ParentToolCallID)+"="+f.Status)
}
}
sort.Strings(out)
return out
}
// skillCancelRow is what a stop reached. Three facts, kept apart: a context that
// closed and a call that returned are different events, and work whose caller
// stopped waiting leaves the same absence behind as work that was cancelled.
func skillCancelRow(arm string, before Observation) row {
if arm != armSkillCancel {
return row{
Semantic: "what a stop reaches", Authority: "the turn's own cancel",
Artifact: "none observed", Reconstruction: "in-process only",
Before: "not stopped in this arm", After: "—", Verdict: verdictNotMeasured,
}
}
return row{
Semantic: "what a stop reaches", Authority: "the turn's own cancel",
Artifact: "subagents/<ref>.meta.json", Reconstruction: "read at one instant, after the record closed",
Before: fmt.Sprintf("%s store=%s", firstProgress(before, probeStopReached),
orNone(join(skillChildren(before)))),
After: "—", Verdict: skillCancelVerdict(before),
}
}
// skillCancelVerdict asks whether the terminal the store kept is the one that
// happened. The cause is observed, not read off a message: the child's own
// context closed, which is what a cancellation does and what a failure does not,
// so a record that files it as failed is naming the wrong ending.
func skillCancelVerdict(before Observation) string {
cancelled := firstProgress(before, probeSkillChildCtxDone) == "true"
terminal := ""
for _, f := range before.Children.Facts {
if f.Kind == "skill" {
terminal = f.Status
}
}
switch {
case terminal == "":
return verdictNotMeasured
case cancelled && terminal != string(sessionstore.SubagentCancelled):
return verdictViolated
default:
return verdictHolds
}
}
// skillRestartRow is what the next process can say. It is separate from the
// store row because "a record survives" and "the execution is explicable" are
// different claims: a completed child on disk explains nothing about a run that
// was cut.
func skillRestartRow(arm string, before, after Observation) row {
stored, rebuilt, cut := skillChildren(after), skillRebuiltNodes(after), skillInterruptions(after)
answer := fmt.Sprintf("store=%s rebuilt=%s cut=%s",
orNone(join(stored)), orNone(join(rebuilt)), orNone(join(cut)))
return row{
Semantic: "what a restart can say about it", Authority: rebuildAuthority,
Artifact: "<stem>.execution.jsonl + subagents/<ref>.meta.json",
Reconstruction: "execgraph.Rebuild over the journal, and the store beside it",
Before: skillAtDeath(arm, before), After: answer,
Verdict: skillRestartVerdict(stored, rebuilt, cut),
}
}
// skillInterruptions are the interruptions that name this skill. The whole list
// is wider than the arm: a queued arm fills its ceiling with a fan-out, and a
// fan-out records everything — reading its interruption as the skill's is how an
// arm reports a gap as closed by evidence belonging to something else.
func skillInterruptions(o Observation) []string {
var out []string
for _, e := range o.InterruptedExecutions {
if strings.HasPrefix(e.ID, skillCallID) || strings.Contains(e.Name, probeSkillName) {
out = append(out, e.ID)
}
}
sort.Strings(out)
return out
}
// skillRestartVerdict keeps two claims apart. That the store remembers a child
// finished is not that a restart can reconstruct the delegated execution: the
// first is a record, the second is provenance, and an arm that folded them would
// report a run nothing can place as recovered.
func skillRestartVerdict(stored, rebuilt, cut []string) string {
switch {
case len(rebuilt) > 0 || len(cut) > 0:
return verdictHolds
case len(stored) > 0:
return verdictRecordOnly
default:
return verdictAbsent
}
}
// verdictRecordOnly is a child the store kept for an execution no reconstruction
// names.
const verdictRecordOnly = "record-only"
// skillRebuiltNodes are the nodes a restart's rebuild draws for this skill.
func skillRebuiltNodes(o Observation) []string {
var out []string
for _, n := range rebuiltGraph(o).Graph.Nodes {
if strings.HasPrefix(n.ID, skillCallID) || strings.Contains(n.Label, probeSkillName) {
out = append(out, n.ID+":"+string(n.State))
}
}
sort.Strings(out)
return out
}
func skillAtDeath(arm string, before Observation) string {
switch {
case queuedSkillArm(arm):
return "held back, never started"
case firstProgress(before, probeChildEntered) != "true":
return "never reached the provider"
case skillFinished(before):
return "ran and returned"
default:
return "executing inside the provider"
}
}
// skillIdentityRow is the seventh dimension, and it reads the two entry points
// apart. A model-invoked skill has a call to join to; the controller's own entry
// point drops that id on purpose, so an empty one there is top-level provenance
// rather than a lost join — reading them together is what stops a correct value
// being reported as a defect.
func skillIdentityRow(arm string, before, after Observation) row {
var got []string
for _, f := range after.Children.Facts {
if f.Kind != "skill" {
continue
}
got = append(got, fmt.Sprintf("%s from=%s model=%s effort=%s",
f.Name, orDash(f.ParentToolCallID), orDash(f.Model), orDash(f.Effort)))
}
sort.Strings(got)
return row{
Semantic: "what identity the record carries", Authority: storeAuthority,
Artifact: "subagents/<ref>.meta.json",
Reconstruction: "the call the model made, against the entry point that made it",
Before: skillLineageExpected(arm), After: orNone(join(got)),
Verdict: skillIdentityVerdict(after),
}
}
func skillLineageExpected(arm string) string {
if arm == armSkillCompleted || arm == armReadOnlySkillCompleted {
return "model-invoked joins the execution under " + skillCallID + "; host-initiated names none"
}
return "model-invoked joins the execution opened under " + skillCallID
}
// skillIdentityVerdict holds when the model-invoked record joins the execution
// the journal opened under the call the model made. The join is checked, never
// one spelling of it: a record's parent is the execution's own identity, so
// comparing it to the bare call id would fail the chain that makes them
// joinable. A host-started run has no call to hang under, so it is not held.
func skillIdentityVerdict(after Observation) string {
opened := map[string]bool{}
for _, id := range skillExecutions(after) {
opened[id] = true
}
joined, records := false, 0
for _, f := range after.Children.Facts {
if f.Kind != "skill" {
continue
}
records++
if opened[f.ParentToolCallID] && strings.HasPrefix(f.ParentToolCallID, skillCallID) {
joined = true
}
}
switch {
case records == 0:
return verdictNotMeasured
case joined:
return verdictHolds
default:
return verdictViolated
}
}
// skillClassVerdictRow is the arm's answer in one line, and it is a conjunction
// rather than a comparison: an execution the scheduler admitted, whose child
// reached the provider, that no live picture, no journal and no active store
// record names, is lost while it is happening — not merely unrecorded.
func skillClassVerdictRow(arm string, before, after Observation) row {
entered := firstProgress(before, probeChildEntered) == "true"
graph := len(skillGraphNodes(before)) > 0
journal := len(skillExecutions(before)) > 0
stored := len(skillChildren(before)) > 0
answer, verdict := "not this arm's question", verdictNotMeasured
switch {
case queuedSkillArm(arm):
answer, verdict = skillQueuedNegative(before)
case readOnlySkillArm(arm) && runningSkillArm(arm) && entered:
answer, verdict = readOnlyClass(graph, journal, stored)
case runningSkillArm(arm) && entered:
if graph || journal || stored {
answer, verdict = fmt.Sprintf("named by graph=%t journal=%t store=%t", graph, journal, stored), verdictHolds
} else {
answer, verdict = "LOST-SILENT: executing, and named by no layer", verdictViolated
}
case runningSkillArm(arm):
answer = "the child never reached the provider"
}
return row{
Semantic: "an execution in flight, named by nothing", Authority: crossAuthority,
Artifact: "none, which is the finding",
Reconstruction: "scheduler admitted + child entered + no graph, journal or store record",
Before: answer, After: "—", Verdict: verdict,
}
}
// readOnlyClass classifies the entry point that promised no durable side
// effects. Absence there is its contract kept, not a gap — but it still holds a
// scheduler slot, so whether anything draws it while it does is a separate
// answer, and a durable record would be the contract broken the other way.
func readOnlyClass(graph, journal, stored bool) (string, string) {
switch {
case journal || stored:
return "an entry point that promised no durable side effects left one", verdictViolated
case graph:
return "ephemeral by contract, and drawn while it holds a slot", verdictEphemeralSeen
default:
return "ephemeral by contract, and unseen while it holds a slot", verdictEphemeralUnseen
}
}
// The two readings of an execution that keeps nothing. Neither is a defect: what
// separates them is whether anyone can see the slot being held.
const (
verdictEphemeralSeen = "ephemeral-seen"
verdictEphemeralUnseen = "ephemeral-unseen"
)
// skillQueuedNegative is the control the running row needs, and it is about the
// store alone. An opening is the orchestration saying work entered it, which a
// refused item did; a store record is a child execution artifact, which it has
// none of. Demanding silence from the journal too would forbid the very record
// that tells a restart the item was held back rather than lost.
func skillQueuedNegative(before Observation) (string, string) {
if len(skillChildren(before)) > 0 {
return "a child record for work the scheduler never admitted", verdictViolated
}
return "held back: opened and queued, with no child execution artifact", verdictHolds
}
func skillGraphNodes(o Observation) []string {
var out []string
for _, n := range o.Graph.Nodes {
if strings.HasPrefix(n.ID, skillCallID) {
out = append(out, n.ID)
}
}
return out
}
// skillArmInvalid reports the premise this arm could not establish. A skill that
// never reached the provider measures nothing about executions in flight, and a
// ceiling that was not occupied measures nothing about admission.
func skillArmInvalid(arm string, before Observation) string {
entered := firstProgress(before, probeChildEntered) == "true"
switch {
case queuedSkillArm(arm) && entered:
return "the skill ran while the ceiling was meant to be full"
case runningSkillArm(arm) && !entered:
return "the skill's child never reached the provider, so nothing was executing at the death"
case (arm == armSkillCompleted || arm == armReadOnlySkillCompleted) && !skillFinished(before):
return "no skill child finished, so the arm compares nothing"
}
return ""
}
func skillFinished(before Observation) bool {
return firstProgress(before, probeSkillChildLeft) == "true"
}
+69
View File
@@ -0,0 +1,69 @@
package main
import "fmt"
// successorRows ask what happens to a recorded interruption once work carries
// on. The derivation has no way out of "interrupted" today, so these rows are
// there to say what that costs rather than to grade it.
func successorRows(arm string, successor, final Observation) []row {
return []row{{
Semantic: "the interruption after work continued", Authority: "the adjudication log",
Artifact: "<stem>.adjudication.jsonl", Reconstruction: "still derived from an open record with no owner",
Before: interruptionState(successor), After: interruptionState(final), Verdict: stickiness(final),
}, {
Semantic: "what that turn's request carried", Authority: "control, projected per request",
Artifact: "none: derived, never stored", Reconstruction: "the interruption block, if one still stands",
Before: fmt.Sprintf("%d block(s)", successor.ModelSeesInterruption),
After: fmt.Sprintf("%d block(s)", final.ModelSeesInterruption), Verdict: verdictStable,
}, {
Semantic: "how the barrier ended", Authority: "the adjudication log",
Artifact: "<stem>.adjudication.jsonl", Reconstruction: "an edge is written only for a barrier a turn received",
Before: journalSummary(successor), After: journalSummary(final), Verdict: verdictStable,
}, {
Semantic: "the effect the dead decision held back", Authority: "the workspace",
Artifact: final.Deferred.MarkerPath, Reconstruction: "no successor turn may release it",
Before: ranOrNot(successor), After: ranOrNot(final), Verdict: heldBack(successor, final),
}}
}
func interruptionState(o Observation) string {
if len(o.Interrupted) == 0 {
return "settled"
}
return fmt.Sprintf("%d still interrupted", len(o.Interrupted))
}
// stickiness is the question these arms were built to ask: does an
// interruption outlive the work that replaced it? A successor that took it over
// leaves nothing behind for later requests to carry.
func stickiness(final Observation) string {
if len(final.Interrupted) == 0 && final.ModelSeesInterruption == 0 {
return "inherited"
}
if len(final.Interrupted) == 0 {
return "sticky context without an owner"
}
return "sticky"
}
func journalSummary(o Observation) string {
var out []string
for _, e := range o.Journal {
switch {
case e.Open():
out = append(out, e.ID+":open")
case e.SupersededBy != "":
out = append(out, e.ID+":"+e.Disposition+" by turn "+e.SupersededBy)
default:
out = append(out, e.ID+":"+e.Disposition)
}
}
return orNone(join(out))
}
func heldBack(successor, final Observation) string {
if successor.Deferred.Executed || final.Deferred.Executed {
return verdictViolated
}
return verdictHolds
}
+533
View File
@@ -0,0 +1,533 @@
package main
import (
"context"
"fmt"
"os"
"sort"
"strings"
"time"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/session/control"
)
// The terminal-disposition arms. SETTLED says an opening is no longer active
// and nothing more, so these ask what that costs: two items can share every
// durable fact and mean opposite things, and a restart that cannot tell them
// apart has lost the answer a dependent needs, not a label a picture wanted.
const (
// armTerminalAdopted reuses an earlier child's answer. It never runs and
// still holds a final answer, which is the one terminal that is answered
// without having executed.
armTerminalAdopted = "terminal-adopted"
// armTerminalSkippedDep is the same lifecycle with the opposite meaning: an
// item whose dependency failed, which never ran and holds no answer.
armTerminalSkippedDep = "terminal-skipped-dep"
// armTerminalCancelled is the other way an item ends without an answer. It
// was built to reach parallel_tasks' skipped branch and found it
// unreachable: running[i] is set when an item is dispatched, not when it is
// admitted, so a cancellation marks every unfinished item cancelled.
armTerminalCancelled = "terminal-cancelled"
// armTerminalContext asks whether a context edge has to be recorded at all,
// or already follows from the ordering and what the upstream answered.
armTerminalContext = "terminal-context"
// armChildTerminal compares the graph's terminal for an executed item
// against what the sub-agent store recorded for the same child. The store
// is the owner of "what happened to this child", so a distinction the graph
// draws and the store does not keep is one no reconstruction can recover.
armChildTerminal = "child-terminal"
// armIdentitySemantics asks what a node's model and effort mean. Two
// producers resolve them through different chains and the store resolves
// them again, so before a rebuild can be held to them, one of those has to
// be the fact the field carries.
armIdentitySemantics = "identity-semantics"
)
func terminalArm(name string) bool {
switch name {
case armTerminalAdopted, armTerminalSkippedDep, armTerminalCancelled, armTerminalContext, armChildTerminal, armIdentitySemantics:
return true
}
return false
}
// terminalSentinels is what this arm's one prompt carries. The adopted arm
// needs a completed reference first, so it names the warm-up fleet too.
func terminalSentinels(arm string) string {
switch arm {
case armTerminalAdopted:
return fleetWarmSentinel + " " + fleetMixedSentinel
case armTerminalCancelled:
return parallelSentinel
case armChildTerminal:
return fleetOutcomesSentinel
case armIdentitySemantics:
return fleetWarmSentinel + " " + fleetIdentitySentinel + " " + parallelIdentity
}
return fleetTerminalSentinel
}
// runTerminalConstruct lets the fan-out settle, then dies before the turn ends.
// The settling is the point: a fan-out publishes an item's skip only in its
// closing delta, so an arm that died mid-flight would never see one.
func runTerminalConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, turn int) error {
runCtx, cancelRun := context.WithCancel(ctx)
defer cancelRun()
go func() { _ = ctrl.Run(runCtx, fanOutTurn(turn+1, terminalSentinels(arm))) }()
if arm == armTerminalCancelled {
if err := cancelBeforeStart(sink, cancelRun); err != nil {
return err
}
}
if arm == armChildTerminal {
if err := cancelAfterOutcomes(sink, cancelRun); err != nil {
return err
}
}
if err := waitForTerminal(sink, arm); err != nil {
return err
}
if err := writeObservation(root, capture("construct", arm, bootSystem, ctrl, sink, root)); err != nil {
return err
}
os.Exit(0)
return nil
}
// cancelBeforeStart interrupts the group once one item is running and another
// is still waiting for a slot, which is the state parallel_tasks splits on. It
// cancels the turn's own context rather than calling Controller.Cancel: the
// synchronous headless Run does not pass through turn admission, so the gate
// that Cancel operates has nothing registered for it.
func cancelBeforeStart(sink *graphSink, cancelRun context.CancelFunc) error {
deadline := time.Now().Add(60 * time.Second)
for time.Now().Before(deadline) {
graph, _ := sink.snapshot()
if runningCount(graph) > 0 && len(queuedWorkers(graph)) > 0 {
cancelRun()
return nil
}
time.Sleep(50 * time.Millisecond)
}
graph, _ := sink.snapshot()
return errUnexpected("one item running and one still queued", waitStates(graph))
}
// cancelAfterOutcomes interrupts once one item has completed and another has
// failed, with a third still running. All three terminals then come from one
// group, so the store's record of them is compared under identical conditions.
func cancelAfterOutcomes(sink *graphSink, cancelRun context.CancelFunc) error {
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
graph, _ := sink.snapshot()
if len(nodesInState(graph, agentgraph.StateCompleted)) > 0 &&
len(nodesInState(graph, agentgraph.StateFailed)) > 0 &&
runningCount(graph) > 0 {
cancelRun()
return nil
}
time.Sleep(50 * time.Millisecond)
}
graph, _ := sink.snapshot()
return errUnexpected("one completed, one failed and one still running", waitStates(graph))
}
func runningCount(g agentgraph.Graph) int {
n := 0
for _, node := range g.Nodes {
if node.Kind != agentgraph.KindGroup && node.State == agentgraph.StateRunning {
n++
}
}
return n
}
// wantedTerminal is the disposition this arm exists to produce.
func wantedTerminal(arm string) agentgraph.NodeState {
if arm == armTerminalAdopted {
return agentgraph.StateAdopted
}
if arm == armTerminalContext {
return agentgraph.StateCompleted
}
if arm == armTerminalCancelled || arm == armChildTerminal {
return agentgraph.StateCancelled
}
if arm == armIdentitySemantics {
return agentgraph.StateAdopted
}
return agentgraph.StateSkipped
}
// waitForTerminal blocks until the graph shows this arm's disposition. The
// identity arm needs more than a state: it compares three producers, so it
// waits until the last of them has settled something rather than the first.
func waitForTerminal(sink *graphSink, arm string) error {
want := wantedTerminal(arm)
deadline := time.Now().Add(90 * time.Second)
for time.Now().Before(deadline) {
graph, _ := sink.snapshot()
if len(nodesInState(graph, want)) > 0 && identityReady(graph, arm) {
return nil
}
time.Sleep(50 * time.Millisecond)
}
graph, _ := sink.snapshot()
return errUnexpected("an item in state "+string(want), waitStates(graph))
}
// identityReady reports whether every producer the identity arm compares has
// published a settled item. Waiting on one of them leaves the others unread.
func identityReady(g agentgraph.Graph, arm string) bool {
if arm != armIdentitySemantics {
return true
}
settled := map[string]int{}
for _, n := range g.Nodes {
if n.Kind == agentgraph.KindGroup || n.State == "" || !n.State.Terminal() {
continue
}
settled[n.ParentID]++
}
return settled["probe_fleet_identity"] >= 4 && settled["probe_parallel_identity"] >= 2
}
func nodesInState(g agentgraph.Graph, want agentgraph.NodeState) []string {
var out []string
for _, n := range g.Nodes {
if n.Kind != agentgraph.KindGroup && n.State == want {
out = append(out, n.ID)
}
}
sort.Strings(out)
return out
}
// terminalRows put the two lifecycles side by side. Adopted and skipped reach
// the same durable shape by different routes and mean opposite things to a
// dependent, so every row here compares what the graph knew against what the
// journal can still be asked.
func terminalRows(arm string, before, after Observation) []row {
rows := []row{
terminalStateRow(arm, before, after),
terminalLifecycleRow(before, after),
terminalAnswerRow(before, after),
}
if arm == armTerminalAdopted {
rows = append(rows, adoptSourceRow(before, after))
}
if arm == armTerminalContext || arm == armTerminalSkippedDep {
rows = append(rows, contextDerivabilityRow(before, after))
}
if arm == armChildTerminal {
rows = append(rows, childOutcomeRow(before, after))
}
if arm == armIdentitySemantics {
rows = append(rows, identitySemanticsRow(before), identityRebuildRow(before, after))
}
return rows
}
// terminalStateRow is the arm's own control and the loss in one line: which
// disposition the graph ended on, and whether anything still says so.
func terminalStateRow(arm string, before, after Observation) row {
want := wantedTerminal(arm)
verdict := verdictNotMeasured
if len(nodesInState(graphOf(before), want)) > 0 {
verdict = verdictHolds
}
return row{
Semantic: "the disposition this arm reached", Authority: "the fan-out's closing delta",
Artifact: "none observed", Reconstruction: "published once, when the group ends",
Before: terminalSummary(before), After: terminalSummary(after), Verdict: verdict,
}
}
// terminalLifecycleRow is the comparison that decides whether a disposition has
// to be recorded. Two items with the same durable lifecycle and opposite
// meanings cannot be told apart by anything a restart reads.
func terminalLifecycleRow(before, after Observation) row {
return valueRow("the durable lifecycle each item left", "the execution journal",
"<stem>.execution.jsonl", "opened / queued / started / settled, per item",
lifecycleSummary(before), lifecycleSummary(after), true)
}
// terminalAnswerRow is what a dependent actually needs. Completed and adopted
// carry a final answer a dependent may start from; skipped and cancelled do
// not, and the difference is the whole reason the distinction exists.
func terminalAnswerRow(before, after Observation) row {
return valueRow("items holding an answer a dependent may use", "agentgraph.NodeState.Answered",
"none observed", "derived from the disposition, which is not recorded",
answeredSummary(before), answeredSummary(after), false)
}
// adoptSourceRow is the provenance an adoption carries: whose answer stood in.
// Reuse is only visible if the source is nameable — otherwise the picture shows
// work that cost nothing with no way to say what paid for it. The comparison is
// the graph's edge against the journal's opening, so a record that named a
// different source would read as a loss rather than as agreement.
func adoptSourceRow(before, after Observation) row {
return valueRow("whose answer an adoption reused", "the adopt edge, recorded with the opening",
"<stem>.execution.jsonl", "read back with the openings, never re-derived",
adoptEdges(before), journalAdoptions(after), true)
}
// journalAdoptions renders the openings' recorded sources the way the graph
// renders its edges, so the two sides of the row are the same measurement.
func journalAdoptions(o Observation) string {
var out []string
for _, e := range o.Executions {
if e.AdoptedFrom != "" {
out = append(out, e.ID+"<-"+e.AdoptedFrom)
}
}
sort.Strings(out)
return join(out)
}
// contextDerivabilityRow asks whether the delivery edge follows from facts that
// are already durable. A context edge that always accompanies an ordering edge
// whose upstream answered is derivable; one that does not is a fact of its own.
func contextDerivabilityRow(before, after Observation) row {
implied, actual := impliedContext(before), contextEdges(before)
verdict := verdictHolds
if implied != actual {
verdict = verdictViolated
}
return row{
Semantic: "whether a context edge follows from the durable facts",
Authority: "the ordering edges and what each upstream answered",
Artifact: "<stem>.execution.jsonl for the ordering; the disposition is not recorded",
Reconstruction: "an ordering edge whose upstream answered delivered its answer",
Before: "implied " + orNone(implied) + " / actual " + orNone(actual),
After: "implied " + orNone(impliedContext(after)) + " / actual: not observable",
Verdict: verdict,
}
}
// executedTerminals are the graph's terminals for items that actually ran.
// Adopted and skipped are excluded: nothing executed under them, so the store
// owes them no record and counting them would read a correct store as short.
func executedTerminals(o Observation) map[string]int {
out := map[string]int{}
for _, n := range o.Graph.Nodes {
if n.Kind == agentgraph.KindGroup || n.State == "" || !n.State.Terminal() {
continue
}
if n.State == agentgraph.StateAdopted || n.State == agentgraph.StateSkipped {
continue
}
out[string(n.State)]++
}
return out
}
// childOutcomeRow compares the two owners of the same fact. The graph draws
// completed, failed and cancelled apart; the store is what a restart reads. A
// terminal the graph distinguishes and the store folds into another is gone for
// good — no journal record could recover it, because the owner never kept it.
func childOutcomeRow(before, after Observation) row {
graphTerminals := executedTerminals(before)
verdict := verdictNotMeasured
if len(graphTerminals) > 0 {
verdict = verdictHolds
for state := range graphTerminals {
if !storeKnows(after, state) {
verdict = verdictViolated
}
}
}
return row{
Semantic: "terminal outcome: what the graph drew, what the store kept",
Authority: "the graph's closing delta against SubagentStore.Meta.Status",
Artifact: "subagents/<ref>.meta.json",
Reconstruction: "the store owns what happened to a child; every terminal it draws it must keep",
Before: "graph " + countSummary(graphTerminals) + " / store " + orNone(childStatusCounts(before)),
After: "store " + orNone(childStatusCounts(after)), Verdict: verdict,
}
}
// storeKnows reports whether any child carries this terminal as its status.
func storeKnows(o Observation, state string) bool {
for _, f := range o.Children.Facts {
if f.Status == state {
return true
}
}
return false
}
func childStatusCounts(o Observation) string {
counts := map[string]int{}
for _, f := range o.Children.Facts {
counts[f.Status]++
}
if len(counts) == 0 {
return ""
}
return countSummary(counts)
}
// identitySemanticsRow lays the three answers beside each other: what the
// caller named, what the graph shows, and what the store resolved. They are not
// expected to agree — the point is which of them a node's field is reporting,
// because a rebuild cannot be held to a field whose meaning is unsettled.
func identitySemanticsRow(before Observation) row {
store := map[string]string{}
for _, f := range before.Children.Facts {
store[f.ParentToolCallID] = orNone(f.Model) + "/" + orNone(f.Effort)
}
var out []string
for _, n := range before.Graph.Nodes {
if n.Kind != agentgraph.KindWorker {
continue
}
out = append(out, fmt.Sprintf("%s graph=%s/%s store=%s", n.ID,
orNone(n.Model), orNone(n.Effort), orNone(store[n.ID])))
}
sort.Strings(out)
return row{
Semantic: "worker identity: what each layer reports", Authority: "the fan-out's spec against the store's resolution",
Artifact: "subagents/<ref>.meta.json", Reconstruction: "the graph resolves a subagent chain; the store resolves the final identity",
Before: join(out), After: "not applicable: read before the boundary", Verdict: verdictStable,
}
}
// identityRebuildRow is the same question asked of the rebuild: whichever of
// those layers the field is meant to carry, does a restart get it back?
func identityRebuildRow(before, after Observation) row {
return valueRow("worker identity after a restart", "execgraph.Rebuild over journal + store",
"<stem>.execution.jsonl + subagents/<ref>.meta.json",
"the journal records no worker identity; the store records the one it resolved",
identitySummary(graphOf(before)), identitySummary(rebuiltGraph(after).Graph), false)
}
func graphOf(o Observation) agentgraph.Graph {
return agentgraph.Graph{Nodes: o.Graph.Nodes, Edges: o.Graph.Edges}
}
func terminalSummary(o Observation) string {
if len(o.Graph.Nodes) == 0 {
return ""
}
counts := map[string]int{}
for _, n := range o.Graph.Nodes {
if n.Kind != agentgraph.KindGroup && n.State != "" {
counts[string(n.State)]++
}
}
return countSummary(counts)
}
// lifecycleSummary renders each item's durable transitions, which is what a
// restart actually has. Two items printing the same line here are two items a
// restart cannot tell apart.
func lifecycleSummary(o Observation) string {
if len(o.Executions) == 0 {
return ""
}
out := make([]string, 0, len(o.Executions))
for _, e := range o.Executions {
marks := []string{"open"}
if e.Disposition != "" && e.Disposition != "pending" {
marks = append(marks, e.Disposition)
}
if e.Queued() {
marks = append(marks, "queued("+e.Cause+")")
}
if e.Started() {
marks = append(marks, "started")
}
if !e.SettledAt.IsZero() {
marks = append(marks, "settled")
}
out = append(out, e.ID+":"+strings.Join(marks, "+"))
}
sort.Strings(out)
return join(out)
}
func answeredSummary(o Observation) string {
var out []string
for _, n := range o.Graph.Nodes {
if n.Kind != agentgraph.KindGroup && n.State.Answered() {
out = append(out, n.ID)
}
}
sort.Strings(out)
return join(out)
}
func adoptEdges(o Observation) string {
var out []string
for _, e := range o.Graph.Edges {
if e.Kind == agentgraph.Adopt {
out = append(out, e.To+"<-"+e.From)
}
}
sort.Strings(out)
return join(out)
}
func contextEdges(o Observation) string {
var out []string
for _, e := range o.Graph.Edges {
if e.Kind == agentgraph.Context {
out = append(out, e.To+"<-"+e.From)
}
}
sort.Strings(out)
return join(out)
}
// impliedContext is what the durable facts alone would predict: every ordering
// edge whose upstream ended up answered.
func impliedContext(o Observation) string {
answered := map[string]bool{}
for _, n := range o.Graph.Nodes {
if n.State.Answered() {
answered[n.ID] = true
}
}
var out []string
for _, e := range o.Executions {
for _, up := range e.DependsOn {
if answered[up] {
out = append(out, e.ID+"<-"+up)
}
}
}
sort.Strings(out)
return join(out)
}
func countSummary(counts map[string]int) string {
keys := make([]string, 0, len(counts))
for k := range counts {
keys = append(keys, k)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, k := range keys {
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
}
return strings.Join(parts, " ")
}
// terminalArmInvalid refuses an arm that never reached its disposition. Every
// row below it compares against a shape that was not established, and reporting
// those as evidence is how a probe answers a question nobody asked.
func terminalArmInvalid(arm string, before Observation) string {
want := wantedTerminal(arm)
if len(nodesInState(graphOf(before), want)) == 0 {
return "no item reached " + string(want) + "; the fan-out ended as " + orNone(terminalSummary(before))
}
if arm == armTerminalContext && contextEdges(before) == "" {
return "the fan-out delivered no upstream answer, so there is no context edge to derive"
}
if arm == armTerminalCancelled && len(nodesInState(graphOf(before), agentgraph.StateSkipped)) > 0 {
return "an item was skipped rather than cancelled; parallel_tasks' skipped branch was thought unreachable"
}
return ""
}
+111
View File
@@ -0,0 +1,111 @@
package main
import (
"fmt"
"slices"
)
// Contract verdicts for an arm that asserts a fact rather than comparing two
// readings: the note either satisfies what todoIdentityProjection claims about
// itself, or it does not.
const (
verdictHolds = "holds"
verdictViolated = "VIOLATED"
)
// todoRows judge the host's step-identity note against the three things it
// claims: there is one of it, it rides the turn tail, and it names the identity
// the host holds now. Each is checked on both sides of the process boundary.
func todoRows(before, after Observation) []row {
return []row{
todoRow("host identity notes in the view", "one note when the history lost the ids, none when it kept them",
before, after, noteCount, exactlyOnce),
todoRow("where the identity note sits", "the note rides the live tail, as a turn-tail note",
before, after, notePlacement, ridesTheTail),
todoRow("the identity the note carries", "the note names the host's current step ids",
before, after, noteIdentity, matchesHost),
}
}
func todoRow(semantic, contract string, before, after Observation,
render func(Observation) string, holds func(Observation) bool) row {
verdict := verdictHolds
switch {
case before.TodoNotes.ViewLen == 0 || after.TodoNotes.ViewLen == 0:
verdict = verdictNotMeasured
case !holds(before) || !holds(after):
verdict = verdictViolated
}
return row{
Semantic: semantic, Authority: "agent todoState (host-owned)",
Artifact: "none: derived for each request, stored nowhere",
Reconstruction: contract,
Before: render(before), After: render(after), Verdict: verdict,
}
}
func noteCount(o Observation) string {
owed := "history lost the ids"
if o.TodoNotes.ReadableInHistory {
owed = "history still shows them"
}
return fmt.Sprintf("%d (%s)", o.TodoNotes.Count, owed)
}
// exactlyOnce is conditional on need: a request whose history still shows the
// ids is owed nothing, and demanding a note there would turn the migration into
// an unconditional append.
func exactlyOnce(o Observation) bool {
if o.TodoNotes.ReadableInHistory {
return o.TodoNotes.Count == 0
}
return o.TodoNotes.Count == 1
}
// notePlacement reports the note's index against the end of the frozen body.
// An index below it means the note was frozen ahead of messages the splice adds
// after it, which is the opposite of riding the tail.
func notePlacement(o Observation) string {
n := o.TodoNotes
if n.Count == 0 {
return "no note"
}
where := "after the live tail"
if n.Indexes[len(n.Indexes)-1] < n.BodyLen {
where = "inside the frozen body"
}
return fmt.Sprintf("index %v of %d, body ends at %d — %s", n.Indexes, n.ViewLen, n.BodyLen, where)
}
// ridesTheTail passes vacuously when no note is owed: placement is a claim
// about a note that exists.
func ridesTheTail(o Observation) bool {
n := o.TodoNotes
if n.Count == 0 {
return n.ReadableInHistory
}
return n.Indexes[len(n.Indexes)-1] >= n.BodyLen
}
func noteIdentity(o Observation) string {
n := o.TodoNotes
return fmt.Sprintf("note %s / host %s", join(dedupe(n.IDs)), join(n.HostIDs))
}
func matchesHost(o Observation) bool {
n := o.TodoNotes
if n.Count == 0 {
return n.ReadableInHistory
}
return len(n.HostIDs) > 0 && slices.Equal(dedupe(n.IDs), n.HostIDs)
}
func dedupe(in []string) []string {
var out []string
for _, s := range in {
if !slices.Contains(out, s) {
out = append(out, s)
}
}
return out
}
+356
View File
@@ -0,0 +1,356 @@
package main
import (
"encoding/json"
"fmt"
"net/http"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"reasonix/internal/contract/agentgraph"
"reasonix/internal/contract/config"
"reasonix/internal/contract/event"
"reasonix/internal/contract/provider"
"reasonix/internal/frontend/serve"
"reasonix/internal/session/control"
"reasonix/internal/state/execjournal"
)
// armUIGraphMixed is the only arm that reads the frontend rather than the host.
// Everything below it has been shown to rebuild the graph exactly; this asks
// the question a value comparison cannot: through which door did each fact
// reach the view? A history replayed as live deltas settles on the same state
// and tells the user work is starting that ended in a process now dead.
const armUIGraphMixed = "ui-graph-mixed"
func uiArm(name string) bool { return name == armUIGraphMixed }
// The two extra fleets this arm's turn opens. The settled one ends before the
// death, which is the only way a skip exists at all — a fan-out publishes one in
// its closing delta — and the live one is dispatched by the resumed process.
const (
fleetSettledSentinel = "PROBE-FLEET-SETTLED"
fleetLiveSentinel = "PROBE-FLEET-LIVE"
)
// envProbeSabotage names a deliberate defect to run under. The arm must go red
// with it and green without: a probe nobody has watched fail is a decoration.
const envProbeSabotage = "REASONIX_PROBE_SABOTAGE"
// envProbeRepo carries the checkout to the child that has to build the observer.
// A child's working directory is its own workspace, so nothing else in the
// process can find the frontend.
const envProbeRepo = "REASONIX_PROBE_REPO"
const (
sabotagePublish = "publish"
sabotageTrajectory = "trajectory"
)
// settledFleet runs to completion with its dependent skipped, so the death
// inherits a terminal the mid-flight fleet cannot produce.
func settledFleet() []provider.Chunk {
return fleetCall("probe_fleet_settled", []map[string]any{
{"id": "up", "prompt": childFail + " settled upstream", "description": "fails", "read_only": true},
{"id": "down", "prompt": childDone + " settled downstream", "description": "skipped by up",
"read_only": true, "depends_on": []string{"up"}},
})
}
// uiMixedFleet is the density fleet plus one item the session ceiling refuses,
// so the death also holds a node that was queued and never admitted.
func uiMixedFleet(adoptRef string) []provider.Chunk {
return fleetCall("probe_fleet_mixed", []map[string]any{
{"id": "m1", "prompt": childDone + " mixed one", "description": "completes", "read_only": true},
{"id": "m2", "prompt": childFail + " mixed two", "description": "fails", "read_only": true},
{"id": "m3", "prompt": childDone + " mixed three", "description": "blocked by m2",
"read_only": true, "depends_on": []string{"m2"}},
{"id": "m4", "adopt_ref": adoptRef, "description": "adopted"},
{"id": "m5", "prompt": childHang + " mixed five", "description": "running at death",
"write_paths": []string{"probe-fanout-out"}},
{"id": "m6", "prompt": childHang + " mixed six", "description": "queued at death", "read_only": true},
})
}
// liveFleet is what the resumed process dispatches while the view is watching.
// Without it a frontend that had stopped folding deltas entirely would pass
// every other row here.
func liveFleet() []provider.Chunk {
return fleetCall("probe_fleet_live", []map[string]any{
{"id": "l1", "prompt": childDone + " live one", "description": "starts after the resume", "read_only": true},
{"id": "l2", "prompt": childDone + " live two", "description": "starts after the resume", "read_only": true},
})
}
// livePrompt opens that fleet from the frontend, through /submit.
func livePrompt() string { return fanOutTurn(200, fleetLiveSentinel) }
// uiBus is the broadcaster the one arm with a frontend needs beside the probe's
// own fold: in the dying process it is what records the wire log a window would
// have recorded, and in the resumed one it is what the client reads.
func uiBus(arm string, sink *graphSink) (*serve.Broadcaster, event.Sink) {
if !uiArm(arm) {
return nil, sink
}
bc := serve.NewBroadcaster()
return bc, teeSink{to: []event.Sink{sink, bc}}
}
// recordLikeAWindow attaches the wire log the trajectory is read from. The
// server is never served: the constructor is what subscribes the recorder, and
// the dying process needs the record, not a socket.
func recordLikeAWindow(ctrl control.SessionAPI, bc *serve.Broadcaster) {
if bc != nil {
_ = serve.New(ctrl, bc, config.ServeConfig{})
}
}
// teeSink lets one runtime's events reach both the probe's fold and the server
// the frontend reads. Nothing production does this; the arm needs the same
// stream on two surfaces.
type teeSink struct{ to []event.Sink }
func (t teeSink) Emit(e event.Event) {
for _, s := range t.to {
s.Emit(e)
}
}
// UIEntry is one state the view passed through, as the frontend's own store
// reports it. States is what the graph showed, keyed by node.
type UIEntry struct {
Phase string `json:"phase"`
Origin string `json:"origin"`
SessionID string `json:"sessionId"`
States map[string]string `json:"states"`
Interruptions []UIInterruption `json:"interruptions"`
IdentityUnknown []string `json:"identityUnknown"`
}
type UIInterruption struct {
Execution string `json:"execution"`
Kind string `json:"kind"`
}
// UIObs is the whole observation sequence, not a final value: the defect this
// arm exists to catch agrees with the host on every final value.
type UIObs struct {
Trace []UIEntry `json:"trace,omitempty"`
// Deltas is what each publication the view folded named. A republished
// history moves no visible state and is still the kernel saying that work
// which ended in a dead process is happening now.
Deltas [][]string `json:"deltas,omitempty"`
Err string `json:"err,omitempty"`
}
// observeUI serves the resumed session on a loopback listener and runs the real
// client against it. The server is the production one and the client is the
// production one; only the shell between them is the probe's, which is what
// lets the trace be read as what a user could have seen.
func observeUI(root armRoot, ctrl control.SessionAPI, bc *serve.Broadcaster, obs Observation) UIObs {
ln, err := serve.ListenLoopback()
if err != nil {
return UIObs{Err: "listen: " + err.Error()}
}
defer ln.Close()
srv := serve.New(ctrl, bc, config.ServeConfig{})
handler := srv.Handler()
if os.Getenv(envProbeSabotage) == sabotagePublish {
handler = republishOnce(handler, func() { publishRebuiltGraph(bc, obs) })
}
go func() { _ = http.Serve(ln, handler) }()
bundle, err := bundleObserver(root)
if err != nil {
return UIObs{Err: err.Error()}
}
out, runErr := runObserver(bundle, serve.LoopbackOrigin(ln))
seen := parseTrace(out)
if runErr != nil && seen.Err == "" {
seen.Err = runErr.Error()
}
return seen
}
// bundleObserver builds the observer with the frontend's own toolchain. It is
// the client's code, imported the way the app imports it — a hand-written stand
// -in would be measuring the probe.
func bundleObserver(root armRoot) (string, error) {
repo := strings.TrimSpace(os.Getenv(envProbeRepo))
if repo == "" {
return "", fmt.Errorf("no checkout named in %s, so the frontend could not be built", envProbeRepo)
}
dir := filepath.Join(repo, "desktop", "frontend-next")
esbuild := filepath.Join(dir, "node_modules", ".bin", "esbuild")
if _, err := os.Stat(esbuild); err != nil {
return "", fmt.Errorf("the frontend's toolchain is not installed: %w", err)
}
out := filepath.Join(root.Dir, "graph-observer.mjs")
cmd := exec.Command(esbuild, filepath.Join("src", "e2e", "graph_observer.ts"),
"--bundle", "--platform=node", "--format=esm", "--outfile="+out)
cmd.Dir = dir
if msg, err := cmd.CombinedOutput(); err != nil {
return "", fmt.Errorf("bundle the observer: %w: %s", err, strings.TrimSpace(string(msg)))
}
return out, nil
}
func runObserver(bundle, base string) ([]byte, error) {
cmd := exec.Command("node", bundle, "--base", base, "--prompt", livePrompt(),
"--sabotage", os.Getenv(envProbeSabotage))
cmd.Stderr = os.Stderr
return cmd.Output()
}
// parseTrace reads the observer's last line. Anything it printed before that is
// the frontend's own noise, which belongs on the terminal and not in the row.
func parseTrace(out []byte) UIObs {
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
var obs UIObs
if len(lines) == 0 {
return UIObs{Err: "the observer printed nothing"}
}
if err := json.Unmarshal([]byte(lines[len(lines)-1]), &obs); err != nil {
return UIObs{Err: "the observer's answer did not parse: " + err.Error()}
}
return obs
}
// republishOnce fires the sabotage after a client has read the snapshot, which
// is the only window where it can be seen at all: what a resume publishes before
// that is numbered below the watermark the client then resumes past, and the
// bootstrap absorbs it. The defect this models is the Studio path — a pane that
// is already attached when a session is resumed under it.
func republishOnce(next http.Handler, fire func()) http.Handler {
var once sync.Once
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r)
if r.URL.Path == "/execution-graph" {
once.Do(func() { go fire() })
}
})
}
// publishRebuiltGraph is the sabotage itself: a resume that helpfully republishes
// what it rebuilt, which is how a view gets told that finished work is starting
// now. Production must never do this, and the arm's own rows are what says so.
func publishRebuiltGraph(sink event.Sink, obs Observation) {
rebuilt := rebuiltGraph(obs)
for _, n := range rebuilt.Graph.Nodes {
sink.Emit(event.Event{Kind: event.GraphDelta, Graph: &agentgraph.Delta{Nodes: []agentgraph.Node{n}}})
}
if len(rebuilt.Graph.Edges) > 0 {
sink.Emit(event.Event{Kind: event.GraphDelta, Graph: &agentgraph.Delta{Edges: rebuilt.Graph.Edges}})
}
// The publication has to reach the socket before the client attaches, or the
// sabotage would be measuring a race instead of a defect.
time.Sleep(200 * time.Millisecond)
}
// historicalIDs is every node the dying process had drawn, groups included: a
// group redrawn as running is the same lie about the same dead process.
func historicalIDs(o Observation) map[string]bool {
out := map[string]bool{}
for _, n := range o.Graph.Nodes {
out[n.ID] = true
}
return out
}
// firstSight is the origin each node carried the first time the view held it.
func firstSight(trace []UIEntry) map[string]string {
out := map[string]string{}
for _, e := range trace {
for id := range e.States {
if _, seen := out[id]; !seen {
out[id] = e.Origin
}
}
}
return out
}
var liveStates = map[string]bool{"pending": true, "queued": true, "running": true}
// ghostSightings are the historical nodes the view ever drew as someone's work
// in progress. One render tick is enough: the user saw it.
func ghostSightings(trace []UIEntry, historical map[string]bool) []string {
seen := map[string]bool{}
for _, e := range trace {
for id, state := range e.States {
if historical[id] && liveStates[state] {
seen[id+":"+state] = true
}
}
}
return sortedKeys(seen)
}
// rebirths are nodes the view lost and introduced again. A snapshot replace is
// not a rebirth; leaving a state and coming back inside one session is.
func rebirths(trace []UIEntry) []string {
gone, twice := map[string]bool{}, map[string]bool{}
held := map[string]bool{}
for _, e := range trace {
if e.Phase == "loading" {
gone, held = map[string]bool{}, map[string]bool{}
continue
}
for id := range held {
if _, still := e.States[id]; !still {
gone[id] = true
}
}
for id := range e.States {
if gone[id] {
twice[id] = true
}
}
held = map[string]bool{}
for id := range e.States {
held[id] = true
}
}
return sortedKeys(twice)
}
func sortedKeys(m map[string]bool) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// wantedInterruptions is what the host says, formatted the way the view reports
// it. The row compares the two rather than a hand-written expectation.
func wantedInterruptions(o Observation) string {
var out []string
for _, i := range rebuiltGraph(o).Interrupted {
kind := execjournal.InterruptedBeforeStart
if i.Started {
kind = execjournal.InterruptedDuringExecution
}
out = append(out, i.Execution+":"+kind)
}
sort.Strings(out)
return join(out)
}
func shownInterruptions(trace []UIEntry) string {
if len(trace) == 0 {
return ""
}
var out []string
for _, i := range trace[len(trace)-1].Interruptions {
out = append(out, i.Execution+":"+i.Kind)
}
sort.Strings(out)
return join(out)
}
+184
View File
@@ -0,0 +1,184 @@
package main
import (
"fmt"
"sort"
)
// The rows this arm exists for. Every other graph row compares values; these
// compare doors. A frontend that rebuilt the same graph by replaying history as
// live transitions would pass all of those and fail all of these.
const uiAuthority = "the frontend's own store, read through the real transport"
func uiRows(before, after Observation) []row {
ui := after.UI
if ui == nil || len(ui.Trace) == 0 {
return []row{uiRow("the view's observation sequence", "a sequence of visible states",
orNone(uiFailure(ui)), verdictNotMeasured)}
}
historical := historicalIDs(before)
first := firstSight(ui.Trace)
return []row{
uiFirstPictureRow(ui.Trace),
uiHistoryOriginRow(historical, first),
uiGhostRow(ui.Trace, historical),
uiInterruptionRow(ui.Trace, after),
uiLiveRow(historical, first),
uiRebirthRow(ui.Trace),
uiRepublishRow(historical, ui.Deltas),
}
}
func uiFailure(ui *UIObs) string {
if ui == nil {
return ""
}
return ui.Err
}
func uiRow(semantic, want, got, verdict string) row {
return row{
Semantic: semantic, Authority: uiAuthority, Artifact: "none — a sequence, not a value",
Reconstruction: want, Before: want, After: got, Verdict: verdict,
}
}
func held(ok bool) string {
if ok {
return verdictHolds
}
return verdictViolated
}
// uiFirstPictureRow reads the handover itself: nothing, then the authority. A
// graph that appears from a delta was assembled out of transitions, whatever it
// settles on afterwards.
func uiFirstPictureRow(trace []UIEntry) row {
got := trace[0].Phase
for _, e := range trace {
if len(e.States) > 0 {
got = trace[0].Phase + " → " + e.Origin
break
}
}
return uiRow("the first graph a resumed view draws", "loading → snapshot", got, held(got == "loading → snapshot"))
}
// uiHistoryOriginRow is the arm's whole point: an execution the dead process
// opened may enter the view only as part of an answer, never as news.
func uiHistoryOriginRow(historical map[string]bool, first map[string]string) row {
var wrong []string
seen := 0
for id, origin := range first {
if !historical[id] {
continue
}
seen++
if origin != "snapshot" {
wrong = append(wrong, id+" via "+origin)
}
}
sort.Strings(wrong)
got := fmt.Sprintf("%d from the snapshot", seen)
if len(wrong) > 0 {
got = join(wrong)
}
return uiRow("how work from the dead process entered the view",
fmt.Sprintf("%d executions, each from the snapshot", len(historical)), got, held(len(wrong) == 0))
}
// uiGhostRow watches every frame, not the last one: a node shown as running for
// a single render and corrected afterwards was still shown as running.
func uiGhostRow(trace []UIEntry, historical map[string]bool) row {
ghosts := ghostSightings(trace, historical)
return uiRow("a dead run shown as work in progress",
"never pending, queued or running", orNone(join(ghosts)), held(len(ghosts) == 0))
}
// uiInterruptionRow compares the two lists that describe the same executions:
// what the host derived, and what the view told the user.
func uiInterruptionRow(trace []UIEntry, after Observation) row {
want, got := wantedInterruptions(after), shownInterruptions(trace)
return uiRow("interruptions the view names", orNone(want), orNone(got), held(want == got))
}
// uiLiveRow is the positive control. Without it a view that had stopped folding
// deltas altogether would satisfy every row above.
func uiLiveRow(historical map[string]bool, first map[string]string) row {
var wrong, fresh []string
for id, origin := range first {
if historical[id] {
continue
}
fresh = append(fresh, id)
if origin != "delta" {
wrong = append(wrong, id+" via "+origin)
}
}
sort.Strings(fresh)
sort.Strings(wrong)
switch {
case len(fresh) == 0:
return uiRow("work started after the resume", "at least one, from a delta",
"nothing new was ever drawn", verdictViolated)
case len(wrong) > 0:
return uiRow("work started after the resume", "each from a delta", join(wrong), verdictViolated)
}
return uiRow("work started after the resume", "each from a delta",
fmt.Sprintf("%d from a delta: %s", len(fresh), join(fresh)), verdictHolds)
}
// uiRebirthRow: a node the view dropped and drew again is a second birth to
// whatever the interface hangs off an introduction, however idempotent the fold
// underneath it was.
func uiRebirthRow(trace []UIEntry) row {
twice := rebirths(trace)
return uiRow("a node introduced twice in one session",
"each node introduced once", orNone(join(twice)), held(len(twice) == 0))
}
// uiRepublishRow is the rule underneath the first-appearance one, and the only
// one that catches a republication at a view that already holds the snapshot:
// after a resume, no delta may name work the dead process opened. Nothing in
// production produces one — an execution belongs to the process that opened it.
func uiRepublishRow(historical map[string]bool, deltas [][]string) row {
named := map[string]bool{}
for _, ids := range deltas {
for _, id := range ids {
if historical[id] {
named[id] = true
}
}
}
old := sortedKeys(named)
return uiRow("deltas naming work the dead process opened",
"none: history is answered, never published", orNone(join(old)), held(len(old) == 0))
}
// uiArmInvalid guards the premise. The states the death has to hold are the
// whole reason this arm can say anything about ghosts, and an observer that
// never ran leaves every row below it describing nothing.
func uiArmInvalid(before, after Observation) string {
if after.UI == nil || len(after.UI.Trace) == 0 {
return "the frontend was never observed: " + orNone(uiFailure(after.UI))
}
var missing []string
for _, state := range wantedStates(armUIGraphMixed) {
if !holdsState(before, string(state)) {
missing = append(missing, string(state))
}
}
if len(missing) > 0 {
return "the death did not hold " + join(missing) + ", so the view had nothing to get wrong about them"
}
return ""
}
func holdsState(o Observation, want string) bool {
for _, n := range o.Graph.Nodes {
if string(n.State) == want {
return true
}
}
return false
}
+119
View File
@@ -0,0 +1,119 @@
package main
import "testing"
// The shapes the probe exists to tell apart, written out rather than run: a
// full arm takes three processes and a live kernel, which is the wrong place to
// find out that a verdict was computed backwards.
func historyOf(ids ...string) map[string]bool {
out := map[string]bool{}
for _, id := range ids {
out[id] = true
}
return out
}
func entry(phase, origin string, states map[string]string) UIEntry {
return UIEntry{Phase: phase, Origin: origin, States: states}
}
func TestAHistoryReadBackHoldsEveryRow(t *testing.T) {
historical := historyOf("g", "g/1", "g/2")
trace := []UIEntry{
entry("loading", "reset", nil),
entry("live", "snapshot", map[string]string{"g": "", "g/1": "completed", "g/2": "failed"}),
entry("live", "delta", map[string]string{"g": "", "g/1": "completed", "g/2": "failed", "n/1": "running"}),
entry("live", "delta", map[string]string{"g": "", "g/1": "completed", "g/2": "failed", "n/1": "completed"}),
}
first := firstSight(trace)
for _, r := range []row{
uiFirstPictureRow(trace),
uiHistoryOriginRow(historical, first),
uiGhostRow(trace, historical),
uiLiveRow(historical, first),
uiRebirthRow(trace),
uiRepublishRow(historical, [][]string{{"n/1"}}),
} {
if r.Verdict != verdictHolds {
t.Errorf("%s = %s (%s), want %s", r.Semantic, r.Verdict, r.After, verdictHolds)
}
}
}
// The defect the arm was built for: the same final state, assembled out of
// transitions. Every value row elsewhere in this probe would still be green.
func TestHistoryArrivingAsNewsIsCaught(t *testing.T) {
historical := historyOf("g/1")
trace := []UIEntry{
entry("loading", "reset", nil),
entry("live", "delta", map[string]string{"g/1": "running"}),
entry("live", "delta", map[string]string{"g/1": "completed"}),
}
first := firstSight(trace)
if got := uiFirstPictureRow(trace); got.Verdict != verdictViolated {
t.Errorf("first picture = %s, want %s", got.Verdict, verdictViolated)
}
if got := uiHistoryOriginRow(historical, first); got.Verdict != verdictViolated {
t.Errorf("history origin = %s, want %s", got.Verdict, verdictViolated)
}
if got := uiGhostRow(trace, historical); got.Verdict != verdictViolated {
t.Errorf("ghost = %s, want %s", got.Verdict, verdictViolated)
}
}
// A snapshot the view holds and a delta that names the same work again: the
// state never moves, and the kernel still said finished work is happening.
func TestRepublishedHistoryIsCaughtWithoutMovingTheState(t *testing.T) {
historical := historyOf("g/1")
trace := []UIEntry{
entry("loading", "reset", nil),
entry("live", "snapshot", map[string]string{"g/1": "completed"}),
entry("live", "delta", map[string]string{"g/1": "completed"}),
}
if got := uiHistoryOriginRow(historical, firstSight(trace)); got.Verdict != verdictHolds {
t.Fatalf("history origin = %s, want %s: the snapshot did arrive first", got.Verdict, verdictHolds)
}
if got := uiRepublishRow(historical, [][]string{{"g/1"}}); got.Verdict != verdictViolated {
t.Errorf("republication = %s, want %s", got.Verdict, verdictViolated)
}
}
// A view that stopped folding deltas would satisfy every rule above, which is
// what the positive control is for.
func TestAViewThatNeverMovesAgainIsCaught(t *testing.T) {
historical := historyOf("g/1")
trace := []UIEntry{
entry("loading", "reset", nil),
entry("live", "snapshot", map[string]string{"g/1": "completed"}),
}
if got := uiLiveRow(historical, firstSight(trace)); got.Verdict != verdictViolated {
t.Errorf("live control = %s (%s), want %s", got.Verdict, got.After, verdictViolated)
}
}
// A node the view dropped and drew again, which is a second introduction to
// anything the interface hangs off one.
func TestANodeIntroducedTwiceIsCaught(t *testing.T) {
trace := []UIEntry{
entry("live", "snapshot", map[string]string{"g/1": "completed"}),
entry("live", "delta", map[string]string{}),
entry("live", "delta", map[string]string{"g/1": "completed"}),
}
if got := uiRebirthRow(trace); got.Verdict != verdictViolated {
t.Errorf("rebirth = %s (%s), want %s", got.Verdict, got.After, verdictViolated)
}
}
// A session switch is not a rebirth: the view is told to forget, and what comes
// back is the next conversation's first picture.
func TestASwitchIsNotARebirth(t *testing.T) {
trace := []UIEntry{
entry("live", "snapshot", map[string]string{"g/1": "completed"}),
entry("loading", "reset", map[string]string{}),
entry("live", "snapshot", map[string]string{"g/1": "completed"}),
}
if got := uiRebirthRow(trace); got.Verdict != verdictHolds {
t.Errorf("rebirth across a switch = %s (%s), want %s", got.Verdict, got.After, verdictHolds)
}
}
+10 -1
View File
@@ -7,7 +7,7 @@ import "fmt"
// arm costs more: did the extra agents produce verified work, or just tokens.
func renderDelegation(results []result) string {
var runs, nested, childCalls, parentCalls, mutations, dupes int
var violations int
var reports, prose, falseDone, downgrades, violations int
var scopeHints, namedFiles, evidencePaths, discoveredPaths int
solved, total, childTokens := 0, 0, 0
for _, r := range results {
@@ -24,6 +24,10 @@ func renderDelegation(results []result) string {
parentCalls += r.ToolCalls - r.SubagentToolCalls
mutations += r.SubagentMutations
dupes += r.DuplicateWorkPaths
reports += r.CompletionReports
prose += r.CompletionsProsedOnly
falseDone += r.FalseCompletions
downgrades += r.CriterionDowngrades
violations += r.WriteScopeViolations
scopeHints += r.ParentScopeHints
namedFiles += r.ParentNamedFiles
@@ -66,6 +70,11 @@ func renderDelegation(results []result) string {
if dupes > 0 {
b += fmt.Sprintf("- **duplicate work**: %d file(s) mutated by more than one child\n", dupes)
}
closed := reports + prose
b += fmt.Sprintf("- closed with a checkable claim: **%d/%d** (%s)\n", reports, closed, pct(reports, closed))
if falseDone > 0 {
b += fmt.Sprintf("- **false completions**: %d run(s), %d criterion claim(s) the host refused\n", falseDone, downgrades)
}
if violations > 0 {
b += fmt.Sprintf("- **write-scope violations**: %d\n", violations)
}
+9 -2
View File
@@ -27,7 +27,9 @@ func TestRenderDelegationExposesWhatDelegationCost(t *testing.T) {
runMetrics: runMetrics{
ToolCalls: 30, SubagentToolCalls: 24,
SubagentRuns: 3, SubagentNestedRuns: 1, SubagentMutations: 5,
DuplicateWorkPaths: 2,
DuplicateWorkPaths: 2,
CompletionReports: 2, CompletionsProsedOnly: 1,
FalseCompletions: 1, CriterionDowngrades: 3,
WriteScopeViolations: 1,
},
}})
@@ -36,6 +38,8 @@ func TestRenderDelegationExposesWhatDelegationCost(t *testing.T) {
"parent **6** tool calls",
"children **24**",
"duplicate work**: 2",
"checkable claim: **2/3**",
"false completions**: 1 run(s), 3 criterion",
"write-scope violations**: 1",
} {
if !strings.Contains(got, want) {
@@ -88,11 +92,14 @@ func TestRenderDelegationSaysWhenEvidenceOriginIsUnscored(t *testing.T) {
func TestRenderDelegationStaysQuietWhenNothingWentWrong(t *testing.T) {
got := renderDelegation([]result{{
Passed: true,
runMetrics: runMetrics{ToolCalls: 12, SubagentToolCalls: 8, SubagentRuns: 2},
runMetrics: runMetrics{ToolCalls: 12, SubagentToolCalls: 8, SubagentRuns: 2, CompletionReports: 2},
}})
for _, unwanted := range []string{"duplicate work", "false completions", "write-scope violations"} {
if strings.Contains(got, unwanted) {
t.Errorf("clean run reported %q:\n%s", unwanted, got)
}
}
if !strings.Contains(got, "checkable claim: **2/2**") {
t.Errorf("clean run lost its completion coverage:\n%s", got)
}
}
+2 -2
View File
@@ -243,11 +243,11 @@ func renderDiff(r diffReport) string {
fmt.Fprintf(&b, "| Tokens (prompt / completion) | %s / %s |\n", comma(r.m.PromptTokens), comma(r.m.CompletionTokens))
fmt.Fprintf(&b, "| Model calls | %d |\n", r.m.Steps)
fmt.Fprintf(&b, "| Cost | %s%.4f |\n", currencySym(r.m.Currency), r.m.Cost)
if r.m.CapabilityRoutes > 0 || r.m.CapabilitySkillInvocations > 0 || r.m.CapabilityMCPCall > 0 {
if r.m.CapabilityRoutes > 0 || r.m.CapabilitySkillInvocations > 0 || r.m.CapabilityMCPCall > 0 || r.m.ReadinessChecks > 0 {
fmt.Fprintf(&b, "| Capability routes (semantic) | %d (%d) |\n", r.m.CapabilityRoutes, r.m.CapabilitySemanticRoutes)
fmt.Fprintf(&b, "| Routed candidates (require / prefer / suggest / declined) | %d (%d / %d / %d / %d) |\n", r.m.CapabilityRoutedCandidates, r.m.CapabilityRoutedRequire, r.m.CapabilityRoutedPrefer, r.m.CapabilityRoutedSuggest, r.m.CapabilityDeclines)
fmt.Fprintf(&b, "| Skill invocations / MCP proxy calls | %d / %d |\n", r.m.CapabilitySkillInvocations, r.m.CapabilityMCPCall)
fmt.Fprintf(&b, "| Review blocks | %d |\n", r.m.CapabilityReviewBlocks)
fmt.Fprintf(&b, "| Review blocks / readiness recoveries | %d / %d |\n", r.m.CapabilityReviewBlocks, r.m.ReadinessRecoveries)
if r.m.CapabilityRouterCost > 0 || r.m.CapabilityRouterLatencyMs > 0 {
fmt.Fprintf(&b, "| Capability-router cost / latency | %s%.4f / %dms |\n", currencySym(r.m.Currency), r.m.CapabilityRouterCost, r.m.CapabilityRouterLatencyMs)
}
+11 -5
View File
@@ -66,11 +66,15 @@ type runMetrics struct {
// Delegation counters mirror internal/frontend/cli.RunMetrics. They are what makes a
// single-agent arm comparable against a delegated one for the same model.
SubagentRuns int `json:"subagent_runs,omitempty"`
SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"`
SubagentMutations int `json:"subagent_mutations,omitempty"`
WriteScopeViolations int `json:"write_scope_violations,omitempty"`
DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"`
SubagentRuns int `json:"subagent_runs,omitempty"`
SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"`
SubagentMutations int `json:"subagent_mutations,omitempty"`
CompletionReports int `json:"completion_reports,omitempty"`
CompletionsProsedOnly int `json:"completions_prose_only,omitempty"`
FalseCompletions int `json:"false_completions,omitempty"`
CriterionDowngrades int `json:"criterion_downgrades,omitempty"`
WriteScopeViolations int `json:"write_scope_violations,omitempty"`
DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"`
// Evidence origin: what the parent's own delegation text scoped and named,
// and how much of what the children looked at they had to find themselves.
ParentScopeHints int `json:"parent_scope_hints,omitempty"`
@@ -81,6 +85,8 @@ type runMetrics struct {
// graph, which is where a scheduling wait is recorded and nowhere else.
Fanout *fanoutMetrics `json:"fanout,omitempty"`
// Optional Delivery capability counters (omitempty for baseline/old metrics).
ReadinessChecks int `json:"readiness_checks,omitempty"`
ReadinessRecoveries int `json:"readiness_recoveries,omitempty"`
CapabilityRoutes int `json:"capability_routes,omitempty"`
CapabilityRoutedCandidates int `json:"capability_routed_candidates,omitempty"`
CapabilityRoutedRequire int `json:"capability_routed_require,omitempty"`
-11
View File
@@ -1,11 +0,0 @@
package main
import (
"os"
"reasonix/internal/base/netdns"
)
func init() {
netdns.Install(netdns.Parse(os.Getenv(netdns.EnvVar)))
}
-19
View File
@@ -72,25 +72,6 @@ go run ./cmd/reasonix serve # from the repository root
cd desktop/frontend-next && pnpm install && pnpm dev # :5273, proxies API paths
```
## Messages sent during a run
**Settings → Appearance → Keyboard shortcuts → Messages sent while running** chooses the default:
- **Steer at the next tool boundary** (default) adds guidance to the current run.
- **Send after the whole run** queues a new turn until the active run finishes.
The choice is local to this device's browser profile, applies across projects,
and survives reloads. It does not change idle sends or messages already queued.
Skill invocations continue to use follow-up turns.
The queue still supports hold/resume, editing, and taking a message back into
the composer. **Send now** overrides the wait for a selected message by stopping
the current response and starting that message as a new turn; it is unavailable
while the queue is held.
Guidance not delivered before a run ends still becomes a follow-up through the
kernel's existing queue.
## Global show/hide shortcut
In the desktop shell, **Settings → Appearance → Global show / hide shortcut**
+1 -1
View File
@@ -129,7 +129,7 @@
!macro rxScript
FileOpen $0 "$PLUGINSDIR\rxclose.ps1" w
FileWrite $0 "param([string]$$Dir, [string]$$Mode, [int]$$Self)$\r$\n"
FileWrite $0 "$$names = @('${APP_EXECUTABLE_FILENAME}', 'reasonix-studio-host.exe')$\r$\n"
FileWrite $0 "$$names = @('${APP_EXECUTABLE_FILENAME}', 'reasonix-studio-host.exe', 'cua-driver.exe')$\r$\n"
FileWrite $0 "$$hit = @(Get-CimInstance -ClassName Win32_Process | ? { $$_.ProcessId -ne $$Self -and (($$_.Path -and $$_.Path.StartsWith($$Dir, 'CurrentCultureIgnoreCase')) -or (-not $$_.Path -and $$names -contains $$_.Name)) })$\r$\n"
FileWrite $0 "if ($$Mode -eq 'find') { if ($$hit.Count -gt 0) { exit 0 } else { exit 1 } }$\r$\n"
FileWrite $0 "$$hit | % { Stop-Process -Id $$_.ProcessId -Force -ErrorAction SilentlyContinue }$\r$\n"
+1 -3
View File
@@ -16,10 +16,8 @@
"smoke": "electron test/smoke.js",
"browser-live": "electron test/browser_live.js",
"shortcut-live": "electron test/globalshortcut_live.js",
"theme-live": "electron test/theme_live.js",
"browser-resize-live": "electron test/browser_resize_live.js",
"browser-fit-live": "electron test/browser_fit_live.js",
"pick-live": "electron test/elementpick_live.js",
"browser-fit-pages": "electron test/browser_fit_pages.js",
"sleep-live": "electron test/sleep_live.js",
"image-live": "electron test/image_live.js",
@@ -33,7 +31,7 @@
"launcher-live": "node test/launcher_live.mjs"
},
"devDependencies": {
"electron": "45.1.0",
"electron": "44.7.2",
"electron-builder": "^26.18.0",
"playwright": "1.64.1"
},
-35
View File
@@ -1,35 +0,0 @@
"use strict";
const fs = require("node:fs");
const path = require("node:path");
// Exit 0 when the artifact is missing or any file under the given paths is
// newer than it, 1 when it is up to date. run-studio.sh asks this before
// rebuilding; the shell has no portable `find` on every platform it runs on.
const [artifact, ...inputs] = process.argv.slice(2);
if (!artifact) process.exit(2);
let built;
try {
built = fs.statSync(artifact).mtimeMs;
} catch {
process.exit(0);
}
const SKIP = new Set(["node_modules", ".git", "dist"]);
function newer(entry) {
let info;
try {
info = fs.statSync(entry);
} catch {
return false;
}
if (!info.isDirectory()) return info.mtimeMs > built;
for (const name of fs.readdirSync(entry)) {
if (SKIP.has(name)) continue;
if (newer(path.join(entry, name))) return true;
}
return false;
}
process.exit(inputs.some(newer) ? 0 : 1);
+5 -5
View File
@@ -9,8 +9,8 @@ importers:
.:
devDependencies:
electron:
specifier: 45.1.0
version: 45.1.0([email protected])
specifier: 44.7.2
version: 44.7.2([email protected])
electron-builder:
specifier: ^26.18.0
version: 26.18.0([email protected])([email protected])
@@ -374,8 +374,8 @@ packages:
resolution: {integrity: sha512-bO3y10YikuUwUuDUQRM4KfwNkKhnpVO7IPdbsrejwN9/AABJzzTQ4GeHwyzNSrVO+tEH3/Np255a3sVZpZDjvg==}
engines: {node: '>=8.0.0'}
electron@45.1.0:
resolution: {integrity: sha512-i7qKVAh/KR5v1pf56NhGGX2QYHcdjvLgS7koL6ui50c+zUDGLRunR95nXcTpHPURKOQ4GDBG8BmXiDgOQ8DD0A==}
electron@44.7.2:
resolution: {integrity: sha512-u3Pbz0Hi1sSSwBMz2v8LiagsfUG4QURPUe/qphM+g4THKP8o4BYdnuboyhlLO8HIC/KfV5rvltyacbP4jEq9yw==}
engines: {node: '>= 22.12.0'}
hasBin: true
@@ -1551,7 +1551,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
electron@45.1.0([email protected]):
electron@44.7.2([email protected]):
dependencies:
'@electron-internal/extract-zip': 1.0.5
'@electron/get': 5.1.0([email protected])
+4 -22
View File
@@ -12,29 +12,11 @@ if [ ! -d "$here/node_modules" ]; then
(cd "$here" && pnpm install --frozen-lockfile)
fi
# A step is skipped when nothing it reads is newer than what it produced;
# STUDIO_REBUILD=1 forces every step.
stale() {
[ -n "${STUDIO_REBUILD:-}" ] || node "$here/packaging/stale.js" "$@"
}
echo "==> frontend"
(cd "$here/../frontend-next" && pnpm install --frozen-lockfile >/dev/null && pnpm build >/dev/null)
root="$here/../.."
web="$here/../frontend-next"
host=$(ls "$here"/bin/reasonix-studio-host* 2>/dev/null | head -n 1 || true)
if stale "$web/dist/index.html" "$web/src" "$web/public" "$web/index.html" "$web/package.json" "$web/pnpm-lock.yaml" "$web/vite.config.ts" "$web/tsconfig.json"; then
echo "==> frontend"
(cd "$web" && pnpm install --frozen-lockfile >/dev/null && pnpm build >/dev/null)
else
echo "==> frontend (up to date)"
fi
if [ -z "$host" ] || stale "$host" "$root/internal" "$root/cmd" "$root/go.mod" "$root/go.sum" "$here/packaging"; then
echo "==> kernel"
(cd "$here" && pnpm build:host)
else
echo "==> kernel (up to date)"
fi
echo "==> kernel"
(cd "$here" && pnpm build:host)
echo "==> studio"
cd "$here" && exec pnpm start
+1 -204
View File
@@ -3,7 +3,6 @@ const crypto = require("node:crypto");
const { WebContentsView, session } = require("electron");
const { installPaneReload } = require("./reload");
const { guestNavigationAllowed, typed } = require("./browserguard");
const { ARM, STOP } = require("./elementpick");
const { InputWake } = require("./inputwake");
const { KeyboardKeeper } = require("./keyboard");
@@ -24,26 +23,6 @@ const PAGE_RADIUS = 8;
// How many pages may stay asleep at once; past this the oldest is buried.
const SLEPT_LIMIT = 18;
// How long a pick waits for the person. Past it the page is left as it was
// found: a picker left armed swallows every click on the page, so the wait is
// a minute and a half, not an afternoon.
const PICK_LIMIT_MS = 90_000;
// A page's own scripts live in world 0; the picker answers from its own.
const PICK_WORLD = 999;
// The page wrote the answer; these repeat the page script's own limits because
// the shell cannot trust it to have run them.
const TEXT_CLIP = 4000;
const NEARBY_CLIP = 300;
const MARKUP_CLIP = 6000;
const VALUE_CLIP = 500;
const ATTR_LIMIT = 64;
const STYLE_CLIP = 200;
const ELEMENT_STRINGS = ["pageUrl", "pageTitle", "tagName", "role", "accessibleName", "selector", "xpath", "text", "nearbyText", "htmlExcerpt"];
const STYLE_KEYS = ["backgroundColor", "color", "display", "fontFamily", "fontSize", "fontWeight"];
const SELECTOR_BASIS = ["testid", "aria-label", "id", "class"];
const CANCEL_REASONS = ["escape", "dismissed", "timeout", "hidden", "navigation", "gone"];
const clip = (value, limit) => (value.length > limit ? value.slice(0, limit) : value);
// A command that switches a CDP domain on. The kernel sends these once at
// attach, and a sleep throws away the session they were said on, so they are
// kept per page and said again to every session a wake builds.
@@ -243,17 +222,6 @@ class BrowserViews {
entry.onEvent(method, params);
});
this.keyboard.watch(contents, () => Date.now() < entry.synthUntil);
// The picker's key belongs to the page while the page holds the keyboard:
// a press that went there never reaches the window's own page, so it is
// taken here and the picker is started where the person is looking.
contents.on("before-input-event", (event, input) => {
if (!input || input.type !== "keyDown" || input.isAutoRepeat || !input.key || !input.shift) return;
if (String(input.key).toLowerCase() !== "c") return;
if (process.platform === "darwin" ? !input.meta : !input.control) return;
if (entry.targetId !== this.shown) return;
event.preventDefault();
this.send("browser:pick-shortcut", { targetId: entry.targetId });
});
this.watchLoads(entry.targetId, entry);
const gone = () => {
// A sleep closes the contents itself, so a death under it is the sleep
@@ -406,7 +374,7 @@ class BrowserViews {
// at it, a navigation or a settle is pending, a certificate or a dialog waits,
// it is sounding, or DevTools is open. Downloads need no guard here.
maySleep(entry) {
if (entry.targetId === this.shown || entry.picking || entry.returning || entry.certificate || entry.dialog) return false;
if (entry.targetId === this.shown || entry.returning || entry.certificate || entry.dialog) return false;
const contents = entry.view && entry.view.webContents;
if (typeof contents?.isCurrentlyAudible === "function" && contents.isCurrentlyAudible()) return false;
if (typeof contents?.isDevToolsOpened === "function" && contents.isDevToolsOpened()) return false;
@@ -601,13 +569,6 @@ class BrowserViews {
for (const entry of this.entries.values()) {
if (entry.view) this.putAway(entry.view, entry);
}
// A pick behind a page nobody can see cannot be used: the person is looking
// at a picture, so the pick is settled instead of swallowing clicks unseen.
for (const entry of this.entries.values()) {
if (!entry.picking) continue;
entry.picking.reason = "hidden";
void this.cancelPick(entry.targetId);
}
}
putAway(view, entry) {
@@ -647,170 +608,6 @@ class BrowserViews {
}
}
// pick arms one page's element picker and answers what the person pointed
// at, or { cancelled: true } when they pressed Escape, navigated away, or
// left it longer than a pick may wait. The highlight and the hover card are
// drawn inside the page: a native view sits above anything this window's own
// page could draw over it.
// The page wrote the answer, so the shell decides what an answer is: fields
// from an allow-list, strings clipped to the page script's own limits, and a
// shape it cannot read treated as gone.
clean(answer) {
const gone = { cancelled: true, reason: "gone" };
if (!answer || typeof answer !== "object") return gone;
if (answer.cancelled) {
const reason = CANCEL_REASONS.includes(answer.reason) ? answer.reason : "gone";
return { cancelled: true, reason };
}
const el = answer.element;
if (!el || typeof el !== "object") return gone;
const out = {};
for (const field of ELEMENT_STRINGS) {
const value = el[field];
if (typeof value !== "string") continue;
out[field] = clip(value, field === "htmlExcerpt" ? MARKUP_CLIP : field === "nearbyText" ? NEARBY_CLIP : TEXT_CLIP);
}
if (SELECTOR_BASIS.includes(el.selectorBasis)) out.selectorBasis = el.selectorBasis;
if (el.attributes && typeof el.attributes === "object") {
const attributes = {};
for (const [key, value] of Object.entries(el.attributes)) {
if (Object.keys(attributes).length >= ATTR_LIMIT) break;
if (typeof value === "string") attributes[clip(key, VALUE_CLIP)] = clip(value, VALUE_CLIP);
}
out.attributes = attributes;
}
if (Number.isFinite(el.capturedAt)) out.capturedAt = el.capturedAt;
if (el.rect && typeof el.rect === "object") {
const { x, y, width, height } = el.rect;
if ([x, y, width, height].every(Number.isFinite)) out.rect = { x, y, width, height };
}
if (el.style && typeof el.style === "object") {
out.style = {};
for (const key of STYLE_KEYS) if (typeof el.style[key] === "string") out.style[key] = clip(el.style[key], STYLE_CLIP);
}
if (!out.selector || !out.tagName) return gone;
return { cancelled: false, element: out };
}
async pick(targetId, labels) {
const entry = this.entries.get(targetId);
if (!entry) return { cancelled: true, reason: "gone" };
// A pick draws inside the page that is on screen; a page that is not shown
// would pick behind the one the person is looking at.
if (entry.targetId !== this.shown) return { cancelled: true, reason: "hidden" };
if (entry.slept) await this.wake(entry).catch(() => {});
const contents = entry.view && entry.view.webContents;
if (!contents || contents.isDestroyed()) return { cancelled: true, reason: "gone" };
// A second pick on the same page replaces the first: one picker, one
// overlay, and the older call answers as cancelled.
if (entry.picking) {
entry.picking.reason = "dismissed";
await this.cancelPick(targetId).catch(() => {});
}
// Why the pick ends when it ends without an element: Escape and a dismissal
// come from the page, the other three from here. The person is told which,
// so "nothing happened" is never the answer.
const picking = (entry.picking = { reason: "" });
const timer = setTimeout(() => {
picking.reason = "timeout";
void this.cancelPick(targetId);
}, PICK_LIMIT_MS);
if (typeof timer.unref === "function") timer.unref();
try {
const answer = this.clean(await contents.executeJavaScript(ARM(labels), false));
if (answer.cancelled) return picking.reason ? { cancelled: true, reason: picking.reason } : answer;
// The page cannot invent an element: the shell re-finds the selector in
// its own world before the answer is trusted.
if (!(await this.lands(contents, answer.element))) return { cancelled: true, reason: "gone" };
// The picture is what the person saw: a complaint about how something
// looks is answered by the element, not only by its markup.
const shot = await this.shot(entry, answer.element.rect).catch(() => "");
if (shot) answer.shot = shot;
return answer;
} catch {
// The page navigated, reloaded, slept, or was closed under the pick.
return { cancelled: true, reason: picking.reason || "navigation" };
} finally {
if (entry.picking === picking) entry.picking = null;
clearTimeout(timer);
}
}
// lands asks the DOM from the shell's own world whether a selector still
// names an element of the tag the pick reported. A page script cannot answer
// this one in the page's place.
async lands(contents, element) {
if (!element || !element.selector || !element.tagName) return false;
const tag = await contents
.executeJavaScriptInIsolatedWorld(PICK_WORLD, [
{
code: `(() => {
const el = document.querySelector(${JSON.stringify(element.selector)});
return el ? el.tagName.toLowerCase() : "";
})()`,
},
])
.catch(() => "");
return tag === String(element.tagName).toLowerCase();
}
// shot captures the element as the view drew it. The page's rect is in CSS
// pixels and the view may be fitted, so the scale is applied and the result
// is kept inside the view; an element scrolled out of sight has no picture.
async shot(entry, rect) {
const contents = entry.view && entry.view.webContents;
if (!contents || contents.isDestroyed() || !rect) return "";
const scale = entry.scale || 1;
const bounds = entry.view.getBounds();
const at = {
x: Math.max(0, Math.round(rect.x * scale)),
y: Math.max(0, Math.round(rect.y * scale)),
width: Math.round(rect.width * scale),
height: Math.round(rect.height * scale),
};
at.width = Math.min(at.width, bounds.width - at.x);
at.height = Math.min(at.height, bounds.height - at.y);
if (at.width < 1 || at.height < 1) return "";
const image = await contents.capturePage(at);
return image.isEmpty() ? "" : image.toDataURL();
}
// verify re-finds a selector after the page changed, so a person can see
// whether an edit landed where the pick pointed.
async verify(targetId, selector) {
const entry = this.entries.get(targetId);
if (!entry || !selector) return { found: false };
const contents = entry.view && entry.view.webContents;
if (!contents || contents.isDestroyed()) return { found: false };
const at = await contents
.executeJavaScriptInIsolatedWorld(PICK_WORLD, [
{
code: `(() => {
const el = document.querySelector(${JSON.stringify(selector)});
if (!el) return { found: false };
const r = el.getBoundingClientRect();
return { found: true, tag: el.tagName.toLowerCase(), rect: { x: r.x, y: r.y, width: r.width, height: r.height } };
})()`,
},
])
.catch(() => ({ found: false }));
if (!at || !at.found || !at.rect) return { found: false };
const shot = await this.shot(entry, at.rect).catch(() => "");
return { found: true, tag: at.tag, rect: at.rect, shot };
}
// cancelPick disarms the picker, which settles a pick still waiting on the
// page. A page that has gone answers nothing, which reads as cancelled.
cancelPick(targetId) {
const entry = this.entries.get(targetId);
const contents = entry && entry.view && entry.view.webContents;
if (!contents || contents.isDestroyed()) return Promise.resolve(false);
return contents.executeJavaScript(STOP, false).then(
() => true,
() => false,
);
}
// navigate starts loading what the person typed and answers "", or the code
// of why the window will not: "scheme" or "network_file".
navigate(targetId, address) {
-572
View File
@@ -1,572 +0,0 @@
"use strict";
// The page half of the element picker, kept as a function so it can be
// stringified into the page. The highlight and the hover card are drawn there
// because a native view sits above anything this window's own page draws.
//
// Nothing here may reach out of the function: the page runs it alone, with no
// module and no binding of this process to refer to.
function pagePicker(labels = {}) {
// The window owns the words and sends them in: the page only draws them.
const L = {
color: "Color",
background: "Background",
font: "Font",
frame: "Element inside an embedded frame — not supported",
shadow: "Element inside a shadow root — not supported",
...labels,
};
const KEY = "__reasonixElementPicker";
const CLIP = 4000;
// Any words of its own already say what the element is, in any language;
// only one that cannot speak — an icon, an image — is given the words
// around it. A Chinese heading is a sentence in a dozen characters, so the
// test is whether it has text at all, not how much.
const NEARBY_OWN = 2;
const NEARBY_LIMIT = 300;
let settlePick = null;
// The promise is the pick: it settles on the click, on Escape, or when the
// page takes the picker away from under it.
const answer = new Promise((done) => {
settlePick = done;
});
const previous = window[KEY];
if (previous) previous.stop();
const TAG_ROLES = {
A: "link",
BUTTON: "button",
IMG: "img",
NAV: "navigation",
MAIN: "main",
FORM: "form",
SELECT: "combobox",
TEXTAREA: "textbox",
H1: "heading",
H2: "heading",
H3: "heading",
H4: "heading",
H5: "heading",
H6: "heading",
};
const INPUT_ROLES = {
checkbox: "checkbox",
radio: "radio",
range: "slider",
button: "button",
submit: "button",
reset: "button",
};
const KEPT = new Set(["id", "class", "href", "src", "alt", "title", "name", "type", "placeholder", "data-testid"]);
// The roles an element can have while saying nothing to a reader by itself:
// an icon button, an image, a field. Context is for these. A decorative
// span has no role, and the page around it does not give it one.
const SPEAKING = new Set(["button", "link", "textbox", "checkbox", "radio", "slider", "combobox", "img"]);
const clip = (text, limit) => (text.length > limit ? text.slice(0, limit) : text);
const textOf = (node) => (node.innerText || node.textContent || "").trim();
// rgb()/rgba() as the page computed them, written the way a person reads a
// colour; a translucent one is left as computed so its alpha is not lost.
const hex = (css) => {
const match = /^rgba?\(([^)]+)\)$/.exec(css || "");
if (!match) return css || "";
const parts = match[1].split(/[\s,/]+/).filter(Boolean).map(Number);
if (parts.length > 3 && parts[3] < 1) return css;
const [r, g, b] = parts;
return ("#" + [r, g, b].map((one) => Math.round(one).toString(16).padStart(2, "0")).join("")).toUpperCase();
};
const roleOf = (el) => {
const stated = el.getAttribute("role");
if (stated) return stated;
if (el.tagName === "INPUT") return INPUT_ROLES[el.type] || "textbox";
return TAG_ROLES[el.tagName] || "";
};
const nameOf = (el) => {
const ids = (el.getAttribute("aria-labelledby") || "").split(/\s+/).filter(Boolean);
if (ids.length) {
const text = ids
.map((id) => {
const node = document.getElementById(id);
return node ? textOf(node) : "";
})
.join(" ")
.trim();
if (text) return clip(text, 500);
}
for (const attr of ["aria-label", "alt", "title", "placeholder"]) {
const value = el.getAttribute(attr);
if (value && value.trim()) return clip(value.trim(), 500);
}
return clip(textOf(el), 500);
};
// What the page declares as an identity outlives what it builds for styling:
// a hash class is gone after the next deploy, a data-testid is not.
const attrValue = (value) => '"' + value.replace(/["\\]/g, "\\$&") + '"';
// A CSS path a person can paste, preferring the page's own identity over its
// styling. Falls back to tag.class with :nth-of-type where a sibling would
// answer the same, up to eight levels of parent.
const selectorOf = (el) => {
const testId = el.getAttribute("data-testid");
if (testId) return { selector: `[data-testid=${attrValue(testId)}]`, basis: "testid" };
const label = el.getAttribute("aria-label");
if (label) return { selector: `${el.tagName.toLowerCase()}[aria-label=${attrValue(label)}]`, basis: "aria-label" };
const own = el.getAttribute("id");
if (own) return { selector: "#" + CSS.escape(own), basis: "id" };
const parts = [];
let node = el;
for (let depth = 0; node && node.nodeType === 1 && depth < 8; depth++) {
const id = node.getAttribute("id");
if (id) {
parts.unshift("#" + CSS.escape(id));
break;
}
let part = node.tagName.toLowerCase();
for (const name of [...node.classList].filter(Boolean).slice(0, 3)) part += "." + CSS.escape(name);
const parent = node.parentElement;
if (parent) {
const same = [...parent.children].filter((child) => child.tagName === node.tagName);
if (same.length > 1) part += `:nth-of-type(${same.indexOf(node) + 1})`;
}
parts.unshift(part);
node = node.parentElement;
}
return { selector: parts.join(" > "), basis: "class" };
};
const xpathOf = (el) => {
const parts = [];
let node = el;
for (let depth = 0; node && node.nodeType === 1 && depth < 12; depth++) {
let index = 1;
for (let sib = node.previousElementSibling; sib; sib = sib.previousElementSibling) {
if (sib.tagName === node.tagName) index++;
}
parts.unshift(`${node.tagName.toLowerCase()}[${index}]`);
node = node.parentElement;
}
return "/" + parts.join("/");
};
// What the element says. A password field never hands over what was typed in
// it: the pick leaves with the fact that it was one, and nothing else.
const textFor = (el) => {
// An editable region holds what the person is writing right now.
if (el.isContentEditable) return "[masked editable text]";
if (el.tagName === "INPUT" && el.type === "password") return "[masked password input]";
if (el.tagName === "INPUT" || el.tagName === "TEXTAREA") {
return clip([el.getAttribute("aria-label"), el.getAttribute("placeholder"), el.getAttribute("name"), el.type].filter(Boolean).join(" "), CLIP);
}
return clip(textOf(el), CLIP);
};
// What surrounds an element, for the ones that cannot speak for themselves.
// An element with text of its own asks for nothing: the nearest article
// around a title is the whole page, which is not context but noise.
// What surrounds an element, for the ones that cannot speak for themselves.
// An element with text of its own asks for nothing, and neither does one
// with no role at all: a decorative mark is not made meaningful by the page
// around it, so the words there would be noise rather than context.
const nearbyFor = (el) => {
if (textOf(el).trim().length >= NEARBY_OWN) return "";
if (!SPEAKING.has(roleOf(el))) return "";
const near = el.closest("article, section, main, form, li, tr, dialog") || el.parentElement;
return near ? clip(textOf(near), NEARBY_LIMIT) : "";
};
// The element's own markup with its scripts and the values a person typed
// taken out, so what goes to the chat is structure, not their input.
const excerptFor = (el) => {
const clone = el.cloneNode(true);
for (const drop of clone.querySelectorAll("script, style, noscript, template")) drop.remove();
for (const field of clone.querySelectorAll("input")) field.removeAttribute("value");
if (clone.tagName === "INPUT") clone.removeAttribute("value");
for (const area of clone.querySelectorAll("textarea")) area.textContent = "";
if (clone.isContentEditable) clone.textContent = "";
return clip(clone.outerHTML, 6000);
};
const describe = (el) => {
const rect = el.getBoundingClientRect();
const computed = getComputedStyle(el);
const attributes = {};
for (const attr of el.attributes) {
const name = attr.name.toLowerCase();
if (name === "value") continue;
if (!KEPT.has(name) && !name.startsWith("aria-")) continue;
attributes[name] = clip(attr.value, 500);
}
const style = {
color: hex(computed.color),
display: computed.display,
fontFamily: computed.fontFamily,
fontSize: computed.fontSize,
fontWeight: computed.fontWeight,
};
if (computed.backgroundColor && computed.backgroundColor !== "rgba(0, 0, 0, 0)" && computed.backgroundColor !== "transparent") {
style.backgroundColor = hex(computed.backgroundColor);
}
const path = selectorOf(el);
const info = {
pageUrl: location.href,
pageTitle: document.title,
tagName: el.tagName.toLowerCase(),
selector: path.selector,
xpath: xpathOf(el),
htmlExcerpt: excerptFor(el),
attributes,
rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
style,
capturedAt: Date.now(),
};
// Which of the four the selector came from, so a reader can tell a stable
// identity from one that a deploy will change.
info.selectorBasis = path.basis;
// A field with no answer is left out rather than sent empty: "there is no
// role" and "the role is an empty string" are different answers, and the
// second is not one this page can give.
const role = roleOf(el);
if (role) info.role = role;
const name = nameOf(el);
if (name) info.accessibleName = name;
const own = textFor(el);
if (own) info.text = own;
const nearby = nearbyFor(el);
if (nearby) info.nearbyText = nearby;
return info;
};
const box = document.createElement("div");
box.dataset.reasonixPick = "box";
box.setAttribute("aria-hidden", "true");
box.style.cssText =
"position:fixed;z-index:2147483646;pointer-events:none;border:2px solid #3b82f6;" +
"border-radius:3px;background:rgba(59,130,246,.12);box-sizing:border-box";
const card = document.createElement("div");
card.dataset.reasonixPick = "card";
card.setAttribute("aria-hidden", "true");
card.style.cssText =
"position:fixed;z-index:2147483647;pointer-events:none;background:#1f2328;color:#e6edf3;" +
"border-radius:8px;padding:9px 12px;font:12px/1.6 ui-sans-serif,system-ui,sans-serif;" +
"box-shadow:0 8px 24px rgba(0,0,0,.28);min-width:190px;max-width:340px";
// One card line. The label is the quiet side, the value the loud one; both
// are set as text, never as markup, because the page chose both.
const line = (label, value, loud) => {
const row = document.createElement("div");
row.style.cssText = "display:flex;gap:12px;justify-content:space-between;align-items:baseline";
const key = document.createElement("span");
key.textContent = label;
key.style.cssText = `flex:none;color:${loud ? "#e6edf3" : "#8b949e"};font-weight:${loud ? 600 : 400}`;
const val = document.createElement("span");
val.textContent = value;
val.style.cssText =
"color:#e6edf3;font-family:ui-monospace,SFMono-Regular,Menlo,monospace;overflow:hidden;" +
"text-overflow:ellipsis;white-space:nowrap" + (loud ? ";font-weight:600" : "");
row.append(key, val);
card.append(row);
};
let target = null;
// The chain of children walked up from, so the down keys can walk back.
let descended = [];
// Shift-click keeps a set: the person is naming a region, not one node.
const chosen = [];
const marks = [];
let armed = true;
let listeners = [];
const clear = () => {
for (const [type, handler, capture] of listeners) document.removeEventListener(type, handler, capture);
listeners = [];
for (const { cover } of frameCovers) cover.remove();
frameCovers.length = 0;
for (const at of marks) at.remove();
marks.length = 0;
box.remove();
card.remove();
if (window[KEY] && window[KEY].stop === stop) delete window[KEY];
};
const settle = (result) => {
if (!armed) return;
armed = false;
clear();
settlePick(result);
};
const stop = (reason = "dismissed") => settle({ cancelled: true, reason });
// Where the card sits: under the element when the viewport has room for it,
// above otherwise, and never past either edge.
// A frame's own document takes the pointer, so the top document never hears a
// move over it. A cover over each frame keeps the picker's events coming and
// says why that area is not pickable instead of going quiet.
const frameCovers = [];
const positionCovers = () => {
for (const { cover, frame } of frameCovers) {
const rect = frame.getBoundingClientRect();
cover.style.left = rect.left + "px";
cover.style.top = rect.top + "px";
cover.style.width = rect.width + "px";
cover.style.height = rect.height + "px";
cover.style.display = rect.width > 0 && rect.height > 0 ? "block" : "none";
}
};
const coverFrames = () => {
for (const { cover } of frameCovers) cover.remove();
frameCovers.length = 0;
for (const frame of document.querySelectorAll("iframe, frame")) {
const cover = document.createElement("div");
cover.dataset.reasonixFrame = "1";
cover.style.cssText =
"position:fixed;z-index:2147483646;pointer-events:auto;background:rgba(59,130,246,0.08);border:1px dashed rgba(59,130,246,0.55);box-sizing:border-box";
cover.addEventListener("pointermove", () => show(frame), true);
cover.addEventListener("click", (event) => {
swallow(event);
show(frame);
}, true);
document.documentElement.append(cover);
frameCovers.push({ cover, frame });
}
positionCovers();
};
const place = () => {
if (!target) return;
const rect = target.getBoundingClientRect();
const width = Math.max(0, rect.width);
const height = Math.max(0, rect.height);
box.style.left = rect.left + "px";
box.style.top = rect.top + "px";
box.style.width = width + "px";
box.style.height = height + "px";
card.style.visibility = "hidden";
card.style.left = "0px";
card.style.top = "0px";
const size = card.getBoundingClientRect();
const below = rect.bottom + 8;
const top = below + size.height <= innerHeight ? below : Math.max(8, rect.top - size.height - 8);
const left = Math.min(Math.max(8, rect.left), Math.max(8, innerWidth - size.width - 8));
card.style.left = left + "px";
card.style.top = top + "px";
card.style.visibility = "visible";
};
// The chain from the document down to the element, short enough to read in
// the card: a person walking up with the arrow keys needs to see where they
// are, not the whole tree.
const ancestorsOf = (el) => {
const chain = [];
for (let node = el; node && node.nodeType === 1; node = node.parentElement) {
chain.unshift(node);
if (chain.length > 8) break;
}
return chain;
};
const crumbLabel = (el) => el.tagName.toLowerCase() + (el.id ? "#" + el.id : "");
// A frame or a shadow host cannot be read from here: the picker only ever
// sees the host element, so it says so instead of handing over a useless
// answer that looks like the real thing.
const blockedOf = (el) => {
if (!el) return "";
if (el.tagName === "IFRAME" || el.tagName === "FRAME") return "iframe";
if (el.shadowRoot) return "shadow";
return "";
};
const show = (el) => {
target = el;
const rect = el.getBoundingClientRect();
const computed = getComputedStyle(el);
card.textContent = "";
const blocked = blockedOf(el);
if (blocked) {
const note = document.createElement("div");
note.textContent = blocked === "iframe" ? L.frame : L.shadow;
note.style.cssText = "max-width:260px;color:#e6edf3;font-size:12px;line-height:1.5";
card.append(note);
box.style.display = "block";
card.style.display = "block";
place();
return;
}
const crumb = document.createElement("div");
crumb.style.cssText = "display:flex;gap:4px;overflow:hidden;white-space:nowrap;margin-bottom:4px;color:#8b949e;font-size:11px";
const chain = ancestorsOf(el).slice(-5);
chain.forEach((node, index) => {
if (index) {
const sep = document.createElement("span");
sep.textContent = "›";
sep.style.opacity = "0.6";
crumb.append(sep);
}
const part = document.createElement("span");
part.textContent = crumbLabel(node);
if (node === el) {
part.style.color = "#e6edf3";
part.style.fontWeight = "600";
}
crumb.append(part);
});
card.append(crumb);
line(el.tagName.toLowerCase(), `${Math.round(rect.width)}×${Math.round(rect.height)}`, true);
line(L.color, hex(computed.color), false);
if (computed.backgroundColor && computed.backgroundColor !== "rgba(0, 0, 0, 0)" && computed.backgroundColor !== "transparent") {
line(L.background, hex(computed.backgroundColor), false);
}
line(L.font, `${computed.fontSize} ${computed.fontFamily}`, false);
box.style.display = "block";
card.style.display = "block";
place();
};
document.documentElement.append(box, card);
coverFrames();
box.style.display = "none";
card.style.display = "none";
// Every gesture is taken from the page while the pick is armed: a click
// chooses an element instead of pressing a button, and no page handler hears
// it either. Escape is the way out that changes nothing.
const swallow = (event) => {
event.preventDefault();
event.stopPropagation();
event.stopImmediatePropagation();
};
let moveAt = null;
let moveFrame = 0;
const onMove = (event) => {
// One hit test per frame: a page with a heavy layout is not asked to answer
// a move it will not draw.
moveAt = event;
if (moveFrame) return;
moveFrame = requestAnimationFrame(() => {
moveFrame = 0;
if (!armed) return;
const at = moveAt;
moveAt = null;
if (!at) return;
const el = document.elementFromPoint(at.clientX, at.clientY);
if (!el || el === box || el === card) return;
if (el.dataset && el.dataset.reasonixFrame) return;
if (el === target) return;
descended = [];
show(el);
});
};
// A picked element keeps a mark with its order, so a set stays visible while
// the person keeps adding to it.
const mark = (el) => {
const rect = el.getBoundingClientRect();
const at = document.createElement("div");
at.dataset.reasonixPicked = "1";
at.setAttribute("aria-hidden", "true");
at.style.cssText = `position:fixed;z-index:2147483645;pointer-events:none;border:1px solid #22c55e;background:rgba(34,197,94,0.12);box-sizing:border-box;left:${rect.left}px;top:${rect.top}px;width:${rect.width}px;height:${rect.height}px`;
const order = document.createElement("span");
order.textContent = String(marks.length + 1);
order.style.cssText = "position:absolute;top:-17px;left:-1px;background:#22c55e;color:#04170a;font:600 10px ui-sans-serif,system-ui,sans-serif;padding:1px 4px;border-radius:3px";
at.append(order);
document.documentElement.append(at);
marks.push(at);
};
const onPick = (event) => {
if (!armed) return;
swallow(event);
// The pointer may have climbed with the arrow keys: what is picked is the
// level the person is looking at, not what happens to be under the cursor.
try {
// A frame or shadow host is not an answer: the click leaves the pick
// armed while the card already says why.
if (target) {
if (blockedOf(target)) return;
if (event.shiftKey) {
chosen.push(target);
mark(target);
return;
}
return settle(chosen.length ? { cancelled: false, elements: [...chosen, target].map(describe) } : { cancelled: false, element: describe(target) });
}
const el = document.elementFromPoint(event.clientX, event.clientY);
if (!el) return settle({ cancelled: true });
if (blockedOf(el)) return;
if (event.shiftKey) {
chosen.push(el);
mark(el);
return;
}
settle(chosen.length ? { cancelled: false, elements: [...chosen, el].map(describe) } : { cancelled: false, element: describe(el) });
} catch (error) {
console.error("reasonix element pick failed", error);
settle({ cancelled: true, reason: "gone" });
}
};
const onKey = (event) => {
if (event.key === "Escape") {
swallow(event);
stop("escape");
return;
}
// Up walks to the parent and down back to the child it left, with the chain
// drawn in the card so neither move is a guess.
if (event.key === "ArrowUp" || event.key === "ArrowLeft") {
swallow(event);
const parent = target && target.parentElement;
if (parent && parent !== document.documentElement) {
descended.push(target);
show(parent);
}
return;
}
if (event.key === "ArrowDown" || event.key === "ArrowRight") {
swallow(event);
const child = descended.pop();
if (child && target && child.parentElement === target) show(child);
return;
}
// Every other key belongs to the picker as well: Enter on a focused button
// would run the page action a click is not allowed to run, and the pick is
// not using the page.
swallow(event);
};
const onScroll = () => {
place();
positionCovers();
};
for (const [type, handler, capture] of [
["pointerdown", swallow, true],
["pointerup", swallow, true],
["mousedown", swallow, true],
["mouseup", swallow, true],
["dblclick", swallow, true],
["contextmenu", swallow, true],
["click", onPick, true],
["pointermove", onMove, true],
["mousemove", onMove, true],
["keydown", onKey, true],
["scroll", onScroll, true],
["resize", onScroll, false],
]) {
document.addEventListener(type, handler, capture);
listeners.push([type, handler, capture]);
}
window[KEY] = { stop };
return answer;
}
// ARM runs the picker in the page's own world; the promise it answers with is
// what executeJavaScript returns, so a pick takes as long as the person does.
const ARM = (labels) => `(${pagePicker.toString()})(${JSON.stringify(labels || {})})`;
// STOP disarms whatever picker is armed, settling that pick as cancelled.
const STOP = `(() => { const picker = window.__reasonixElementPicker; if (picker) picker.stop(); })()`;
module.exports = { ARM, STOP };
+6 -9
View File
@@ -5,20 +5,17 @@
// compositor leaves uncovered at a fractional scale matches the app.
const GROUND = { light: "#f4f1ea", dark: "#060a10" };
function groundFor(prefs, systemDark, painted) {
const source = themeSourceFor(prefs, painted);
const dark = source === "system" ? systemDark : source === "dark";
// "auto" and anything unrecognised follow the system, as the page does.
function groundFor(prefs, systemDark) {
const theme = prefs?.["rx-theme"];
const dark = theme === "dark" ? true : theme === "light" ? false : systemDark;
return dark ? GROUND.dark : GROUND.light;
}
// The scheme the window frame takes. Windows draws a frameless window's border
// from it, so a dark page on a light system needs it set to stay borderless.
function themeSourceFor(prefs, painted) {
const theme = painted?.mode ?? prefs?.["rx-theme"];
if (theme === "scheduled") {
const scheme = painted?.scheme ?? prefs?.["rx-theme-resolved"];
return scheme === "dark" || scheme === "light" ? scheme : "system";
}
function themeSourceFor(prefs) {
const theme = prefs?.["rx-theme"];
return theme === "dark" || theme === "light" ? theme : "system";
}
-22
View File
@@ -1,22 +0,0 @@
const net = require("node:net");
function probeHost(origin, timeoutMs = 4000) {
return new Promise((resolve, reject) => {
const url = new URL(origin);
const socket = net.connect({ host: url.hostname, port: Number(url.port) });
const fail = (cause) => {
socket.destroy();
const err = new Error(`the host at ${origin} did not accept a connection from the shell (${cause})`);
err.code = "unreachable";
reject(err);
};
socket.setTimeout(timeoutMs, () => fail(`no answer in ${timeoutMs / 1000}s`));
socket.once("error", (e) => fail(e.code || e.message));
socket.once("connect", () => {
socket.destroy();
resolve();
});
});
}
module.exports = { probeHost };
+3 -19
View File
@@ -39,7 +39,6 @@ const { processMemoryHandler } = require("./procmem");
const { captureHandler } = require("./capture");
const { createPowerGuard, keepsAwake } = require("./powerguard");
const { openLogs, redactArgv, failStartup } = require("./shelllog");
const { probeHost } = require("./hostprobe");
const { raise } = require("./raise");
const { keepCredential, restoreIfStale } = require("./credential");
const { actHandler } = require("./acts");
@@ -224,7 +223,6 @@ cua = await startEmbedded(computerDriver, contract, text => logs.host.raw(text))
tray = installTray(client, { onOpen: showWindow, onQuit: () => app.quit() });
win.on("close", onWindowClose);
hostAgentBrowser();
await probeHost(origin);
await win.loadURL(origin + PAGE_PATH);
await tray?.refresh();
// After the window, deliberately. This asks about an install left behind by
@@ -339,12 +337,9 @@ function fitted() {
}
const storedPrefs = () => loadPrefs(prefsFile(app.getPath("userData")));
let paintedTheme = null;
const ground = () => groundFor(storedPrefs(), nativeTheme.shouldUseDarkColors, paintedTheme);
const ground = () => groundFor(storedPrefs(), nativeTheme.shouldUseDarkColors);
const followTheme = () => {
const source = themeSourceFor(storedPrefs(), paintedTheme);
if (nativeTheme.themeSource !== source) nativeTheme.themeSource = source;
if (win && !win.isDestroyed()) win.setBackgroundColor(ground());
nativeTheme.themeSource = themeSourceFor(storedPrefs());
};
function createWindow() {
@@ -405,11 +400,9 @@ const uiLang = () => uiLanguage(loadPrefs(prefsFile(app.getPath("userData"))), a
registerPrefs(ipcMain, () => prefsFile(app.getPath("userData")), fromWindow, () => {
installApplicationMenu(uiLang);
followTheme();
if (win && !win.isDestroyed()) win.setBackgroundColor(ground());
void powerGuard?.refresh();
browserViews?.setSleepAfterMs(pageSleepAfterMs(storedPrefs()));
}, paint => {
paintedTheme = paint;
followTheme();
});
nativeTheme.on("updated", () => {
if (win && !win.isDestroyed()) win.setBackgroundColor(ground());
@@ -453,15 +446,6 @@ ipcMain.handle("browser:control", (event, targetId, action) => {
ipcMain.handle("browser:navigate", (event, targetId, address) =>
fromWindow(event) ? (browserViews?.navigate(String(targetId), String(address)) ?? "scheme") : "scheme",
);
ipcMain.handle("browser:pick", (event, targetId, labels) =>
fromWindow(event) ? (browserViews?.pick(String(targetId), labels) ?? { cancelled: true }) : { cancelled: true },
);
ipcMain.handle("browser:pick-cancel", (event, targetId) =>
fromWindow(event) ? (browserViews?.cancelPick(String(targetId)) ?? false) : false,
);
ipcMain.handle("browser:verify", (event, targetId, selector) =>
fromWindow(event) ? (browserViews?.verify(String(targetId), String(selector)) ?? { found: false }) : { found: false },
);
ipcMain.handle("browser:trust-certificate", (event, targetId) =>
fromWindow(event) ? (browserViews?.trustCertificate(String(targetId)) ?? false) : false,
);
+1 -10
View File
@@ -52,7 +52,7 @@ function savePrefs(file, prefs) {
// in the same process and must not reach this file.
// fileOf is asked on each use: the profile directory is settled during boot,
// after these handlers exist.
function registerPrefs(ipcMain, fileOf, fromWindow, onSave = () => {}, onTheme = () => {}) {
function registerPrefs(ipcMain, fileOf, fromWindow, onSave = () => {}) {
ipcMain.on("prefs:load", (event) => {
event.returnValue = fromWindow(event) ? pickPagePrefs(loadPrefs(fileOf())) : {};
});
@@ -71,15 +71,6 @@ function registerPrefs(ipcMain, fileOf, fromWindow, onSave = () => {}, onTheme =
}
event.returnValue = true;
});
ipcMain.on("prefs:theme", (event, paint) => {
if (!fromWindow(event) || !event.senderFrame || event.senderFrame !== event.sender.mainFrame) return;
if (!paint || typeof paint !== "object" || Array.isArray(paint)) return;
const { mode, scheme } = paint;
if (mode !== "auto" && mode !== "scheduled" && mode !== "light" && mode !== "dark") return;
if (scheme !== "light" && scheme !== "dark") return;
if ((mode === "light" || mode === "dark") && mode !== scheme) return;
onTheme({ mode, scheme });
});
}
function pickPagePrefs(prefs) {
-8
View File
@@ -22,7 +22,6 @@ contextBridge.exposeInMainWorld("reasonixHost", {
platform: process.platform,
titleBar,
setUnreadBadge: (unread) => ipcRenderer.invoke("window:unread-badge", unread),
syncTheme: (mode, scheme) => ipcRenderer.send("prefs:theme", { mode, scheme }),
minimiseWindow: () => ipcRenderer.invoke("window:minimise"),
toggleMaximiseWindow: () => ipcRenderer.invoke("window:toggle-maximise"),
isWindowMaximised: () => ipcRenderer.invoke("window:is-maximised"),
@@ -65,17 +64,10 @@ contextBridge.exposeInMainWorld("reasonixHost", {
freezeBrowserView: () => ipcRenderer.invoke("browser:freeze"),
controlBrowserView: (targetId, action) => ipcRenderer.invoke("browser:control", String(targetId), String(action)),
navigateBrowserView: (targetId, address) => ipcRenderer.invoke("browser:navigate", String(targetId), String(address)),
// Arm this page's element picker, and let a person put it away again. The
// answer is the element they pointed at, or that they picked nothing.
pickBrowserElement: (targetId, labels) => ipcRenderer.invoke("browser:pick", String(targetId), labels),
cancelBrowserPick: (targetId) => ipcRenderer.invoke("browser:pick-cancel", String(targetId)),
verifyBrowserElement: (targetId, selector) => ipcRenderer.invoke("browser:verify", String(targetId), String(selector)),
// Why a page did not load, and a login a page or proxy asks for. The password
// goes back to this window's main process only; the kernel never sees it.
onBrowserLoadState: (listener) => subscribe("browser:load-state", listener),
onBrowserLogin: (listener) => subscribe("browser:login", listener),
// The picker's key, pressed in the page rather than in this one.
onBrowserPickShortcut: (listener) => subscribe("browser:pick-shortcut", listener),
trustBrowserCertificate: (targetId) => ipcRenderer.invoke("browser:trust-certificate", String(targetId)),
answerBrowserLogin: (id, username, password) =>
ipcRenderer.invoke("browser:login-answer", String(id), String(username || ""), String(password || "")),
Loaded 100 of 2372 files, more files were not shown because too many files have changed in this diff. Show more