mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-09 17:11:47 -10:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 esengine
|
4effa7a785 |
No files matched your search
@@ -1,7 +0,0 @@
|
||||
# Logins allowed to open pull requests, one per line, matched case-insensitively.
|
||||
# Repository owners, members and collaborators, and bots, are always allowed.
|
||||
lancy69
|
||||
chuiwenwei
|
||||
xxoingr
|
||||
kevin9327
|
||||
imcreazy
|
||||
@@ -2,9 +2,6 @@
|
||||
Target branch: 2.x features and fixes go to `studio`. 1.x takes bug fixes,
|
||||
provider/API compatibility, release/updater, security and platform stability
|
||||
only, on `main-v2`.
|
||||
|
||||
Pull requests are limited to maintainers and invited contributors; one from
|
||||
anyone else is closed automatically. Open an issue instead.
|
||||
-->
|
||||
|
||||
## Summary
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
[
|
||||
{
|
||||
"package": "reasonix/internal/runtime/delegation",
|
||||
"tests": [
|
||||
"TestAncestorCopyAdmissionCoversUnpublishedRun",
|
||||
"TestAncestorCopyAdmissionCoversUnpublishedRun/continue_then_continue",
|
||||
"TestAncestorCopyAdmissionCoversUnpublishedRun/continue_then_legacy",
|
||||
"TestAncestorCopyAdmissionCoversUnpublishedRun/legacy_then_continue",
|
||||
"TestAncestorCopyAdmissionCoversUnpublishedRun/legacy_then_legacy",
|
||||
"TestAncestorCopyAdmissionAllowsDifferentParents",
|
||||
"TestAncestorCopyAdmissionReleasesAbandonedPreparation",
|
||||
"TestAncestorCopyAdmissionReleasesAfterInvalidCopy",
|
||||
"TestAncestorCopyAdmissionAllowsDifferentSources",
|
||||
"TestAncestorCopyAdmissionUsesNearestSourceIdentity",
|
||||
"TestAncestorCopyAdmissionReleasesAfterBusyCurrentReference",
|
||||
"TestTaskAncestorCopyAdmissionBeforeRunningMetadata",
|
||||
"TestTaskAncestorCopyAdmissionBeforeRunningMetadata/continue_from",
|
||||
"TestTaskAncestorCopyAdmissionBeforeRunningMetadata/fork_from",
|
||||
"TestTaskAncestorCopyAdmissionReleasesUnstartedBackground",
|
||||
"TestTaskAncestorCopyAdmissionReleasesUnstartedBackground/continue_from",
|
||||
"TestTaskAncestorCopyAdmissionReleasesUnstartedBackground/fork_from"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,26 @@
|
||||
[
|
||||
{
|
||||
"package": "reasonix/internal/runtime/agent",
|
||||
"tests": [
|
||||
"TestCarriedProofInvalidatesPermissionChange",
|
||||
"TestCarriedProofFileStateCompatibility",
|
||||
"TestCarriedProofFileStateCompatibility/legacy_file",
|
||||
"TestCarriedProofFileStateCompatibility/legacy_absent",
|
||||
"TestCarriedProofFileStateCompatibility/future",
|
||||
"TestCarriedProofFileStateCompatibility/current_file",
|
||||
"TestCarriedProofFileStateCompatibility/current_absent",
|
||||
"TestCarriedProofRejectsDirectoryState",
|
||||
"TestCarriedProofStandsOnlyForUnchangedContent",
|
||||
"TestCarriedProofSeesADeletion"
|
||||
]
|
||||
},
|
||||
{
|
||||
"package": "reasonix/internal/assembly/boot",
|
||||
"tests": [
|
||||
"TestEffectCarriedProofTracksFileState",
|
||||
"TestEffectCarriedProofTracksFileState/unchanged",
|
||||
"TestEffectCarriedProofTracksFileState/content",
|
||||
"TestEffectCarriedProofTracksFileState/mode"
|
||||
]
|
||||
}
|
||||
]
|
||||
+304
-177
@@ -15,7 +15,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.ref == 'refs/heads/studio' }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
# Cheap path gate for pull requests. PRs confined to docs/release-notes
|
||||
@@ -170,6 +170,23 @@ jobs:
|
||||
shell: pwsh
|
||||
run: Get-CimInstance Win32_Processor | Select-Object -ExpandProperty Name
|
||||
|
||||
- name: test (Windows concurrency contracts)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
node --test scripts/windows-concurrency-tests.test.mjs
|
||||
node scripts/windows-concurrency-tests.mjs "$RUNNER_TEMP/concurrency-windows.jsonl"
|
||||
|
||||
- name: upload Windows concurrency inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: concurrency-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/concurrency-windows.jsonl
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: test
|
||||
if: env.RUN_STEPS == 'true' && runner.os != 'Windows'
|
||||
env:
|
||||
@@ -179,6 +196,35 @@ jobs:
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
run: go test ./...
|
||||
|
||||
- name: test (Windows subagent status admission)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^TestSubagent(PersistedStatusAdmission|UnknownStatusStillReadableForDiagnosis)$' ./internal/runtime/delegation | tee "$RUNNER_TEMP/subagent-status-admission.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
["future-paused", "missing", "empty", "COMPLETED", "completed", "cancelled", "failed", "interrupted", "running"] as $states |
|
||||
["continue", "copy", "ancestor", "legacy_ancestor", "current_copy"] as $routes |
|
||||
(["TestSubagentPersistedStatusAdmission", "TestSubagentUnknownStatusStillReadableForDiagnosis"] +
|
||||
[$states[] | "TestSubagentPersistedStatusAdmission/" + .] +
|
||||
[$states[] as $state | $routes[] | "TestSubagentPersistedStatusAdmission/" + $state + "/" + .]) as $required |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/delegation"] - $packages | length) == 0 and
|
||||
($required - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/subagent-status-admission.jsonl"
|
||||
|
||||
- name: upload Windows subagent status admission inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: subagent-status-admission-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/subagent-status-admission.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows composition links)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
@@ -255,30 +301,23 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -p 4 -timeout=18m -json ./internal/platform/worktree/... ./internal/state/mutation/... | tee "$RUNNER_TEMP/isolation-owners.jsonl" |
|
||||
go test -count=1 -p 4 -timeout=18m -json ./internal/runtime/isolation/... ./internal/platform/worktree/... ./internal/state/mutation/... | tee "$RUNNER_TEMP/isolation-owners.jsonl" |
|
||||
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $passed |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
(["reasonix/internal/platform/worktree", "reasonix/internal/state/mutation"] - $passed | length) == 0 and
|
||||
(["TestHeldPathsCoverOnlyTheSameFileAcrossCaseSpellings",
|
||||
(["reasonix/internal/runtime/isolation", "reasonix/internal/platform/worktree", "reasonix/internal/state/mutation"] - $passed | length) == 0 and
|
||||
(["TestResultOperationUsesCrossProcessOwnership",
|
||||
"TestActualResultOperationHoldsOwnershipThroughWriteAndCleanup/restore",
|
||||
"TestActualResultOperationHoldsOwnershipThroughWriteAndCleanup/remove",
|
||||
"TestApplyPathsWithUntrustedModesPreservesTheIndex/content",
|
||||
"TestApplyPathsWithUntrustedModesPreservesTheIndex/source_index_mode",
|
||||
"TestApplyPathsWithUntrustedModesPreservesTheIndex/candidate_mode",
|
||||
"TestApplyPathsWithUntrustedModesPreservesTheIndex/candidate_type",
|
||||
"TestApplyPathsEmulatedLinksUseSnapshotFilters",
|
||||
"TestHeldPathsCoverOnlyTheSameFileAcrossCaseSpellings",
|
||||
"TestLegacyWaitDoesNotConsumeExecutionCapacity"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/isolation-owners.jsonl"
|
||||
jq -se '
|
||||
. as $events |
|
||||
{"reasonix/internal/platform/worktree": [
|
||||
"TestWriteChangesRejectsRedirectedDeletion",
|
||||
"TestWriteChangesUnlinksLeafDirectoryLink"
|
||||
]} |
|
||||
to_entries | all(.[]; . as $package |
|
||||
([ $events[] | select(.Package == $package.key and .Test == null and
|
||||
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
|
||||
length == 1 and .[0].Action == "pass") and
|
||||
all(.value[]; . as $test |
|
||||
[ $events[] | select(.Package == $package.key and .Test == $test and
|
||||
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
|
||||
length == 1 and .[0].Action == "pass"))
|
||||
' "$RUNNER_TEMP/isolation-owners.jsonl"
|
||||
|
||||
- name: upload Windows isolation test inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
@@ -289,15 +328,60 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows background resource owners)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 25
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -p 2 -timeout=18m -json ./internal/tools/jobs/... ./internal/runtime/delegation/... | tee "$RUNNER_TEMP/background-resources.jsonl" |
|
||||
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
(["reasonix/internal/tools/jobs", "reasonix/internal/runtime/delegation"] - $packages | length) == 0 and
|
||||
(["TestJobFinalizationCoversInvokedAndUninvoked/success",
|
||||
"TestJobFinalizationCoversInvokedAndUninvoked/error",
|
||||
"TestJobFinalizationCoversInvokedAndUninvoked/panic",
|
||||
"TestJobFinalizationCoversInvokedAndUninvoked/closed",
|
||||
"TestJobFinalizationCoversInvokedAndUninvoked/invalid_session",
|
||||
"TestJobFinalizationCoversInvokedAndUninvoked/invalid_kind",
|
||||
"TestJobFinalizationFailuresAttemptAllCallbacksAndKeepCauses/invoked",
|
||||
"TestJobFinalizationFailuresAttemptAllCallbacksAndKeepCauses/uninvoked",
|
||||
"TestJobFinalizationPrecedesCompletionPublication",
|
||||
"TestJobFinalizationCloseBetweenPublicationAndLaunch",
|
||||
"TestJobFinalizationWaitsForClosureAfterLogicalCancellation",
|
||||
"TestBackgroundNoLaunchFinalizesOwnedResources/task/closed_manager",
|
||||
"TestBackgroundNoLaunchFinalizesOwnedResources/task/invalid_session",
|
||||
"TestBackgroundNoLaunchFinalizesOwnedResources/fleet/closed_manager",
|
||||
"TestBackgroundNoLaunchFinalizesOwnedResources/fleet/invalid_session"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/background-resources.jsonl"
|
||||
|
||||
- name: upload Windows background resource inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: background-resources-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/background-resources.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows delegated Goal usage scope)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(UsageScope.*)$' ./internal/contract/event ./internal/session/control ./internal/assembly/boot | tee "$RUNNER_TEMP/goal-usage-scope.jsonl"
|
||||
go test -count=1 -timeout=3m -json -run '^Test(UsageScope.*|DelegatedUsageKeepsGoalOwner|EffectDelegatedUsageKeepsGoalOwner)$' ./internal/contract/event ./internal/session/control ./internal/assembly/boot | tee "$RUNNER_TEMP/goal-usage-scope.jsonl"
|
||||
jq -se '
|
||||
["TestUsageScopeKeepsEarlierEventOwner",
|
||||
"TestUsageScopeWithoutCapability" ] as $required |
|
||||
["TestUsageScopePreservesCapturedOwnerAndWire", "TestUsageScopeKeepsEarlierEventOwner",
|
||||
"TestUsageScopeWithoutCapability", "TestDelegatedUsageKeepsGoalOwner",
|
||||
"TestDelegatedUsageKeepsGoalOwner/chat_to_goal", "TestDelegatedUsageKeepsGoalOwner/replaced_goal",
|
||||
"TestDelegatedUsageKeepsGoalOwner/cleared_goal", "TestDelegatedUsageKeepsGoalOwner/resumed_goal",
|
||||
"TestDelegatedUsageKeepsGoalOwner/same_goal_next_turn", "TestDelegatedUsageKeepsGoalOwner/between_turns",
|
||||
"TestDelegatedUsageKeepsGoalOwner/nested_after_replacement", "TestEffectDelegatedUsageKeepsGoalOwner",
|
||||
"TestEffectDelegatedUsageKeepsGoalOwner/chat_to_goal", "TestEffectDelegatedUsageKeepsGoalOwner/replaced_goal",
|
||||
"TestEffectDelegatedUsageKeepsGoalOwner/nested_chat_to_goal", "TestEffectDelegatedUsageKeepsGoalOwner/same_goal",
|
||||
"TestEffectDelegatedUsageKeepsGoalOwner/completed_before_goal"] as $required |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
all(.[]; .Action != "fail" and .Action != "skip") and
|
||||
@@ -314,19 +398,60 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows planner local budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^TestEffectPlannerConfiguredTaskBudget$' ./internal/assembly/boot | tee "$RUNNER_TEMP/planner-local-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestEffectPlannerConfiguredTaskBudget", "TestEffectPlannerConfiguredTaskBudget/tokens",
|
||||
"TestEffectPlannerConfiguredTaskBudget/cost", "TestEffectPlannerConfiguredTaskBudget/wall",
|
||||
"TestEffectPlannerConfiguredTaskBudget/unconfigured", "TestEffectPlannerConfiguredTaskBudget/explicit_zero",
|
||||
"TestEffectPlannerConfiguredTaskBudget/unpriced_cost", "TestEffectPlannerConfiguredTaskBudget/missing_usage_cost",
|
||||
"TestEffectPlannerConfiguredTaskBudget/context_bound", "TestEffectPlannerConfiguredTaskBudget/context_disables",
|
||||
"TestEffectPlannerConfiguredTaskBudget/executor_only"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/planner-local-budget.jsonl"
|
||||
|
||||
- name: upload Windows planner local budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: planner-local-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/planner-local-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows Controller routing root budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(SemanticRootUsage)$' ./internal/runtime/capability | tee "$RUNNER_TEMP/controller-routing-budget.jsonl"
|
||||
go test -count=1 -timeout=3m -json -run '^Test(SemanticRootUsage|EffectRoutingRootBudget)$' ./internal/runtime/capability ./internal/assembly/boot | tee "$RUNNER_TEMP/controller-routing-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/capability"] - $packages | length) == 0 and
|
||||
(["reasonix/internal/runtime/capability", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
([
|
||||
"TestEffectRoutingRootBudget",
|
||||
"TestEffectRoutingRootBudget/balanced",
|
||||
"TestEffectRoutingRootBudget/cached_continuation",
|
||||
"TestEffectRoutingRootBudget/cached_new_scope",
|
||||
"TestEffectRoutingRootBudget/cached_new_task",
|
||||
"TestEffectRoutingRootBudget/cached_reset",
|
||||
"TestEffectRoutingRootBudget/context_zero",
|
||||
"TestEffectRoutingRootBudget/dual",
|
||||
"TestEffectRoutingRootBudget/headless",
|
||||
"TestEffectRoutingRootBudget/interactive",
|
||||
"TestEffectRoutingRootBudget/missing",
|
||||
"TestEffectRoutingRootBudget/unconfigured",
|
||||
"TestSemanticRootUsage",
|
||||
"TestSemanticRootUsage/billed_error",
|
||||
"TestSemanticRootUsage/cache",
|
||||
@@ -354,17 +479,22 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(PreparedRootBudget|CoordinatorRootBudget|EffectCoordinatorRoot)' ./internal/runtime/agent ./internal/runtime/coordinator | tee "$RUNNER_TEMP/coordinator-root-budget.jsonl"
|
||||
go test -count=1 -timeout=3m -json -run '^Test(PreparedRootBudget|CoordinatorRootBudget|EffectCoordinatorRoot)' ./internal/runtime/agent ./internal/runtime/coordinator ./internal/assembly/boot | tee "$RUNNER_TEMP/coordinator-root-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/agent", "reasonix/internal/runtime/coordinator"] - $packages | length) == 0 and
|
||||
(["TestCoordinatorRootBudgetRoutes",
|
||||
(["reasonix/internal/runtime/agent", "reasonix/internal/runtime/coordinator", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestPreparedRootBudgetLifetime", "TestPreparedRootBudgetLifetime/continuation",
|
||||
"TestPreparedRootBudgetLifetime/new_scope", "TestPreparedRootBudgetLifetime/new_task",
|
||||
"TestPreparedRootBudgetLifetime/reset", "TestPreparedRootBudgetLifetime/new_session",
|
||||
"TestPreparedRootBudgetProjectionAndPreservedTurn", "TestCoordinatorRootBudgetRoutes",
|
||||
"TestCoordinatorRootBudgetRoutes/raw", "TestCoordinatorRootBudgetRoutes/tool_planner",
|
||||
"TestCoordinatorRootBudgetRoutes/failure_fallback", "TestCoordinatorRootBudgetRoutes/plan_only_continuation",
|
||||
"TestCoordinatorRootBudgetRoutes/approval_declined_continuation", "TestCoordinatorRootBudgetRoutes/approval_allowed"
|
||||
] - $tests | length) == 0
|
||||
"TestCoordinatorRootBudgetRoutes/approval_declined_continuation", "TestCoordinatorRootBudgetRoutes/approval_allowed",
|
||||
"TestEffectCoordinatorRootTokenBudget", "TestEffectCoordinatorRootTokenBudget/shared",
|
||||
"TestEffectCoordinatorRootTokenBudget/unconfigured", "TestEffectCoordinatorRootTokenBudget/context_disables",
|
||||
"TestEffectCoordinatorRootTokenBudget/missing_planner", "TestEffectCoordinatorRootTokenBudget/executor_only"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/coordinator-root-budget.jsonl"
|
||||
|
||||
- name: upload Windows Coordinator root budget inventory
|
||||
@@ -422,8 +552,10 @@ jobs:
|
||||
"TestRootWallBudgetReachesDelegatedAgents/context", "TestRootWallBudgetReachesDelegatedAgents/unconfigured",
|
||||
"TestRootWallBudgetReachesDelegatedAgents/explicit_zero", "TestRootWallBudgetContinuationAndReset",
|
||||
"TestRootWallBudgetContinuationAndReset/continuation", "TestRootWallBudgetContinuationAndReset/new_task",
|
||||
"TestRootWallBudgetContinuationAndReset/reset", "TestRootWallBudgetContinuationAndReset/override"
|
||||
] - $tests | length) == 0
|
||||
"TestRootWallBudgetContinuationAndReset/reset", "TestRootWallBudgetContinuationAndReset/override",
|
||||
"TestRootWallBudgetProjectionKeepsJobDeadline", "TestEffectRootWallBudgetReachesManagedJobs",
|
||||
"TestEffectRootWallBudgetReachesManagedJobs/task", "TestEffectRootWallBudgetReachesManagedJobs/capability",
|
||||
"TestEffectRootWallBudgetReachesManagedJobs/fleet"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-wall-budget.jsonl"
|
||||
|
||||
- name: upload Windows root wall budget inventory
|
||||
@@ -441,19 +573,22 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootToken|EffectRootToken|BudgetCountsKnownReportedTokens|ReportedToken)' ./internal/runtime/agent | tee "$RUNNER_TEMP/root-token-budget.jsonl"
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootToken|EffectRootToken|BudgetCountsKnownReportedTokens|ReportedToken|EffectReportedTotalTokenBudget)' ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/root-token-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/agent"] - $packages | length) == 0 and
|
||||
(["reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestRootTokenBudgetIncludesChildProviderUsage", "TestRootTokenBudgetIncludesChildProviderUsage/configured",
|
||||
"TestRootTokenBudgetIncludesChildProviderUsage/context", "TestRootTokenBudgetIncludesChildProviderUsage/unconfigured",
|
||||
"TestRootTokenBudgetIncludesChildProviderUsage/explicit_zero", "TestRootTokenBudgetSurvivesNestedAndConcurrentChildren",
|
||||
"TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/nested", "TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/siblings",
|
||||
"TestRootTokenBudgetSurvivesNestedAndConcurrentChildren/child_override", "TestRootTokenBudgetContinuationAndReset",
|
||||
"TestRootTokenBudgetContinuationAndReset/continuation", "TestRootTokenBudgetContinuationAndReset/new_task",
|
||||
"TestRootTokenBudgetContinuationAndReset/reset" ] - $tests | length) == 0
|
||||
"TestRootTokenBudgetContinuationAndReset/reset", "TestRootTokenScopeAtomicAccountingAndJobCancellation",
|
||||
"TestEffectRootTokenBudgetReachesDelegatedJobs", "TestEffectRootTokenBudgetReachesDelegatedJobs/foreground",
|
||||
"TestEffectRootTokenBudgetReachesDelegatedJobs/foreground_capability", "TestEffectRootTokenBudgetReachesDelegatedJobs/background",
|
||||
"TestEffectRootTokenBudgetReachesDelegatedJobs/background_capability", "TestEffectRootTokenBudgetReachesDelegatedJobs/background_fleet"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-token-budget.jsonl"
|
||||
jq -se '
|
||||
. as $events |
|
||||
@@ -467,7 +602,17 @@ jobs:
|
||||
"TestBudgetCountsKnownReportedTokens/negative_split",
|
||||
"TestBudgetCountsKnownReportedTokens/zero",
|
||||
"TestBudgetCountsKnownReportedTokens/negative",
|
||||
"TestReportedTokenCountersSaturate"]} |
|
||||
"TestReportedTokensEnforceLocalCapOnInheritedRoot",
|
||||
"TestReportedTokensFollowOriginalAuxiliaryRoot",
|
||||
"TestReportedTokenCountersSaturate"],
|
||||
"reasonix/internal/assembly/boot": [
|
||||
"TestEffectReportedTotalTokenBudget",
|
||||
"TestEffectReportedTotalTokenBudget/split",
|
||||
"TestEffectReportedTotalTokenBudget/total_only",
|
||||
"TestEffectReportedTotalTokenBudget/partial_split",
|
||||
"TestEffectReportedTotalTokenBudget/lower_total",
|
||||
"TestEffectReportedTotalTokenBudget/missing_total",
|
||||
"TestEffectReportedTotalTokenBudget/disabled"]} |
|
||||
to_entries | all(.[]; . as $package |
|
||||
([ $events[] | select(.Package == $package.key and .Test == null and
|
||||
(.Action == "pass" or .Action == "fail" or .Action == "skip")) ] |
|
||||
@@ -526,7 +671,7 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(HostStorageEnvironment|HostObjectStorage|HostExplicitStorageControls|DetachedStorageEnvironment)$' ./internal/platform/gitcmd ./internal/platform/worktree ./internal/assembly/boot | tee "$RUNNER_TEMP/git-storage-environment.jsonl"
|
||||
go test -count=1 -timeout=3m -json -run '^Test(HostStorageEnvironment|HostObjectStorage|HostExplicitStorageControls|DetachedStorageEnvironment|WorktreePreparationOwnsItsIndex|EffectIsolatedTaskOwnsGitStorage)$' ./internal/platform/gitcmd ./internal/platform/worktree ./internal/assembly/boot | tee "$RUNNER_TEMP/git-storage-environment.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
@@ -536,6 +681,10 @@ jobs:
|
||||
"TestDetachedStorageEnvironment",
|
||||
"TestDetachedStorageEnvironment/explicit_env",
|
||||
"TestDetachedStorageEnvironment/inherited_env",
|
||||
"TestEffectIsolatedTaskOwnsGitStorage",
|
||||
"TestEffectIsolatedTaskOwnsGitStorage/ambient_existing",
|
||||
"TestEffectIsolatedTaskOwnsGitStorage/ambient_missing",
|
||||
"TestEffectIsolatedTaskOwnsGitStorage/control",
|
||||
"TestHostExplicitStorageControls",
|
||||
"TestHostExplicitStorageControls/explicit_object_directory",
|
||||
"TestHostExplicitStorageControls/private_index",
|
||||
@@ -546,8 +695,15 @@ jobs:
|
||||
"TestHostStorageEnvironment",
|
||||
"TestHostStorageEnvironment/GIT_ALTERNATE_OBJECT_DIRECTORIES",
|
||||
"TestHostStorageEnvironment/GIT_INDEX_FILE",
|
||||
"TestHostStorageEnvironment/GIT_OBJECT_DIRECTORY"
|
||||
] - $tests | length) == 0
|
||||
"TestHostStorageEnvironment/GIT_OBJECT_DIRECTORY",
|
||||
"TestWorktreePreparationOwnsItsIndex",
|
||||
"TestWorktreePreparationOwnsItsIndex/candidate_ambient_existing",
|
||||
"TestWorktreePreparationOwnsItsIndex/candidate_ambient_missing",
|
||||
"TestWorktreePreparationOwnsItsIndex/candidate_control",
|
||||
"TestWorktreePreparationOwnsItsIndex/delivery_ambient_existing",
|
||||
"TestWorktreePreparationOwnsItsIndex/delivery_ambient_missing",
|
||||
"TestWorktreePreparationOwnsItsIndex/delivery_control"
|
||||
] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/git-storage-environment.jsonl"
|
||||
|
||||
- name: upload Windows Git storage environment inventory
|
||||
@@ -597,19 +753,20 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootWebSearch|EffectRootWebSearch)' ./internal/tools/websearch | tee "$RUNNER_TEMP/root-websearch-budget.jsonl"
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootWebSearch|EffectRootWebSearch)' ./internal/tools/websearch ./internal/assembly/boot | tee "$RUNNER_TEMP/root-websearch-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/tools/websearch"] - $packages | length) == 0 and
|
||||
(["reasonix/internal/tools/websearch", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestRootWebSearchUsageTerminal", "TestRootWebSearchUsageTerminal/reported",
|
||||
"TestRootWebSearchUsageTerminal/no_reporter", "TestRootWebSearchUsageTerminal/missing",
|
||||
"TestRootWebSearchUsageTerminal/no_observer", "TestRootWebSearchUsageTerminal/input_refused",
|
||||
"TestRootWebSearchUsageTerminal/no_factory", "TestRootWebSearchUsageTerminal/factory_error",
|
||||
"TestRootWebSearchUsageTerminal/billed_failure", "TestRootWebSearchUsageTerminal/cancel_after_usage",
|
||||
"TestRootWebSearchUsageTerminal/cancel_before_start", "TestRootWebSearchUsageTerminal/stream_error"
|
||||
] - $tests | length) == 0
|
||||
"TestRootWebSearchUsageTerminal/cancel_before_start", "TestRootWebSearchUsageTerminal/stream_error",
|
||||
"TestEffectRootWebSearchTokenBudget", "TestEffectRootWebSearchTokenBudget/reported",
|
||||
"TestEffectRootWebSearchTokenBudget/disabled", "TestEffectRootWebSearchTokenBudget/missing"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-websearch-budget.jsonl"
|
||||
|
||||
- name: upload Windows root websearch budget inventory
|
||||
@@ -627,13 +784,21 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^(TestProfileRootBudgetLifetime|TestProfileBudgetPreparationUsesTurnAdmission|TestEffectExplicitSkillRootBudget)$' ./internal/session/control | tee "$RUNNER_TEMP/explicit-skill-root-budget.jsonl"
|
||||
go test -count=1 -timeout=3m -json -run '^(TestProfileRootBudgetLifetime|TestProfileBudgetPreparationUsesTurnAdmission|TestEffectExplicitSkillRootBudget)$' ./internal/session/control ./internal/assembly/boot | tee "$RUNNER_TEMP/explicit-skill-root-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/session/control"] - $packages | length) == 0 and
|
||||
(["TestProfileBudgetPreparationUsesTurnAdmission" ] - $tests | length) == 0
|
||||
(["reasonix/internal/session/control", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestProfileRootBudgetLifetime", "TestProfileRootBudgetLifetime/new_task",
|
||||
"TestProfileRootBudgetLifetime/same_scope", "TestProfileRootBudgetLifetime/new_scope",
|
||||
"TestProfileRootBudgetLifetime/reset", "TestProfileRootBudgetLifetime/new_session",
|
||||
"TestProfileRootBudgetLifetime/inherited", "TestProfileRootBudgetLifetime/context_zero",
|
||||
"TestProfileBudgetPreparationUsesTurnAdmission", "TestEffectExplicitSkillRootBudget",
|
||||
"TestEffectExplicitSkillRootBudget/profile", "TestEffectExplicitSkillRootBudget/preview",
|
||||
"TestEffectExplicitSkillRootBudget/slash", "TestEffectExplicitSkillRootBudget/chips",
|
||||
"TestEffectExplicitSkillRootBudget/goal", "TestEffectExplicitSkillRootBudget/context_zero",
|
||||
"TestEffectExplicitSkillRootBudget/unconfigured"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/explicit-skill-root-budget.jsonl"
|
||||
|
||||
- name: upload Windows explicit skill root budget inventory
|
||||
@@ -651,16 +816,19 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootTriage|EffectRootTriage)' ./internal/runtime/agent | tee "$RUNNER_TEMP/root-triage-budget.jsonl"
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootTriage|EffectRootTriage)' ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/root-triage-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/agent"] - $packages | length) == 0 and
|
||||
(["reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestRootTriageTokenBudget", "TestRootTriageTokenBudget/classifier", "TestRootTriageTokenBudget/screen",
|
||||
"TestRootTriageTokenBudget/billed_failure", "TestRootTriageTokenBudget/missing", "TestRootTriageTokenBudget/disabled",
|
||||
"TestRootTriageTokenBudget/no_sink", "TestRootTriageTokenBudget/counts_without_total",
|
||||
"TestRootTriageOutsideRunDoesNotCreateScope" ] - $tests | length) == 0
|
||||
"TestRootTriageOutsideRunDoesNotCreateScope", "TestEffectRootTriageScreenBudget",
|
||||
"TestEffectRootTriageScreenBudget/reported", "TestEffectRootTriageScreenBudget/flagged",
|
||||
"TestEffectRootTriageScreenBudget/disabled", "TestEffectRootTriageScreenBudget/missing",
|
||||
"TestEffectRootTriageScreenBudget/screen_disabled"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-triage-budget.jsonl"
|
||||
|
||||
- name: upload Windows root triage budget inventory
|
||||
@@ -672,22 +840,54 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows root auxiliary budget)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(RootAuxiliary|EffectRootAuxiliary)' ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/root-auxiliary-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestRootAuxiliaryAutomaticCompaction", "TestRootAuxiliaryTokenBudget",
|
||||
"TestRootAuxiliaryTokenBudget/summary", "TestRootAuxiliaryTokenBudget/summary_failed",
|
||||
"TestRootAuxiliaryTokenBudget/summary_missing", "TestRootAuxiliaryTokenBudget/summary_disabled",
|
||||
"TestRootAuxiliaryTokenBudget/diagnosis", "TestRootAuxiliaryTokenBudget/diagnosis_failed",
|
||||
"TestRootAuxiliaryTokenBudget/diagnosis_missing", "TestRootAuxiliaryTokenBudget/diagnosis_disabled",
|
||||
"TestRootAuxiliaryManualCompactionOutsideRun", "TestRootAuxiliaryBudgetProjectionKeepsOldTask",
|
||||
"TestEffectRootAuxiliarySummaryBudget", "TestEffectRootAuxiliarySummaryBudget/configured",
|
||||
"TestEffectRootAuxiliarySummaryBudget/disabled", "TestEffectRootAuxiliarySummaryBudget/missing_usage"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/root-auxiliary-budget.jsonl"
|
||||
|
||||
- name: upload Windows root auxiliary budget inventory
|
||||
if: always() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: root-auxiliary-budget-windows-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/root-auxiliary-budget.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: test (Windows local cost quote)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(LocalBudgetUsesOccurrenceQuote|LocalCost.*|UsageQuote.*)$' ./internal/contract/event ./internal/runtime/agent | tee "$RUNNER_TEMP/local-cost-quote.jsonl"
|
||||
go test -count=1 -timeout=3m -json -run '^Test(LocalBudgetUsesOccurrenceQuote|LocalCost.*|UsageQuote.*|EffectLocalCostUsesOccurrenceQuote)$' ./internal/contract/event ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/local-cost-quote.jsonl"
|
||||
jq -se '
|
||||
["TestLocalBudgetUsesOccurrenceQuote", "TestLocalBudgetUsesOccurrenceQuote/peak", "TestLocalBudgetUsesOccurrenceQuote/offpeak",
|
||||
"TestLocalCostKeepsOriginalCurrencyBuckets", "TestLocalCostFixedPointThresholdAndUnknown",
|
||||
"TestUsageQuoteUsesOneOwnerAndOccurrence", "TestUsageQuoteLegacyAndNilSinks"
|
||||
] as $required |
|
||||
"TestUsageQuoteUsesOneOwnerAndOccurrence", "TestUsageQuoteLegacyAndNilSinks",
|
||||
"TestEffectLocalCostUsesOccurrenceQuote", "TestEffectLocalCostUsesOccurrenceQuote/peak",
|
||||
"TestEffectLocalCostUsesOccurrenceQuote/offpeak", "TestEffectLocalCostUsesOccurrenceQuote/relay"] as $required |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $passed |
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
all(.[]; .Action != "fail" and .Action != "skip") and
|
||||
(($required - $passed) | length == 0) and
|
||||
((["reasonix/internal/contract/event", "reasonix/internal/runtime/agent"] - $packages) | length == 0)
|
||||
((["reasonix/internal/contract/event", "reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages) | length == 0)
|
||||
' "$RUNNER_TEMP/local-cost-quote.jsonl"
|
||||
|
||||
- name: upload Windows local cost quote inventory
|
||||
@@ -737,18 +937,24 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -count=1 -timeout=3m -json -run '^Test(AuxiliaryUsageObserver|BoundedUsageObserver|RootBounded|EffectRootBounded)' ./internal/contract/provider ./internal/model/boundedllm ./internal/runtime/agent | tee "$RUNNER_TEMP/bounded-root-budget.jsonl"
|
||||
go test -count=1 -timeout=3m -json -run '^Test(AuxiliaryUsageObserver|BoundedUsageObserver|RootBounded|EffectRootBounded)' ./internal/contract/provider ./internal/model/boundedllm ./internal/runtime/agent ./internal/assembly/boot | tee "$RUNNER_TEMP/bounded-root-budget.jsonl"
|
||||
jq -se '
|
||||
[.[] | select(.Test == null and .Action == "pass") | .Package] as $packages |
|
||||
[.[] | select(.Test != null and .Action == "pass") | .Test] as $tests |
|
||||
all(.[]; .Action != "skip" and .Action != "fail") and
|
||||
(["reasonix/internal/contract/provider", "reasonix/internal/model/boundedllm",
|
||||
"reasonix/internal/runtime/agent"] - $packages | length) == 0 and
|
||||
"reasonix/internal/runtime/agent", "reasonix/internal/assembly/boot"] - $packages | length) == 0 and
|
||||
(["TestAuxiliaryUsageObserverReplacesAndCopies",
|
||||
"TestBoundedUsageObserverOwnsOneTerminal", "TestBoundedUsageObserverOwnsOneTerminal/reported",
|
||||
"TestBoundedUsageObserverOwnsOneTerminal/billed_failure", "TestBoundedUsageObserverOwnsOneTerminal/without_sink",
|
||||
"TestBoundedUsageObserverOwnsOneTerminal/missing", "TestBoundedUsageObserverOwnsOneTerminal/input_refused"
|
||||
] - $tests | length) == 0
|
||||
"TestBoundedUsageObserverOwnsOneTerminal/missing", "TestBoundedUsageObserverOwnsOneTerminal/input_refused",
|
||||
"TestRootBoundedReviewerBudget", "TestRootBoundedReviewerBudget/reported",
|
||||
"TestRootBoundedReviewerBudget/billed_failure", "TestRootBoundedReviewerBudget/missing_usage",
|
||||
"TestRootBoundedReviewerBudget/disabled", "TestRootBoundedReviewerBudget/no_emission",
|
||||
"TestRootBoundedReviewerBudget/projected_job_context", "TestRootBoundedNestedBudgetChargesOnce",
|
||||
"TestRootBoundedLateProjectionStaysWithOldTask", "TestEffectRootBoundedAdvisorBudget",
|
||||
"TestEffectRootBoundedAdvisorBudget/reported", "TestEffectRootBoundedAdvisorBudget/disabled",
|
||||
"TestEffectRootBoundedAdvisorBudget/missing_usage"] - $tests | length) == 0
|
||||
' "$RUNNER_TEMP/bounded-root-budget.jsonl"
|
||||
|
||||
- name: upload Windows bounded root budget inventory
|
||||
@@ -777,6 +983,42 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
run: go test -count=1 -timeout=3m -run '^TestBashPowerShellExecuteDetailedContract$|^TestBashPowerShell51PreflightRejectsAndAndDetailed$|^TestBashPowerShellOutputIsUTF8$|^TestBashPowerShellSurfacesNonZeroExit$|^TestBashPowerShellRejectsChaining$' ./internal/tools/builtin
|
||||
|
||||
# Full push suite uses the same 18m package timeout as Windows smoke and a
|
||||
# 35m step budget above it. -count=1 as at the smoke step. A failure prints
|
||||
# the failed tests' output and the 15 slowest boot tests.
|
||||
- name: test (full)
|
||||
if: env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name != 'pull_request'
|
||||
timeout-minutes: 35
|
||||
env:
|
||||
# Run the prompt-cache prefix-stability guard (TestCacheHit*) in CI: a
|
||||
# regression there silently tanks the cache hit rate the project is
|
||||
# built around.
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
# Bound sandbox helper children in Windows tests so a failed OS-level
|
||||
# launch cannot pin the Actions step after Go's package timeout fires.
|
||||
WINDOWS_SANDBOX_WAIT_MS: "20000"
|
||||
shell: bash
|
||||
run: |
|
||||
# internal/assembly/boot alone takes 14-15 minutes on a Windows runner;
|
||||
# the package limit leaves headroom inside the step limit above.
|
||||
go test -count=1 -p 4 -timeout=28m -json ./... | tee "$RUNNER_TEMP/full.jsonl" |
|
||||
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
|
||||
|
||||
- name: test (full) failure detail
|
||||
if: failure() && env.RUN_STEPS == 'true' && runner.os == 'Windows' && github.event_name != 'pull_request'
|
||||
shell: bash
|
||||
run: |
|
||||
f="$RUNNER_TEMP/full.jsonl"
|
||||
[ -s "$f" ] || exit 0
|
||||
echo "::group::output of failed tests"
|
||||
jq -rn '[inputs | select(.Test != null)] | group_by(.Package + "/" + .Test)
|
||||
| map(select(any(.[]; .Action == "fail")))[]
|
||||
| map(select(.Action == "output") | .Output) | join("")' "$f" || true
|
||||
echo "::endgroup::"
|
||||
echo "slowest internal/assembly/boot tests (seconds, name):"
|
||||
jq -rn '[inputs | select((.Package | endswith("/internal/assembly/boot")) and .Test != null and .Action == "pass")]
|
||||
| sort_by(-.Elapsed)[:15][] | "\(.Elapsed)\t\(.Test)"' "$f" || true
|
||||
|
||||
race:
|
||||
needs: changes
|
||||
if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.code != 'false')
|
||||
@@ -811,130 +1053,15 @@ jobs:
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
run: go test -race ./internal/runtime/agent/... ./internal/runtime/delegation/... ./internal/platform/worktree/... ./internal/state/mutation/... ./internal/ext/plugin/... ./internal/tools/jobs/... ./internal/base/proc/... ./internal/safety/sandbox/... ./internal/base/filelock/... ./internal/contract/eventwire/... ./internal/platform/remote/... ./internal/ext/extension/... ./internal/assembly/boot/... ./internal/session/control/... ./internal/contract/tool/...
|
||||
run: go test -race ./internal/runtime/agent/... ./internal/runtime/delegation/... ./internal/runtime/isolation/... ./internal/platform/worktree/... ./internal/state/mutation/... ./internal/ext/plugin/... ./internal/tools/jobs/... ./internal/base/proc/... ./internal/safety/sandbox/... ./internal/base/filelock/... ./internal/contract/eventwire/... ./internal/platform/remote/... ./internal/ext/extension/... ./internal/assembly/boot/... ./internal/session/control/... ./internal/contract/tool/...
|
||||
|
||||
# Push-only full suites. internal/frontend/serve and internal/assembly/boot run
|
||||
# hundreds of serial tests and set the wall time of one whole-tree `go test`,
|
||||
# so each is cut by test name into shards that run on their own runners. The
|
||||
# -run / -skip patterns partition the names exactly: every test lands in one
|
||||
# shard. `rest` is every other package. Pull requests keep the focused steps
|
||||
# in the `test` and `race` jobs above.
|
||||
test-windows-shards:
|
||||
if: github.event_name != 'pull_request'
|
||||
name: test (windows-latest, ${{ matrix.shard }})
|
||||
runs-on: windows-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { shard: rest }
|
||||
- { shard: serve-1, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[A-E]' }
|
||||
- { shard: serve-2, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[F-R]' }
|
||||
- { shard: serve-3, pkg: ./internal/frontend/serve, flag: -skip, filter: '^Test[A-R]' }
|
||||
- { shard: boot-1, pkg: ./internal/assembly/boot, flag: -skip, filter: '^TestEffect([^0-9A-C]|$)' }
|
||||
- { shard: boot-2, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect[D-O]' }
|
||||
- { shard: boot-3, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect([^0-9A-O]|$)' }
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: test (shard)
|
||||
timeout-minutes: 40
|
||||
env:
|
||||
# Run the prompt-cache prefix-stability guard (TestCacheHit*) in CI: a
|
||||
# regression there silently tanks the cache hit rate the project is
|
||||
# built around.
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
# Bound sandbox helper children in Windows tests so a failed OS-level
|
||||
# launch cannot pin the Actions step after Go's package timeout fires.
|
||||
WINDOWS_SANDBOX_WAIT_MS: "20000"
|
||||
SHARD: ${{ matrix.shard }}
|
||||
PKG: ${{ matrix.pkg }}
|
||||
FLAG: ${{ matrix.flag }}
|
||||
FILTER: ${{ matrix.filter }}
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "$SHARD" = rest ]; then
|
||||
mapfile -t pkgs < <(go list ./... | tr -d '\r' | grep -v -E '/internal/(frontend/serve|assembly/boot)$')
|
||||
else
|
||||
pkgs=("$PKG")
|
||||
fi
|
||||
filter=()
|
||||
if [ -n "$FILTER" ]; then filter=("$FLAG" "$FILTER"); fi
|
||||
go test -count=1 -p 4 -timeout=36m -json "${filter[@]}" "${pkgs[@]}" | tee "$RUNNER_TEMP/shard.jsonl" |
|
||||
jq -rj --unbuffered 'select(.Action == "output" and .Test == null) | .Output'
|
||||
|
||||
- name: test (shard) failure detail
|
||||
if: failure()
|
||||
shell: bash
|
||||
run: |
|
||||
f="$RUNNER_TEMP/shard.jsonl"
|
||||
[ -s "$f" ] || exit 0
|
||||
echo "::group::output of failed tests"
|
||||
jq -rn '[inputs | select(.Test != null)] | group_by(.Package + "/" + .Test)
|
||||
| map(select(any(.[]; .Action == "fail")))[]
|
||||
| map(select(.Action == "output") | .Output) | join("")' "$f" || true
|
||||
echo "::endgroup::"
|
||||
echo "slowest tests (seconds, package, name):"
|
||||
jq -rn '[inputs | select(.Test != null and .Action == "pass")]
|
||||
| sort_by(-.Elapsed)[:15][] | "\(.Elapsed) \(.Package) \(.Test)"' "$f" || true
|
||||
|
||||
race-shards:
|
||||
if: github.event_name != 'pull_request'
|
||||
name: race (${{ matrix.shard }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { shard: rest }
|
||||
- { shard: serve-1, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[A-E]' }
|
||||
- { shard: serve-2, pkg: ./internal/frontend/serve, flag: -run, filter: '^Test[F-R]' }
|
||||
- { shard: serve-3, pkg: ./internal/frontend/serve, flag: -skip, filter: '^Test[A-R]' }
|
||||
- { shard: boot-1, pkg: ./internal/assembly/boot, flag: -skip, filter: '^TestEffect([^0-9A-C]|$)' }
|
||||
- { shard: boot-2, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect[D-O]' }
|
||||
- { shard: boot-3, pkg: ./internal/assembly/boot, flag: -run, filter: '^TestEffect([^0-9A-O]|$)' }
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Install and verify Linux sandbox backend
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y bubblewrap
|
||||
if sysctl -n kernel.unprivileged_userns_clone >/dev/null 2>&1; then
|
||||
sudo sysctl -w kernel.unprivileged_userns_clone=1
|
||||
fi
|
||||
if sysctl -n kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then
|
||||
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
||||
fi
|
||||
bwrap --ro-bind / / --dev /dev --proc /proc -- true || echo "::warning::bubblewrap is installed but this runner denies user namespaces; unavailable-backend tests will run fail-closed"
|
||||
|
||||
- name: test -race (shard)
|
||||
timeout-minutes: 30
|
||||
- name: test -race (full)
|
||||
if: github.event_name != 'pull_request'
|
||||
env:
|
||||
REASONIX_RELEASE_CACHE_GUARD: "1"
|
||||
SHARD: ${{ matrix.shard }}
|
||||
PKG: ${{ matrix.pkg }}
|
||||
FLAG: ${{ matrix.flag }}
|
||||
FILTER: ${{ matrix.filter }}
|
||||
run: |
|
||||
if [ "$SHARD" = rest ]; then
|
||||
mapfile -t pkgs < <(go list ./... | grep -v -E '/internal/(frontend/serve|assembly/boot)$')
|
||||
else
|
||||
pkgs=("$PKG")
|
||||
fi
|
||||
filter=()
|
||||
if [ -n "$FILTER" ]; then filter=("$FLAG" "$FILTER"); fi
|
||||
go test -race -timeout=25m "${filter[@]}" "${pkgs[@]}"
|
||||
# internal/assembly/boot under -race runs past ten minutes on a shared
|
||||
# runner; the package limit leaves headroom over the default.
|
||||
run: go test -race -timeout=25m ./...
|
||||
|
||||
# sdk/go is a nested stdlib-only module invisible to root `go test ./...`.
|
||||
# Its DTOs are generated from internal/ext/extension/protocol, and the
|
||||
|
||||
@@ -43,20 +43,20 @@ jobs:
|
||||
if: matrix.branch != ''
|
||||
run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
|
||||
uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.language == 'go' && 'autobuild' || 'none' }}
|
||||
- name: Perform CodeQL Analysis
|
||||
if: matrix.branch == ''
|
||||
uses: github/codeql-action/analyze@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
|
||||
uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
# Results from a checked-out line are filed under that line's ref, not the
|
||||
# default branch the schedule happened to run from.
|
||||
- name: Perform CodeQL Analysis (${{ matrix.branch }})
|
||||
if: matrix.branch != ''
|
||||
uses: github/codeql-action/analyze@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
|
||||
uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
ref: refs/heads/${{ matrix.branch }}
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
name: PR author gate
|
||||
|
||||
# Pull requests are limited to maintainers and the logins in .github/pr-authors.txt.
|
||||
# This runs in the base repository's context, checks out the base branch only, and
|
||||
# never executes pull-request code; the allowlist it reads is the base branch's.
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: pr-author-gate-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
gate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- run: node scripts/pr-author-gate.mjs
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
@@ -200,7 +200,7 @@ jobs:
|
||||
cache: true
|
||||
cache-dependency-path: go.sum
|
||||
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
@@ -608,7 +608,7 @@ jobs:
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
@@ -911,7 +911,7 @@ jobs:
|
||||
# The page every archive carries beside the executable, built once here
|
||||
# rather than once per target: serve.FindPage looks for
|
||||
# frontend-next/dist next to the binary.
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
cache: true
|
||||
cache-dependency-path: go.sum
|
||||
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
@@ -16,7 +16,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: studio-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.ref == 'refs/heads/studio' }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
@@ -108,7 +108,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- if: steps.spa.outputs.run == 'true'
|
||||
uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
@@ -143,7 +143,6 @@ jobs:
|
||||
node perf/hoverradius.mjs
|
||||
node perf/jump-ends.mjs
|
||||
node perf/modelrows.mjs
|
||||
node perf/model-menu-badge.mjs
|
||||
node perf/explorersize.mjs
|
||||
node perf/git-panel.mjs
|
||||
node perf/session-move-to.mjs
|
||||
@@ -158,17 +157,9 @@ jobs:
|
||||
node perf/codeeditor-size.mjs
|
||||
node perf/composer-reflow.mjs
|
||||
node perf/composer-toolbar-row.mjs
|
||||
node perf/control-icons.mjs
|
||||
node perf/nav-selection.mjs
|
||||
node perf/action-icons.mjs
|
||||
node perf/rowhit.mjs
|
||||
node perf/prefs-wide-centered.mjs
|
||||
node perf/settings-fullscreen.mjs
|
||||
node perf/settings-page-column.mjs
|
||||
node perf/sidebar-tools.mjs
|
||||
node perf/select-alignment.mjs
|
||||
node perf/provider-fields.mjs
|
||||
node perf/thought-clock.mjs
|
||||
node perf/tab-pip.mjs
|
||||
node perf/syntax-tokens.mjs
|
||||
node perf/tab-close-inside.mjs
|
||||
@@ -197,7 +188,6 @@ jobs:
|
||||
node perf/web-access.mjs
|
||||
node perf/peers-settings.mjs
|
||||
node perf/output-cap.mjs
|
||||
node perf/tool-output-clip.mjs
|
||||
node perf/agent-run-tab.mjs
|
||||
node perf/changes-cap.mjs
|
||||
node perf/session-menu-fit.mjs
|
||||
@@ -226,21 +216,6 @@ jobs:
|
||||
node perf/narrow-rail.mjs
|
||||
node perf/selfcheck.mjs branch-tooltip
|
||||
|
||||
- name: Keep UI control previews
|
||||
if: always() && steps.spa.outputs.run == 'true'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: studio-ui-control-previews
|
||||
path: |
|
||||
desktop/frontend-next/perf/shots/control-icons-*.png
|
||||
desktop/frontend-next/perf/shots/nav-selection-*.png
|
||||
desktop/frontend-next/perf/shots/settings-full-*.png
|
||||
desktop/frontend-next/perf/shots/sidebar-tools-*.png
|
||||
desktop/frontend-next/perf/shots/select-alignment-*.png
|
||||
desktop/frontend-next/perf/shots/provider-fields-*.png
|
||||
desktop/frontend-next/perf/shots/thought-clock-*.png
|
||||
retention-days: 7
|
||||
|
||||
# What a hosted page does over the relay, end to end: the production bundle,
|
||||
# the real Studio host and kernel, a real Chromium; only the accounts API and
|
||||
# the relay are stand-ins (tools/relayfake). Event-transport changes are
|
||||
@@ -274,7 +249,7 @@ jobs:
|
||||
cache: true
|
||||
|
||||
- if: steps.scope.outputs.run == 'true'
|
||||
uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
@@ -459,7 +434,7 @@ jobs:
|
||||
cache: true
|
||||
cache-dependency-path: desktop/go.sum
|
||||
|
||||
- uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# Build output and local tooling state.
|
||||
build/
|
||||
.gradle/
|
||||
.kotlin/
|
||||
local.properties
|
||||
*.iml
|
||||
.idea/
|
||||
|
||||
# The signing key and the file that names it.
|
||||
#
|
||||
# A key is not part of the project: it is the identity one person ships under,
|
||||
# and publishing it would let anyone else sign an APK that installs over
|
||||
# everyone's. Neither the keystore nor its password belongs in version control.
|
||||
# `app/build.gradle` reads both from here when present and falls back to the
|
||||
# debug key when they are not, so a fresh clone builds without them.
|
||||
keystore/
|
||||
|
||||
# Build inputs, produced by scripts/ before Gradle runs.
|
||||
# jniLibs/ holds the cross-compiled kernel, assets/page/ the built
|
||||
# frontend. assets/LICENSE is hand-placed and stays tracked.
|
||||
app/src/main/jniLibs/
|
||||
app/src/main/assets/page/
|
||||
@@ -0,0 +1,216 @@
|
||||
---
|
||||
owner: @esengine
|
||||
backup: @SivanCola
|
||||
status: active
|
||||
reviewed: 2026-10-20
|
||||
---
|
||||
|
||||
# Reasonix Mobile — the Android shell
|
||||
|
||||
Runs Reasonix as an Android app: the kernel ships inside the APK and starts on the
|
||||
device, the interface is the same `desktop/frontend-next` build the desktop Studio
|
||||
serves, and the agent reaches servers over SSH.
|
||||
|
||||
This file is for people building it. What a person installing it needs — what it
|
||||
does, how to pair with a computer, what it cannot do — belongs in the release
|
||||
notes.
|
||||
|
||||
## One command
|
||||
|
||||
```bash
|
||||
./scripts/build.sh release # or debug
|
||||
```
|
||||
|
||||
Three steps, in this order:
|
||||
|
||||
1. `build-kernel.sh` → `app/src/main/jniLibs/arm64-v8a/libreasonix.so`
|
||||
2. `sync-page.sh` → `app/src/main/assets/page/`
|
||||
3. `./gradlew assembleRelease`
|
||||
|
||||
Output: `app/build/outputs/apk/release/app-release.apk`
|
||||
|
||||
The kernel and the page are **inputs to packaging, not Gradle build steps**.
|
||||
Running `gradle assembleRelease` on its own fails on purpose and names the script
|
||||
that produces whichever input is missing (`requireBuildInputs` in
|
||||
`app/build.gradle`): an APK with no kernel in it should not be buildable by
|
||||
accident.
|
||||
|
||||
## Requirements
|
||||
|
||||
| Component | Version |
|
||||
| --- | --- |
|
||||
| JDK | 21 (17 also works) |
|
||||
| Android SDK | `platforms;android-35`, `build-tools;35.0.0`, `platform-tools` |
|
||||
| Gradle | the committed wrapper, 8.14.3 (for AGP 8.11.1) |
|
||||
| Go | 1.26+ |
|
||||
| Node / pnpm | Node 24+ / pnpm 10 |
|
||||
|
||||
```bash
|
||||
export ANDROID_HOME="$HOME/Android/Sdk"
|
||||
export ANDROID_SDK_ROOT="$ANDROID_HOME"
|
||||
export JAVA_HOME="/path/to/jdk-21"
|
||||
export PATH="$PATH:$(go env GOPATH)/bin"
|
||||
```
|
||||
|
||||
`ANDROID_HOME` is the one thing nothing sets for you.
|
||||
|
||||
## Scripts
|
||||
|
||||
| Script | What it does |
|
||||
| --- | --- |
|
||||
| `build.sh [debug\|release]` | The whole chain |
|
||||
| `build-kernel.sh` | Cross-compiles the kernel only (`GOOS=android GOARCH=arm64 CGO_ENABLED=0`) |
|
||||
| `sync-page.sh` | Builds the frontend, copies `dist/` into assets, copies `LICENSE` |
|
||||
| `verify-page.sh` | **No phone needed**: starts the real kernel against the packaged page |
|
||||
| `run.sh [debug\|release]` | `adb install`, launch, follow the log |
|
||||
|
||||
`verify-page.sh` is the one to run after a change: it starts the kernel with the
|
||||
same arguments `Kernel.kt` uses (`--auth token`, `--token-file`, `--port-file`,
|
||||
`-page`) and checks that it binds a port and answers 200. Without a device, that
|
||||
is as deep as the chain can be verified.
|
||||
|
||||
## Where the code is
|
||||
|
||||
| File | Owns |
|
||||
| --- | --- |
|
||||
| `MainActivity.kt` | The WebView, the back gesture, downloads, external links, renderer restart |
|
||||
| `Kernel.kt` | Unpacking the page, starting and stopping the kernel, waiting for the port, health probes, logs, the workspace directory |
|
||||
| `KernelService.kt` | The foreground service that keeps a long turn alive |
|
||||
| `NativeBridge.kt` | `@JavascriptInterface`: what the page may ask the phone for |
|
||||
| `BrowserShellServer.kt` | Loopback HTTP server the kernel's browser tools drive the overlay WebView through |
|
||||
| `Downloader.kt` | Writing a kernel-produced file through the system document picker |
|
||||
|
||||
The page reaches the shell through `window.reasonixHost` and `ReasonixNative`,
|
||||
defined in `desktop/frontend-next/src/port/host.ts` (`AndroidHost`).
|
||||
|
||||
**Phone and desktop share one page build.** What differs only on a phone is
|
||||
decided at runtime, not by a second set of components. `phoneShell()` in
|
||||
`src/port/host.ts` and the `[data-shell="android"]` selector in `mobile.css` both
|
||||
read `<html data-shell="android">`, which `index.html` sets before the bundle
|
||||
loads.
|
||||
|
||||
One attribute feeding both is what keeps style and components from disagreeing
|
||||
about what a phone is.
|
||||
|
||||
## What the page may ask the phone for
|
||||
|
||||
`NativeBridge` is reachable from whatever document the WebView holds, so every
|
||||
method that touches the device asks `trusted()` first.
|
||||
|
||||
The activity answers by comparing the **main frame's** origin against the
|
||||
kernel's and against the one paired computer the person scanned a code for. A
|
||||
page that navigated elsewhere is not trusted, even though it may still hold the
|
||||
object.
|
||||
|
||||
Five methods answer without the check — `version`, `systemTheme`, `safeArea`,
|
||||
`setLayerOpen`, `setRunning` — because they report back what the page already
|
||||
knows about its own window and grant nothing.
|
||||
|
||||
`openInApp(url)` is the pairing decision, not a navigation one: it loads a URL in
|
||||
this window only for the origin the scanner just returned, and only once.
|
||||
Without that, any page holding the kernel origin could hand itself the bridge.
|
||||
|
||||
External links leave through the browser. A non-web scheme is routed to the
|
||||
system only for `mailto:` and `tel:`; everything else is dropped, because an
|
||||
`intent://` URL can start an arbitrary exported activity with caller-supplied
|
||||
extras.
|
||||
|
||||
## Permissions, and what they cost
|
||||
|
||||
`MANAGE_EXTERNAL_STORAGE` — opening a folder on the phone as a workspace. The
|
||||
kernel is a Go binary: it opens real paths with `open(2)`, and SAF hands back a
|
||||
`content://` URI it cannot open.
|
||||
|
||||
Requested from the folder-picker action, never at launch. Android shows no dialog
|
||||
for it and opens system settings instead.
|
||||
|
||||
**There is no OS-level sandbox around this kernel.** The PR sets the sandbox to
|
||||
off on Android, so confining the agent to the chosen folder is Reasonix's own
|
||||
write-root check and nothing else. A shell tool with all-files access can reach
|
||||
all shared storage.
|
||||
|
||||
`REQUEST_IGNORE_BATTERY_OPTIMIZATIONS` — Doze freezes the app group, which stops
|
||||
a turn that is still running. Asked once when a turn actually runs, never at
|
||||
launch, and the person can refuse. Refusing costs battery life during long turns
|
||||
and nothing else.
|
||||
|
||||
**Google Play restricts this permission**: fine for a sideloaded build, a
|
||||
store-policy risk if this is ever listed.
|
||||
|
||||
`FOREGROUND_SERVICE` / `FOREGROUND_SERVICE_DATA_SYNC` — keeping the kernel alive
|
||||
while the screen is off. `dataSync` is capped per 24 hours from Android 15. It is
|
||||
the closest honest description of "an agent is working".
|
||||
|
||||
`CAMERA` — reading the pairing code a computer shows. Requested when the person
|
||||
asks to scan, never at launch. `ScanActivity` is not exported.
|
||||
|
||||
`READ_EXTERNAL_STORAGE` / `WRITE_EXTERNAL_STORAGE` — the same folder walk below
|
||||
API 33. `maxSdkVersion`-gated; above 33 they are superseded by the permission
|
||||
above.
|
||||
|
||||
`allowBackup=false` with explicit extraction rules, so a `keystore` or a token in
|
||||
app storage never leaves the device in a cloud backup.
|
||||
|
||||
## Cleartext HTTP
|
||||
|
||||
The kernel serves its page on `127.0.0.1:<port>`, and a paired computer serves
|
||||
the same page over its LAN address. Both are plain HTTP.
|
||||
|
||||
Neither address can be written into a `network-security-config`: the port is
|
||||
chosen at start, the LAN address at scan time, and the format matches literals
|
||||
and domains, not subnets.
|
||||
|
||||
So the config permits cleartext and the boundary is in the code instead.
|
||||
`shouldOverrideUrlLoading` sends every other http(s) origin to the browser, and
|
||||
the bridge answers only the kernel and the just-paired computer. A plaintext page
|
||||
on the LAN cannot become the document holding the bridge.
|
||||
|
||||
## What goes into the APK
|
||||
|
||||
| Content | Size | Why it is shaped this way |
|
||||
| --- | --- | --- |
|
||||
| `lib/arm64-v8a/libreasonix.so` | 45 MB | Go static binary, stripped. It is a `.so` rather than a file in `assets/` because only `nativeLibraryDir` is guaranteed executable — a binary copied to app storage fails on a device whose data partition is mounted `noexec`. |
|
||||
| `assets/page/` | 14 MB | The frontend build, including the Inter and Noto Sans SC fonts, `noCompress` |
|
||||
| `assets/LICENSE` | 1 KB | The repository's MIT licence, verbatim. MIT requires the notice and the licence text to accompany a binary distribution. |
|
||||
|
||||
`assets/LICENSE` is generated by `sync-page.sh` from the repository's root
|
||||
`LICENSE`, not committed a second time: a checked-in copy drifts when the root
|
||||
file is updated.
|
||||
|
||||
It sits in `assets/` rather than `assets/page/` because that directory is removed
|
||||
and rebuilt on every build.
|
||||
|
||||
A release APK is about 31 MB.
|
||||
|
||||
## Building
|
||||
|
||||
**The Gradle wrapper is committed**, jar and all. `build.sh` refuses to run
|
||||
without `gradlew` rather than generating one from whatever `gradle` is on `PATH`:
|
||||
the dependency versions are part of the build.
|
||||
|
||||
`gradle-wrapper.properties` pins `distributionSha256Sum`, so a mirror that serves
|
||||
a different distribution zip fails verification instead of being unpacked and
|
||||
run. Dependency repositories are the official ones only — a mirror belongs in a
|
||||
user-level init script.
|
||||
|
||||
**Signing.** A release with no key configured comes out **unsigned**
|
||||
(`app-release-unsigned.apk`) with a warning. It used to fall back to the debug
|
||||
key, which produced an artifact that looked shippable and was signed by a
|
||||
per-machine key nobody chose: it could not install over the previous build.
|
||||
|
||||
Point the build at a real key by writing `keystore/keystore.properties`
|
||||
(`storeFile`, `storePassword`, `keyAlias`, and optionally `keyPassword`) or by
|
||||
setting the matching `REASONIX_KEYSTORE*` environment variables. Key material
|
||||
stays out of the repository and out of build logs.
|
||||
|
||||
Debug builds always use the debug key. Signing a debug build with the release key
|
||||
would let anyone who can read the keystore produce a build that installs over the
|
||||
shipped one under the same signature.
|
||||
|
||||
**Non-ASCII checkout paths.** AGP refuses to build under a path with non-ASCII
|
||||
characters on Windows unless `android.overridePathCheck=true` is set. That is a
|
||||
property of one machine's disk layout, not of this project, so it is not in the
|
||||
repository's `gradle.properties`.
|
||||
|
||||
Set it in a user-level `~/.gradle/gradle.properties` or pass
|
||||
`-Pandroid.overridePathCheck=true`.
|
||||
@@ -0,0 +1,242 @@
|
||||
// The app module.
|
||||
//
|
||||
// Two things ship inside the APK and neither is fetched at runtime: the kernel
|
||||
// binary, and the built page it serves. Bundling the page is what makes the app
|
||||
// work offline and start instantly; bundling the kernel is what lets it run
|
||||
// `reasonix serve` on the phone itself, which is the whole point — the agent
|
||||
// starts locally and reaches the server over SSH, so the interface, the model
|
||||
// catalogue and the credentials all live on the phone.
|
||||
//
|
||||
// The kernel is placed as `jniLibs/arm64-v8a/libreasonix.so`, not under
|
||||
// `assets/`. That is not a trick for its own sake: Android permits executing a
|
||||
// file under `nativeLibraryDir` and nothing under `filesDir` has that
|
||||
// guarantee, so a binary shipped the ordinary way would have to be copied out
|
||||
// and made executable at runtime — which fails outright on a device mounted
|
||||
// noexec, and fails confusingly on the ones that are merely unusual. `scripts/
|
||||
// build-kernel.sh` writes it there; the name is fixed by the packaging rules.
|
||||
plugins {
|
||||
id "com.android.application"
|
||||
id "org.jetbrains.kotlin.android"
|
||||
}
|
||||
|
||||
// One version, read by the About screen and the update check.
|
||||
def versionMajor = 2
|
||||
def versionMinor = 0
|
||||
def versionPatch = 0
|
||||
def versionBuild = 1
|
||||
|
||||
// Where the two build inputs are expected. Both are produced from the checkout
|
||||
// beside this module, by the scripts named in each message.
|
||||
def upstream = rootProject.file("..")
|
||||
def pageDir = new File(projectDir, "src/main/assets/page")
|
||||
def kernelSo = new File(projectDir, "src/main/jniLibs/arm64-v8a/libreasonix.so")
|
||||
|
||||
android {
|
||||
namespace "io.reasonix.mobile.app"
|
||||
compileSdk 35
|
||||
|
||||
defaultConfig {
|
||||
applicationId "io.reasonix.mobile.app"
|
||||
minSdk 26
|
||||
// 35: the API level this app is actually built and tested against.
|
||||
//
|
||||
// It was briefly lowered to 26 while chasing an install failure that
|
||||
// turned out to be a package-name collision with an unrelated app on the
|
||||
// user's phone — the signature error named the wrong cause. Nothing about
|
||||
// the target level ever caused it, so it goes back to the level the code
|
||||
// is really written for.
|
||||
//
|
||||
// One consequence to keep in mind: from 35, Android 15 forces the window
|
||||
// edge-to-edge and ignores a request to lay out inside the safe area. The
|
||||
// window therefore *is* drawn under the status bar and the home indicator,
|
||||
// and the only correct answer is for the page to pad itself — which is
|
||||
// what `MainActivity` feeds real inset values for, since a WebView does
|
||||
// not implement `env(safe-area-inset-*)`.
|
||||
targetSdk 35
|
||||
versionCode versionMajor * 10000 + versionMinor * 100 + versionPatch * 10 + versionBuild
|
||||
versionName "${versionMajor}.${versionMinor}.${versionPatch}"
|
||||
|
||||
// The kernel is built for arm64 only, so every other ABI would ship an
|
||||
// APK that installs and then cannot start. Naming the one real ABI keeps
|
||||
// the store listing and the installer honest.
|
||||
ndk {
|
||||
abiFilters "arm64-v8a"
|
||||
}
|
||||
|
||||
resourceConfigurations += ["zh", "en"]
|
||||
}
|
||||
|
||||
// Android identifies an app by package name *and* signing key: an APK signed
|
||||
// with a different key is not an update, it is an impostor, and the
|
||||
// installer refuses it with "the signature does not match the installed
|
||||
// app". Gradle's default release key is the debug key in the user's home
|
||||
// directory, which is per-machine, so a build made on a second computer
|
||||
// could never be installed over one made on the first.
|
||||
//
|
||||
// Whoever ships a build therefore pins a key — and that key is theirs, not
|
||||
// the project's. It lives beside the checkout, in `keystore/`, which is
|
||||
// ignored; so does the file naming it. A checkout without either signs with
|
||||
// the debug key instead, which is what a contributor building locally wants
|
||||
// and is what the README tells them to expect. Point the build at a real key
|
||||
// by writing `keystore/keystore.properties` (storeFile, storePassword,
|
||||
// keyAlias) or by setting REASONIX_KEYSTORE / REASONIX_KEYSTORE_PASSWORD /
|
||||
// REASONIX_KEY_ALIAS.
|
||||
def keyProperties = new Properties()
|
||||
def keyPropertiesFile = rootProject.file("keystore/keystore.properties")
|
||||
if (keyPropertiesFile.exists()) {
|
||||
keyPropertiesFile.withInputStream { keyProperties.load(it) }
|
||||
}
|
||||
def keyStorePath = keyProperties.getProperty("storeFile") ?: System.getenv("REASONIX_KEYSTORE")
|
||||
def keyStorePassword = keyProperties.getProperty("storePassword") ?: System.getenv("REASONIX_KEYSTORE_PASSWORD")
|
||||
// The key's own password is a separate fact from the store's. PKCS#12 keystores
|
||||
// almost always use one password for both, so falling back to the store's
|
||||
// password keeps those working — but assuming they are equal is how a keystore
|
||||
// that separates them gets a build failure at signing time, after the work is
|
||||
// done, with an error that does not say which of the two was wrong.
|
||||
def keyPasswordValue = keyProperties.getProperty("keyPassword") ?: System.getenv("REASONIX_KEY_PASSWORD") ?: keyStorePassword
|
||||
def keyAliasName = keyProperties.getProperty("keyAlias") ?: System.getenv("REASONIX_KEY_ALIAS")
|
||||
def haveReleaseKey = keyStorePath && keyStorePassword && keyAliasName &&
|
||||
rootProject.file(keyStorePath).exists()
|
||||
|
||||
signingConfigs {
|
||||
if (haveReleaseKey) {
|
||||
releaseKey {
|
||||
storeFile rootProject.file(keyStorePath)
|
||||
storePassword keyStorePassword
|
||||
keyAlias keyAliasName
|
||||
keyPassword keyPasswordValue
|
||||
// v1 is not optional here. Android 7+ reads v2 and ignores v1, so
|
||||
// a v2-only APK installs on a modern phone — and then fails on the
|
||||
// compatibility layers older devices and HarmonyOS' 卓易通 run,
|
||||
// which know only the JAR signature and report its absence as a
|
||||
// signature *mismatch*. Signing both is what makes one APK install
|
||||
// everywhere.
|
||||
enableV1Signing = true
|
||||
enableV2Signing = true
|
||||
enableV3Signing = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing native is compiled here — the one .so is a prebuilt the kernel
|
||||
// script writes — so the external native build is left alone entirely.
|
||||
buildTypes {
|
||||
debug {
|
||||
applicationIdSuffix ".debug"
|
||||
versionNameSuffix "-debug"
|
||||
// The debug key, always. Signing a debug build with the release key
|
||||
// would mean anyone who can read the release keystore can produce a
|
||||
// build that installs over the shipped one under the same signature,
|
||||
// and it buys nothing: the debug package is suffixed `.debug`, so the
|
||||
// two already install side by side.
|
||||
}
|
||||
release {
|
||||
// The kernel is a stripped Go binary and the page is minified, but
|
||||
// the Kotlin shell still benefits and shipping the shrunk build is
|
||||
// how a size regression gets noticed.
|
||||
minifyEnabled true
|
||||
shrinkResources true
|
||||
proguardFiles getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro"
|
||||
// A release with no key of its own comes out *unsigned*
|
||||
// (`app-release-unsigned.apk`). It used to fall back to the debug key
|
||||
// so the artifact would install, which produced a release that looked
|
||||
// shippable and was signed by a per-machine key nobody chose: it could
|
||||
// not be installed over the previous build, and every machine made a
|
||||
// different one. An unsigned artifact says what it is.
|
||||
if (haveReleaseKey) {
|
||||
signingConfig signingConfigs.releaseKey
|
||||
} else {
|
||||
logger.warn(
|
||||
"release build has no signing key: producing an unsigned APK. " +
|
||||
"Write keystore/keystore.properties (storeFile, storePassword, keyAlias) " +
|
||||
"or set REASONIX_KEYSTORE / REASONIX_KEYSTORE_PASSWORD / REASONIX_KEY_ALIAS."
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
compileOptions {
|
||||
sourceCompatibility JavaVersion.VERSION_17
|
||||
targetCompatibility JavaVersion.VERSION_17
|
||||
}
|
||||
kotlinOptions {
|
||||
jvmTarget = "17"
|
||||
}
|
||||
|
||||
buildFeatures {
|
||||
// AGP 8 stops generating BuildConfig unless asked. `Kernel` reads the
|
||||
// version name and code out of it to decide whether the bundled page
|
||||
// needs unpacking, so it is asked for.
|
||||
buildConfig = true
|
||||
}
|
||||
|
||||
packaging {
|
||||
jniLibs {
|
||||
// `extractNativeLibs=true`: the kernel must exist as a real file on
|
||||
// disk to be executed, and an uncompressed-and-mmap'd .so is not
|
||||
// one. This is the only reason the flag is on.
|
||||
useLegacyPackaging true
|
||||
}
|
||||
resources {
|
||||
excludes += ["META-INF/*.version", "META-INF/*.kotlin_module", "kotlin/**", "DebugProbesKt.bin"]
|
||||
}
|
||||
}
|
||||
|
||||
// The page is read out of assets by `Kernel.publish`, not by the resource
|
||||
// system, so it is not compressed — a page is already-compressed text and
|
||||
// shrinking it again only costs startup time.
|
||||
androidResources {
|
||||
noCompress += ["json", "woff2", "ttf", "woff", "txt", "svg"]
|
||||
}
|
||||
|
||||
lint {
|
||||
abortOnError true
|
||||
warningsAsErrors false
|
||||
checkReleaseBuilds true
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation "androidx.core:core-ktx:1.15.0"
|
||||
implementation "androidx.appcompat:appcompat:1.7.0"
|
||||
implementation "androidx.activity:activity-ktx:1.9.3"
|
||||
implementation "androidx.webkit:webkit:1.12.1"
|
||||
implementation "androidx.lifecycle:lifecycle-runtime-ktx:2.8.7"
|
||||
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.9.0"
|
||||
|
||||
// Scanning the pairing code a computer shows.
|
||||
//
|
||||
// CameraX for the preview and the frame stream, ZXing for the decode, and
|
||||
// deliberately not ML Kit: this app installs on phones whose HarmonyOS
|
||||
// compatibility layer has no Google Play Services, so a decoder that needs
|
||||
// GMS would work on the developer's phone and fail on the user's. ZXing
|
||||
// core is pure Java with no runtime dependency at all, which is what makes
|
||||
// the scanner work on both.
|
||||
implementation "androidx.camera:camera-core:1.4.1"
|
||||
implementation "androidx.camera:camera-camera2:1.4.1"
|
||||
implementation "androidx.camera:camera-lifecycle:1.4.1"
|
||||
implementation "androidx.camera:camera-view:1.4.1"
|
||||
implementation "com.google.zxing:core:3.5.3"
|
||||
}
|
||||
|
||||
// Fail before the packaging step with something a person can act on, rather
|
||||
// than shipping an APK whose kernel is missing or whose page is a directory the
|
||||
// build never filled. The two messages name the script that produces each.
|
||||
tasks.register("requireBuildInputs") {
|
||||
doFirst {
|
||||
if (!kernelSo.exists()) {
|
||||
throw new GradleException(
|
||||
"内核二进制缺失：${kernelSo}\n先运行 ./scripts/build-kernel.sh（在 android/ 目录下）交叉编译内核。")
|
||||
}
|
||||
if (!pageDir.exists() || !new File(pageDir, "index.html").exists()) {
|
||||
throw new GradleException(
|
||||
"前端页面缺失：${pageDir}/index.html\n先运行 ./scripts/sync-page.sh（在 android/ 目录下）构建并同步页面。")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks.configureEach { task ->
|
||||
if (task.name.startsWith("merge") && (task.name.contains("Assets") || task.name.contains("JniLibFolders"))) {
|
||||
task.dependsOn("requireBuildInputs")
|
||||
}
|
||||
}
|
||||
Vendored
+32
@@ -0,0 +1,32 @@
|
||||
# R8 rules for the shell.
|
||||
#
|
||||
# There is very little to keep here, and that is the point: the app's own code
|
||||
# is reached from the manifest and from Android's own callbacks, both of which
|
||||
# R8 already traces, and nothing in it is loaded reflectively. What follows is
|
||||
# the two places where a name is looked up rather than called.
|
||||
|
||||
# The WebView's JavaScript bridge, if one is ever added, is resolved by name on
|
||||
# the Java side and by the string the page uses. Keeping annotated members makes
|
||||
# that survive shrinking rather than failing at runtime with a method-not-found
|
||||
# that only shows up on a release build.
|
||||
-keepclassmembers class * {
|
||||
@android.webkit.JavascriptInterface <methods>;
|
||||
}
|
||||
|
||||
# The Kernel's child process is started by absolute path, and its own log is
|
||||
# read as text; neither is reflective, so nothing else needs keeping.
|
||||
|
||||
# Line numbers make a stack trace from a release build readable, which is the
|
||||
# only way a crash report from a phone can be acted on.
|
||||
-keepattributes SourceFile,LineNumberTable
|
||||
-renamesourcefileattribute SourceFile
|
||||
|
||||
# Kotlin's `@Metadata` annotation is present on every class the compiler touched
|
||||
# and nothing in this app reflects over it. It is tempting to `-assumenosideeffects`
|
||||
# it away, and that was tried: any pattern broad enough to name its members also
|
||||
# matches the methods every class inherits from `Object` — `equals`, `hashCode`,
|
||||
# `toString` — and R8 then treats those as removable, which is a silent
|
||||
# behaviour change for a few kilobytes. Left as a warning suppression instead:
|
||||
# the annotation is dropped from the shrunk output anyway, because nothing reads
|
||||
# it.
|
||||
-dontwarn kotlin.Metadata
|
||||
@@ -0,0 +1,112 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!--
|
||||
The shell's manifest.
|
||||
|
||||
There is no INTERNET-adjacent permission beyond INTERNET itself: the page and
|
||||
the kernel talk over loopback, and reaching a server is the kernel's job over
|
||||
a socket it owns. Reading and writing files goes through the system's document
|
||||
picker (SAF), which hands back a grant for one file at a time — so no
|
||||
storage permission is declared and none is needed.
|
||||
|
||||
`usesCleartextTraffic` is off here, and cleartext is re-enabled in the network
|
||||
security config only because the kernel serves its page on `127.0.0.1` and a
|
||||
paired computer serves the same page over its LAN address — both plain HTTP.
|
||||
The config says what that costs and what bounds it.
|
||||
-->
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
xmlns:tools="http://schemas.android.com/tools">
|
||||
|
||||
<uses-permission android:name="android.permission.INTERNET" />
|
||||
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
|
||||
<!-- Only to tell the person the kernel is still running while the screen is
|
||||
off; it is a foreground service with a dismissible notification, not a
|
||||
background wake-up. -->
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
|
||||
<!-- From API 34 every foreground-service *type* needs its own permission as
|
||||
well as the general one: naming dataSync in the manifest without asking
|
||||
for it is a SecurityException at startForeground, on the one code path
|
||||
that keeps a long task alive. -->
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
|
||||
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
|
||||
<!-- Only to read the pairing code a computer shows. Requested at the
|
||||
moment the person asks to scan, never at launch. -->
|
||||
<uses-permission android:name="android.permission.CAMERA" />
|
||||
<!-- The kernel is a child process, and a task left running is one the person
|
||||
expects to finish with the screen off. Android's own doze can freeze the
|
||||
whole app group instead, which is what the shell asks to be exempt from
|
||||
— once, when a turn is actually running, never at launch. It is a
|
||||
request the person can refuse, and refusing it only costs battery life
|
||||
during a long turn. -->
|
||||
<uses-permission android:name="android.permission.REQUEST_IGNORE_BATTERY_OPTIMIZATIONS" />
|
||||
<!-- Every-file access, for opening a folder on this phone as a workspace.
|
||||
The kernel is a plain Go binary: it opens paths with `open(2)`, and a
|
||||
document URI is not a path it can open. SAF would hand back exactly
|
||||
that unusable URI, so the one honest permission is the broad one —
|
||||
granted by the person in system settings, which is the only place
|
||||
Android offers it. Nothing here reads storage for any other purpose. -->
|
||||
<uses-permission
|
||||
android:name="android.permission.MANAGE_EXTERNAL_STORAGE"
|
||||
tools:ignore="ScopedStorage" />
|
||||
<!-- Below API 33 the same folder walk needs the older pair; from 33 they are
|
||||
superseded by the permission above and asking for them would be a lie. -->
|
||||
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" android:maxSdkVersion="32" />
|
||||
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" android:maxSdkVersion="29" />
|
||||
|
||||
<!-- A phone without a touchscreen cannot drive this interface; saying so
|
||||
keeps the listing honest rather than letting it install on a TV. -->
|
||||
<uses-feature android:name="android.hardware.touchscreen" android:required="true" />
|
||||
<!-- The camera is optional: a phone without one is still a fine client for
|
||||
a kernel it already knows, and it can still pair by opening the link the
|
||||
computer shows. Requiring it would take the app off those devices. -->
|
||||
<uses-feature android:name="android.hardware.camera" android:required="false" />
|
||||
<uses-feature android:name="android.hardware.camera.autofocus" android:required="false" />
|
||||
|
||||
<application
|
||||
android:name=".App"
|
||||
android:allowBackup="false"
|
||||
android:dataExtractionRules="@xml/data_extraction_rules"
|
||||
android:fullBackupContent="false"
|
||||
android:icon="@mipmap/ic_launcher"
|
||||
android:label="@string/app_name"
|
||||
android:networkSecurityConfig="@xml/network_security_config"
|
||||
android:supportsRtl="true"
|
||||
android:theme="@style/Theme.Reasonix"
|
||||
android:usesCleartextTraffic="false"
|
||||
tools:targetApi="35">
|
||||
|
||||
<activity
|
||||
android:name=".MainActivity"
|
||||
android:exported="true"
|
||||
android:configChanges="orientation|screenSize|screenLayout|keyboardHidden|uiMode|density|fontScale|smallestScreenSize"
|
||||
android:launchMode="singleTask"
|
||||
android:windowSoftInputMode="adjustResize"
|
||||
android:theme="@style/Theme.Reasonix">
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
</intent-filter>
|
||||
</activity>
|
||||
|
||||
<!-- The kernel outlives the screen: a turn that takes minutes would
|
||||
otherwise be killed when the phone is pocketed. The service is
|
||||
`stopWithTask="false"` so swiping the app away ends it deliberately
|
||||
rather than silently, and `foregroundServiceType="dataSync"` is the
|
||||
closest honest description of "an agent is working". -->
|
||||
<service
|
||||
android:name=".KernelService"
|
||||
android:exported="false"
|
||||
android:foregroundServiceType="dataSync"
|
||||
android:stopWithTask="false" />
|
||||
|
||||
<!-- Reading the pairing code a computer shows. Its own activity rather
|
||||
than a dialog, because the camera preview needs the whole screen to
|
||||
be worth pointing at anything. Not exported: only this app starts
|
||||
it, and a scanner another app could open is one that could be made
|
||||
to read a code on someone else's behalf. -->
|
||||
<activity
|
||||
android:name=".ScanActivity"
|
||||
android:exported="false"
|
||||
android:screenOrientation="portrait"
|
||||
android:theme="@style/Theme.Reasonix.Scan" />
|
||||
</application>
|
||||
</manifest>
|
||||
@@ -0,0 +1,59 @@
|
||||
package io.reasonix.mobile.app
|
||||
|
||||
import android.app.Application
|
||||
import android.app.NotificationChannel
|
||||
import android.app.NotificationManager
|
||||
import android.content.Context
|
||||
|
||||
/**
|
||||
* The process-wide facts this app needs.
|
||||
*
|
||||
* The kernel is one child process, so there is exactly one of it no matter how
|
||||
* many activities exist: it is held here, lazily, and the activity and the
|
||||
* service both reach the same instance. Nothing else is stored — the theme, the
|
||||
* language and every preference belong to the page and live in its localStorage.
|
||||
*/
|
||||
class App : Application() {
|
||||
|
||||
/**
|
||||
* The kernel, created on first use.
|
||||
*
|
||||
* Lazy rather than eager so that merely launching the process to answer a
|
||||
* notification does not unpack a 15MB page.
|
||||
*/
|
||||
val kernel: Kernel by lazy { Kernel(this) }
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
createChannel()
|
||||
}
|
||||
|
||||
/**
|
||||
* A channel the person can mute without muting the app, and one they can
|
||||
* find: an agent that runs for minutes on a phone has to say so somewhere
|
||||
* the system will not kill, and a notification is the only place Android
|
||||
* offers. `IMPORTANCE_MIN` because the notification is a receipt, not an
|
||||
* interruption — the work it reports is on another machine.
|
||||
*/
|
||||
private fun createChannel() {
|
||||
// No SDK_INT guard: minSdk is 26, which is the release NotificationChannel
|
||||
// arrived in, so every device this app installs on has it.
|
||||
val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
|
||||
if (manager.getNotificationChannel(CHANNEL_KERNEL) != null) return
|
||||
val channel = NotificationChannel(
|
||||
CHANNEL_KERNEL,
|
||||
getString(R.string.channel_kernel),
|
||||
NotificationManager.IMPORTANCE_MIN,
|
||||
).apply {
|
||||
description = getString(R.string.channel_kernel_desc)
|
||||
setShowBadge(false)
|
||||
enableVibration(false)
|
||||
setSound(null, null)
|
||||
}
|
||||
manager.createNotificationChannel(channel)
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val CHANNEL_KERNEL = "reasonix-kernel"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,699 @@
|
||||
package io.reasonix.mobile.app
|
||||
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.Canvas
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.util.Base64
|
||||
import android.util.Log
|
||||
import android.webkit.WebView
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
import java.io.BufferedInputStream
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.io.OutputStream
|
||||
import java.net.ServerSocket
|
||||
import java.net.Socket
|
||||
import java.net.SocketException
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
|
||||
/**
|
||||
* The kernel's browser, as a loopback HTTP server.
|
||||
*
|
||||
* The Go kernel (`internal/platform/browser/shell.go`) drives a WebView by
|
||||
* POSTing JSON to a handful of paths on 127.0.0.1. This class is the other end
|
||||
* of that conversation: it accepts connections on a random port, parses the
|
||||
* minimal HTTP the kernel sends, and translates each path into an operation on
|
||||
* the overlay WebView.
|
||||
*
|
||||
* The server is deliberately tiny — no connection pooling, no keep-alive, no
|
||||
* chunked encoding — because the kernel is its only client and sends one
|
||||
* request per connection. A `ServerSocket` accept loop on a daemon thread is
|
||||
* the whole transport; each request is handled synchronously on its own thread.
|
||||
*
|
||||
* WebView operations must run on the UI thread. Each handler posts to the main
|
||||
* looper and waits on a latch, with a timeout that bounds the worst case (a
|
||||
* page that hangs the JS thread) to something the kernel's own 30-second client
|
||||
* timeout will not outlast.
|
||||
*
|
||||
* Memory note on `/screenshot`: a full-page `WebView.draw` into an ARGB_8888
|
||||
* bitmap at device resolution can reach tens of MB for a long page. The bitmap
|
||||
* is downsampled by a factor of 2 in each dimension before PNG encoding, which
|
||||
* keeps the peak allocation under ~10 MB for a typical 1080p viewport. The
|
||||
* original is recycled immediately after the downsample.
|
||||
*/
|
||||
/** One thing a page asked the network for, as the WebView's request
|
||||
* interceptor reports it. The interceptor is handed the method and the URL
|
||||
* and nothing about the response, so a caller may show exactly this much and
|
||||
* must not read it as a completed exchange. */
|
||||
data class BrowserRequest(val method: String, val url: String, val kind: String)
|
||||
|
||||
class BrowserShellServer(
|
||||
/** The overlay WebView, when one already exists. The routes that act on an
|
||||
* open page — back, forward, reload, close — use this: with no page there
|
||||
* is nothing to act on, and creating one to do nothing to it would start a
|
||||
* renderer nobody asked for. */
|
||||
private val existingView: () -> WebView?,
|
||||
/** The overlay WebView, created on first use. The routes that drive a page
|
||||
* use this, because the kernel can open one before the panel has ever been
|
||||
* shown: the page's own panel asks for the view when it is displayed, and
|
||||
* an agent's `browser_open` can arrive first. Requiring the view to exist
|
||||
* already answered every such call with "browser view not ready", which
|
||||
* reads as a broken engine rather than as a view that was not created yet.
|
||||
* Called on the UI thread. */
|
||||
private val ensureView: () -> WebView,
|
||||
/** Why the page in the view is the browser's own error page rather than a
|
||||
* site, as `(url, chromium error)`; null when the last load succeeded.
|
||||
* `/read` asks this, because extracting `innerText` from the error page
|
||||
* returns the browser's own "this page could not be loaded" notice, which
|
||||
* the kernel would hand to the model as though the site had said it. */
|
||||
private val loadFailure: () -> Pair<String, String>? = { null },
|
||||
/** Begin a navigation: forget the load failure recorded for the page being
|
||||
* left, drop its request list, and start the wait a following `/read`
|
||||
* joins. Called on the UI thread, before the new address is loaded, so the
|
||||
* wait is in place before any callback for the new page can fire. */
|
||||
private val startNavigation: () -> Unit = {},
|
||||
/** What the page currently in the view has asked the network for, oldest
|
||||
* first. The WebView reports a request's method and URL to its interceptor
|
||||
* but nothing about the response, so that is all a caller may show. */
|
||||
private val requests: () -> List<BrowserRequest> = { emptyList() },
|
||||
/** Wait for the navigation last asked for to settle — the page arrived, or
|
||||
* its load failed — up to the given milliseconds. Called on the request's
|
||||
* own thread, never the UI thread: the settle is signalled from the UI
|
||||
* thread, so waiting on it there would deadlock. `/load` starts a
|
||||
* navigation and returns before the page has been visited; without this a
|
||||
* `/read` right behind it still sees the URL of the page before, and a
|
||||
* failure recorded for that page was handed back as the new one's answer. */
|
||||
private val awaitNavigation: (Long) -> Unit = {},
|
||||
) {
|
||||
|
||||
companion object {
|
||||
private const val TAG = "BrowserShellServer"
|
||||
|
||||
/** How long a UI-thread operation may take before the handler gives up.
|
||||
* The kernel's client timeout is 30 s; staying under it means the kernel
|
||||
* sees a clean error rather than a dropped connection. */
|
||||
private const val UI_TIMEOUT_MS = 25_000L
|
||||
|
||||
/** The maximum text `/read` returns. Matches the kernel's `maxReadText`
|
||||
* (400 << 10). The shell truncates here and says so, rather than letting
|
||||
* the kernel truncate silently. */
|
||||
private const val MAX_READ_TEXT = 400 * 1024
|
||||
|
||||
/** Screenshot downsample factor. 2 means half width and half height. */
|
||||
private const val SCREENSHOT_SCALE = 2
|
||||
|
||||
/** The address the kernel dials. Both ends must agree on the family:
|
||||
* see [start] for what goes wrong when they do not. */
|
||||
private const val LOOPBACK_V4 = "127.0.0.1"
|
||||
|
||||
/** How long `/read` waits for a navigation it follows to settle before
|
||||
* reading whatever the view is on. A page that never finishes loading
|
||||
* must not hold the tool call open to the kernel's own timeout. */
|
||||
private const val NAV_SETTLE_MS = 10_000L
|
||||
|
||||
/** How long `/read` then waits for the page's own script to answer.
|
||||
* `NAV_SETTLE_MS + READ_SCRIPT_MS` has to stay under [UI_TIMEOUT_MS],
|
||||
* and that under the kernel's 30-second client timeout: a read that
|
||||
* waited out both budgets would outlast the caller, which would see a
|
||||
* dropped connection instead of this shell's answer. 10 + 15 = 25 s. */
|
||||
private const val READ_SCRIPT_MS = 15_000L
|
||||
}
|
||||
|
||||
@Volatile
|
||||
private var serverSocket: ServerSocket? = null
|
||||
|
||||
/** The port the server is listening on. Valid after [start] returns. */
|
||||
@Volatile
|
||||
var port: Int = 0
|
||||
private set
|
||||
|
||||
/**
|
||||
* The bearer credential every request must carry. Minted by the activity
|
||||
* and handed to the kernel through the child's environment, so the server
|
||||
* answers only the process this shell started.
|
||||
*
|
||||
* A phone's loopback is not a boundary: any other app may connect to
|
||||
* 127.0.0.1, and the routes here run script in and read text out of
|
||||
* whatever page is loaded. Binding to `getLoopbackAddress()` keeps other
|
||||
* machines out but not other apps, so the credential is what makes "only
|
||||
* the kernel this shell started" true.
|
||||
*/
|
||||
@Volatile
|
||||
var token: String = ""
|
||||
private set
|
||||
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
|
||||
/**
|
||||
* Start listening on 127.0.0.1 with an ephemeral port. The port is available
|
||||
* via [port] immediately after this returns, and the credential the kernel
|
||||
* must present via [token].
|
||||
*/
|
||||
fun start() {
|
||||
token = newToken()
|
||||
// The loopback address is spelled out as IPv4, not taken from
|
||||
// `InetAddress.getLoopbackAddress()`. That call is allowed to answer
|
||||
// either family and Android's libcore answers IPv6 (`::1`), while the
|
||||
// kernel's client is hardcoded to `http://127.0.0.1:<port>`
|
||||
// (`internal/assembly/boot/browser.go`). A socket bound to `::1` does
|
||||
// not answer a dial to `127.0.0.1`, so every browser call failed with
|
||||
// "connection refused" while the port was bound and reported — the
|
||||
// failure looked like a server that never started. Both ends have to
|
||||
// name the same family, and IPv4 is the one the kernel names.
|
||||
val ss = ServerSocket(0, 16, java.net.InetAddress.getByName(LOOPBACK_V4))
|
||||
serverSocket = ss
|
||||
port = ss.localPort
|
||||
Log.i(TAG, "browser shell server listening on $LOOPBACK_V4:$port")
|
||||
|
||||
Thread({
|
||||
while (!ss.isClosed) {
|
||||
try {
|
||||
val client = ss.accept()
|
||||
Thread({ handleClient(client, token) }, "browser-shell-conn").apply { isDaemon = true }.start()
|
||||
} catch (_: SocketException) {
|
||||
// Server was closed; exit the loop.
|
||||
}
|
||||
}
|
||||
}, "browser-shell-accept").apply { isDaemon = true }.start()
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
runCatching { serverSocket?.close() }
|
||||
serverSocket = null
|
||||
}
|
||||
|
||||
/** A fresh 256-bit credential, hex-encoded, for one launch of the server. */
|
||||
private fun newToken(): String {
|
||||
val bytes = ByteArray(32)
|
||||
java.security.SecureRandom().nextBytes(bytes)
|
||||
return bytes.joinToString("") { "%02x".format(it) }
|
||||
}
|
||||
|
||||
// ---- HTTP plumbing --------------------------------------------------------
|
||||
|
||||
private fun handleClient(client: Socket, expected: String) {
|
||||
try {
|
||||
client.soTimeout = 30_000
|
||||
val input = BufferedInputStream(client.getInputStream())
|
||||
val output = client.getOutputStream()
|
||||
|
||||
// Parse the request line.
|
||||
val requestLine = readLine(input) ?: return
|
||||
val parts = requestLine.split(" ", limit = 3)
|
||||
if (parts.size < 2) return
|
||||
val method = parts[0]
|
||||
val path = parts[1].substringBefore('?')
|
||||
|
||||
// Read headers.
|
||||
var contentLength = 0
|
||||
var bearer: String? = null
|
||||
while (true) {
|
||||
val header = readLine(input) ?: break
|
||||
if (header.isEmpty()) break
|
||||
val lower = header.lowercase()
|
||||
if (lower.startsWith("content-length:")) {
|
||||
contentLength = header.substringAfter(':').trim().toIntOrNull() ?: 0
|
||||
} else if (lower.startsWith("authorization:")) {
|
||||
bearer = header.substringAfter(':').trim().removePrefix("Bearer ").trim()
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing behind this server is safe to hand out on the strength of
|
||||
// an address: the port is on loopback, which any other app on the
|
||||
// phone can dial, and the routes run script in and read text out of
|
||||
// the loaded page. The credential the kernel was started with is the
|
||||
// only thing that says a caller is that kernel. Every route is
|
||||
// refused without it, /health included — a probe needs no exemption.
|
||||
if (expected.isEmpty() || bearer != expected) {
|
||||
writeResponse(output, 401, """{"error":"unauthorized"}""")
|
||||
return
|
||||
}
|
||||
|
||||
// Read body.
|
||||
val body = if (contentLength > 0) {
|
||||
val buf = ByteArray(contentLength)
|
||||
var read = 0
|
||||
while (read < contentLength) {
|
||||
val n = input.read(buf, read, contentLength - read)
|
||||
if (n == -1) break
|
||||
read += n
|
||||
}
|
||||
String(buf, 0, read, Charsets.UTF_8)
|
||||
} else ""
|
||||
|
||||
// Route.
|
||||
val (status, responseBody) = route(method, path, body)
|
||||
writeResponse(output, status, responseBody)
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "connection error: ${e.message}")
|
||||
} finally {
|
||||
runCatching { client.close() }
|
||||
}
|
||||
}
|
||||
|
||||
private fun route(method: String, path: String, body: String): Pair<Int, String> {
|
||||
return when {
|
||||
method == "GET" && path == "/health" -> 200 to """{"ok":true}"""
|
||||
method == "POST" && path == "/load" -> handleLoad(body)
|
||||
method == "POST" && path == "/close" -> handleClose()
|
||||
method == "POST" && path == "/read" -> handleRead()
|
||||
method == "POST" && path == "/screenshot" -> handleScreenshot()
|
||||
method == "POST" && path == "/click" -> handleClick(body)
|
||||
method == "POST" && path == "/eval" -> handleEval(body)
|
||||
method == "POST" && path == "/back" -> handleBack()
|
||||
method == "POST" && path == "/forward" -> handleForward()
|
||||
method == "POST" && path == "/reload" -> handleReload()
|
||||
method == "POST" && path == "/requests" -> handleRequests()
|
||||
else -> 404 to """{"error":"not found"}"""
|
||||
}
|
||||
}
|
||||
|
||||
private fun writeResponse(out: OutputStream, status: Int, body: String) {
|
||||
val reason = when (status) {
|
||||
200 -> "OK"
|
||||
404 -> "Not Found"
|
||||
500 -> "Internal Server Error"
|
||||
else -> "Error"
|
||||
}
|
||||
val bytes = body.toByteArray(Charsets.UTF_8)
|
||||
val header = "HTTP/1.1 $status $reason\r\n" +
|
||||
"Content-Type: application/json\r\n" +
|
||||
"Content-Length: ${bytes.size}\r\n" +
|
||||
"Connection: close\r\n" +
|
||||
"\r\n"
|
||||
out.write(header.toByteArray(Charsets.UTF_8))
|
||||
out.write(bytes)
|
||||
out.flush()
|
||||
}
|
||||
|
||||
private fun readLine(input: BufferedInputStream): String? {
|
||||
val sb = StringBuilder()
|
||||
while (true) {
|
||||
val b = input.read()
|
||||
if (b == -1) return if (sb.isEmpty()) null else sb.toString()
|
||||
if (b == '\r'.code) {
|
||||
val next = input.read()
|
||||
if (next == '\n'.code) return sb.toString()
|
||||
sb.append(b.toChar())
|
||||
if (next != -1) sb.append(next.toChar())
|
||||
continue
|
||||
}
|
||||
if (b == '\n'.code) return sb.toString()
|
||||
sb.append(b.toChar())
|
||||
}
|
||||
}
|
||||
|
||||
// ---- endpoint handlers ----------------------------------------------------
|
||||
|
||||
/** POST /load {"url":"..."} → {"url":"...","title":"..."} */
|
||||
private fun handleLoad(body: String): Pair<Int, String> {
|
||||
val json = runCatching { JSONObject(body) }.getOrNull()
|
||||
?: return 500 to """{"error":"invalid request body"}"""
|
||||
val url = json.optString("url", "")
|
||||
if (url.isEmpty()) return 500 to """{"error":"missing url"}"""
|
||||
|
||||
val result = AtomicReference("")
|
||||
val latch = CountDownLatch(1)
|
||||
mainHandler.post {
|
||||
val wv = ensureView()
|
||||
// The failure recorded for the page being left stops being this
|
||||
// tab's, and the wait a following `/read` joins is armed before the
|
||||
// new address is loaded, so no callback for the new page can fire
|
||||
// ahead of it. `/load` answers before the page arrives, which is
|
||||
// what left the old failure to be read back as the new page's.
|
||||
startNavigation()
|
||||
wv.loadUrl(url)
|
||||
// The title and final URL are not available synchronously after
|
||||
// loadUrl; answer with what was requested. The kernel stores this
|
||||
// as the tab's URL and reads the real one on the next /read.
|
||||
result.set(JSONObject().apply {
|
||||
put("url", url)
|
||||
put("title", "")
|
||||
}.toString())
|
||||
latch.countDown()
|
||||
}
|
||||
if (!latch.await(UI_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
|
||||
return 500 to """{"error":"timeout waiting for UI thread"}"""
|
||||
}
|
||||
return 200 to result.get()
|
||||
}
|
||||
|
||||
/** POST /close {} → 200 */
|
||||
private fun handleClose(): Pair<Int, String> {
|
||||
val latch = CountDownLatch(1)
|
||||
mainHandler.post {
|
||||
val wv = existingView()
|
||||
if (wv != null) {
|
||||
wv.loadUrl("about:blank")
|
||||
wv.clearHistory()
|
||||
}
|
||||
latch.countDown()
|
||||
}
|
||||
if (!latch.await(UI_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
|
||||
return 500 to """{"error":"timeout"}"""
|
||||
}
|
||||
return 200 to "{}"
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /read {} → {"title":"...","url":"...","text":"..."}
|
||||
*
|
||||
* Injects a script that extracts readable text from the page, preferring
|
||||
* `<article>` or `<main>` and falling back to `body.innerText`. The result
|
||||
* is truncated to 400 KB (matching the kernel's `maxReadText`); when
|
||||
* truncated, a note is appended so the model knows the text was cut.
|
||||
*/
|
||||
private fun handleRead(): Pair<Int, String> {
|
||||
// The page the caller just asked for has not necessarily been visited
|
||||
// yet: `/load` returns as soon as it has started the navigation. Waiting
|
||||
// here, off the UI thread, is what makes this read answer about that
|
||||
// page instead of the one before it.
|
||||
awaitNavigation(NAV_SETTLE_MS)
|
||||
val result = AtomicReference("")
|
||||
val latch = CountDownLatch(1)
|
||||
mainHandler.post {
|
||||
val wv = ensureView()
|
||||
// A page that did not load is not content. Whether the view is on
|
||||
// the browser's own error page is read from the view itself rather
|
||||
// than from a remembered failure: a failure recorded for the page
|
||||
// before a new navigation would otherwise be handed back as this
|
||||
// page's answer, long after the page the caller asked for had
|
||||
// loaded. The remembered failure is still what names the address
|
||||
// that failed and why, since the error page's own URL does not.
|
||||
if ((wv.url ?: "").startsWith("chrome-error:")) {
|
||||
val (url, reason) = loadFailure() ?: ((wv.url ?: "") to "ERR_FAILED")
|
||||
result.set(JSONObject().apply {
|
||||
put("url", url)
|
||||
put("title", "")
|
||||
put("reason", reason)
|
||||
put("text", "")
|
||||
}.toString())
|
||||
latch.countDown()
|
||||
return@post
|
||||
}
|
||||
// The extraction script: prefer article/main for content pages,
|
||||
// fall back to body. innerText preserves visual line breaks. It
|
||||
// also stamps a ref on every element a person could act on, so the
|
||||
// kernel can name one in an act step the way the desktop backend
|
||||
// names an accessibility node. A ref already on an element is kept,
|
||||
// so it stays the same across snapshots of one document; a fresh
|
||||
// document starts the numbering over, and the kernel only accepts a
|
||||
// ref from the snapshot it last read.
|
||||
val script = """
|
||||
(function(){
|
||||
var seq = window.__rxRefSeq || 0;
|
||||
var used = {};
|
||||
var marked = document.querySelectorAll('[data-rx-ref]');
|
||||
for (var i = 0; i < marked.length; i++) { used[marked[i].getAttribute('data-rx-ref')] = true; }
|
||||
function roleOf(el) {
|
||||
var tag = el.tagName.toLowerCase();
|
||||
var role = (el.getAttribute('role') || '').toLowerCase();
|
||||
if (role) return role;
|
||||
if (tag === 'a') return 'link';
|
||||
if (tag === 'button') return 'button';
|
||||
if (tag === 'select') return 'combobox';
|
||||
if (tag === 'textarea') return 'textbox';
|
||||
if (tag === 'input') {
|
||||
var t = (el.getAttribute('type') || 'text').toLowerCase();
|
||||
if (t === 'checkbox' || t === 'radio' || t === 'submit' || t === 'button' || t === 'reset') return t;
|
||||
return 'textbox';
|
||||
}
|
||||
return 'generic';
|
||||
}
|
||||
function nameOf(el) {
|
||||
var n = el.getAttribute('aria-label') || el.getAttribute('placeholder') || el.getAttribute('name') || '';
|
||||
if (!n) { n = (el.innerText || el.textContent || '').trim(); }
|
||||
return n.replace(/\s+/g, ' ').slice(0, 160);
|
||||
}
|
||||
var sel = 'a[href],button,input,select,textarea,[role=button],[role=link],[role=checkbox],[role=radio],[role=tab],[role=menuitem],[contenteditable=true]';
|
||||
var els = document.querySelectorAll(sel);
|
||||
var elements = [];
|
||||
for (var j = 0; j < els.length; j++) {
|
||||
var el = els[j];
|
||||
var ref = el.getAttribute('data-rx-ref');
|
||||
if (!ref) {
|
||||
do { seq++; ref = 'e' + seq; } while (used[ref]);
|
||||
el.setAttribute('data-rx-ref', ref);
|
||||
used[ref] = true;
|
||||
}
|
||||
var raw = (el.value != null && String(el.value) !== '') ? String(el.value).slice(0, 160) : '';
|
||||
elements.push({ref: ref, role: roleOf(el), name: nameOf(el), value: raw});
|
||||
}
|
||||
window.__rxRefSeq = seq;
|
||||
var el2 = document.querySelector('article') || document.querySelector('main') || document.body;
|
||||
var text = el2 ? (el2.innerText || '') : '';
|
||||
var title = document.title || '';
|
||||
var url = location.href;
|
||||
return JSON.stringify({title: title, url: url, text: text, elements: elements});
|
||||
})()
|
||||
""".trimIndent()
|
||||
wv.evaluateJavascript(script) { value ->
|
||||
// evaluateJavascript returns a JSON-encoded string (the result
|
||||
// is double-encoded). Strip the outer quotes.
|
||||
val parsed = parseJsResult(value)
|
||||
if (parsed != null) {
|
||||
try {
|
||||
val obj = JSONObject(parsed)
|
||||
var text = obj.optString("text", "")
|
||||
if (text.length > MAX_READ_TEXT) {
|
||||
text = text.substring(0, MAX_READ_TEXT) +
|
||||
"\n\n[text truncated at 400 KB]"
|
||||
}
|
||||
result.set(JSONObject().apply {
|
||||
put("title", obj.optString("title", ""))
|
||||
put("url", obj.optString("url", ""))
|
||||
put("text", text)
|
||||
put("elements", obj.optJSONArray("elements") ?: JSONArray())
|
||||
}.toString())
|
||||
} catch (e: Exception) {
|
||||
result.set(JSONObject().apply {
|
||||
put("title", "")
|
||||
put("url", wv.url ?: "")
|
||||
put("text", parsed.take(MAX_READ_TEXT))
|
||||
}.toString())
|
||||
}
|
||||
} else {
|
||||
result.set(JSONObject().apply {
|
||||
put("title", "")
|
||||
put("url", wv.url ?: "")
|
||||
put("text", "")
|
||||
}.toString())
|
||||
}
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
if (!latch.await(READ_SCRIPT_MS, TimeUnit.MILLISECONDS)) {
|
||||
return 500 to """{"error":"timeout waiting for page script"}"""
|
||||
}
|
||||
return 200 to result.get()
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /screenshot {} → {"data":"<base64>","mime":"image/png"}
|
||||
*
|
||||
* Draws the WebView into a bitmap, downsamples by SCREENSHOT_SCALE to keep
|
||||
* memory bounded, and encodes as PNG → base64. For a 1080×2400 viewport the
|
||||
* full bitmap is ~10 MB; the downsampled one is ~2.5 MB. The full bitmap is
|
||||
* recycled immediately after scaling.
|
||||
*/
|
||||
private fun handleScreenshot(): Pair<Int, String> {
|
||||
val result = AtomicReference("")
|
||||
val failed = AtomicBoolean(false)
|
||||
val latch = CountDownLatch(1)
|
||||
mainHandler.post {
|
||||
val wv = ensureView()
|
||||
try {
|
||||
val width = wv.width.takeIf { it > 0 } ?: 1080
|
||||
val height = wv.height.takeIf { it > 0 } ?: 1920
|
||||
val bmp = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
|
||||
val canvas = Canvas(bmp)
|
||||
wv.draw(canvas)
|
||||
|
||||
// Downsample to bound the PNG size and the base64 string.
|
||||
val scaled = Bitmap.createScaledBitmap(
|
||||
bmp,
|
||||
width / SCREENSHOT_SCALE,
|
||||
height / SCREENSHOT_SCALE,
|
||||
true,
|
||||
)
|
||||
if (scaled !== bmp) bmp.recycle()
|
||||
|
||||
val baos = ByteArrayOutputStream()
|
||||
scaled.compress(Bitmap.CompressFormat.PNG, 85, baos)
|
||||
scaled.recycle()
|
||||
val b64 = Base64.encodeToString(baos.toByteArray(), Base64.NO_WRAP)
|
||||
result.set(JSONObject().apply {
|
||||
put("data", b64)
|
||||
put("mime", "image/png")
|
||||
}.toString())
|
||||
} catch (e: Exception) {
|
||||
// A 200 with an error body would be read by the kernel as a
|
||||
// screenshot whose data is empty, and it would build a `data:`
|
||||
// URI out of nothing. The status is what tells it this failed.
|
||||
result.set("""{"error":"screenshot failed: ${e.message}"}""")
|
||||
failed.set(true)
|
||||
}
|
||||
latch.countDown()
|
||||
}
|
||||
if (!latch.await(UI_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
|
||||
return 500 to """{"error":"timeout"}"""
|
||||
}
|
||||
if (failed.get()) return 500 to result.get()
|
||||
return 200 to result.get()
|
||||
}
|
||||
|
||||
/** POST /click {"selector":"..."} → {"ok":true|false} */
|
||||
private fun handleClick(body: String): Pair<Int, String> {
|
||||
val json = runCatching { JSONObject(body) }.getOrNull()
|
||||
?: return 500 to """{"error":"invalid request body"}"""
|
||||
val selector = json.optString("selector", "")
|
||||
if (selector.isEmpty()) return 500 to """{"error":"missing selector"}"""
|
||||
|
||||
val result = AtomicReference("")
|
||||
val latch = CountDownLatch(1)
|
||||
mainHandler.post {
|
||||
val wv = ensureView()
|
||||
// Escape the selector for embedding in a JS string.
|
||||
val escaped = selector.replace("\\", "\\\\").replace("'", "\\'")
|
||||
val script = """
|
||||
(function(){
|
||||
try {
|
||||
var el = document.querySelector('$escaped');
|
||||
if (!el) return JSON.stringify({ok: false});
|
||||
el.click();
|
||||
return JSON.stringify({ok: true});
|
||||
} catch(e) {
|
||||
return JSON.stringify({ok: false});
|
||||
}
|
||||
})()
|
||||
""".trimIndent()
|
||||
wv.evaluateJavascript(script) { value ->
|
||||
val parsed = parseJsResult(value)
|
||||
result.set(parsed ?: """{"ok":false}""")
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
if (!latch.await(UI_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
|
||||
return 500 to """{"error":"timeout"}"""
|
||||
}
|
||||
return 200 to result.get()
|
||||
}
|
||||
|
||||
/** POST /eval {"script":"..."} → {"result":"..."} */
|
||||
private fun handleEval(body: String): Pair<Int, String> {
|
||||
val json = runCatching { JSONObject(body) }.getOrNull()
|
||||
?: return 500 to """{"error":"invalid request body"}"""
|
||||
val script = json.optString("script", "")
|
||||
if (script.isEmpty()) return 500 to """{"error":"missing script"}"""
|
||||
|
||||
val result = AtomicReference("")
|
||||
val latch = CountDownLatch(1)
|
||||
mainHandler.post {
|
||||
val wv = ensureView()
|
||||
wv.evaluateJavascript(script) { value ->
|
||||
// The kernel expects a plain string in "result". The value from
|
||||
// evaluateJavascript is JSON-encoded; unwrap it.
|
||||
val unwrapped = parseJsResult(value) ?: ""
|
||||
result.set(JSONObject().apply {
|
||||
put("result", unwrapped)
|
||||
}.toString())
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
if (!latch.await(UI_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
|
||||
return 500 to """{"error":"timeout waiting for eval"}"""
|
||||
}
|
||||
return 200 to result.get()
|
||||
}
|
||||
|
||||
/** POST /back → 200 */
|
||||
private fun handleBack(): Pair<Int, String> {
|
||||
val latch = CountDownLatch(1)
|
||||
mainHandler.post {
|
||||
existingView()?.let { if (it.canGoBack()) it.goBack() }
|
||||
latch.countDown()
|
||||
}
|
||||
if (!latch.await(UI_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
|
||||
return 500 to """{"error":"timeout"}"""
|
||||
}
|
||||
return 200 to "{}"
|
||||
}
|
||||
|
||||
/** POST /forward → 200 */
|
||||
private fun handleForward(): Pair<Int, String> {
|
||||
val latch = CountDownLatch(1)
|
||||
mainHandler.post {
|
||||
existingView()?.let { if (it.canGoForward()) it.goForward() }
|
||||
latch.countDown()
|
||||
}
|
||||
if (!latch.await(UI_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
|
||||
return 500 to """{"error":"timeout"}"""
|
||||
}
|
||||
return 200 to "{}"
|
||||
}
|
||||
|
||||
/** POST /reload → 200 */
|
||||
private fun handleReload(): Pair<Int, String> {
|
||||
val latch = CountDownLatch(1)
|
||||
mainHandler.post {
|
||||
val wv = existingView()
|
||||
// A view sitting on the browser's own error page is not on the site,
|
||||
// and reloading it would retry the error page itself. The address
|
||||
// that failed is loaded again instead — which is the page a caller
|
||||
// means by "reload" — read before `startNavigation` clears it.
|
||||
val failed = if (wv != null && (wv.url ?: "").startsWith("chrome-error:"))
|
||||
loadFailure()?.first else null
|
||||
startNavigation()
|
||||
if (wv != null) {
|
||||
if (failed != null) wv.loadUrl(failed) else wv.reload()
|
||||
}
|
||||
latch.countDown()
|
||||
}
|
||||
if (!latch.await(UI_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
|
||||
return 500 to """{"error":"timeout"}"""
|
||||
}
|
||||
return 200 to "{}"
|
||||
}
|
||||
|
||||
/** POST /requests {} → {"requests":[{"method","url","kind"},…]} */
|
||||
private fun handleRequests(): Pair<Int, String> {
|
||||
val arr = JSONArray()
|
||||
for (r in requests()) {
|
||||
arr.put(JSONObject().apply {
|
||||
put("method", r.method)
|
||||
put("url", r.url)
|
||||
put("kind", r.kind)
|
||||
})
|
||||
}
|
||||
return 200 to JSONObject().apply { put("requests", arr) }.toString()
|
||||
}
|
||||
|
||||
// ---- helpers ----------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Unwrap the result of `evaluateJavascript`. The callback receives a
|
||||
* JSON-encoded string: a JS string "hello" arrives as `"\"hello\""`, a
|
||||
* number 42 as `"42"`, null as `"null"`. Strip the outer JSON encoding to
|
||||
* get the raw value.
|
||||
*/
|
||||
private fun parseJsResult(value: String?): String? {
|
||||
if (value == null || value == "null") return null
|
||||
// The value is a JSON literal. If it is a JSON string, decode it.
|
||||
return try {
|
||||
if (value.startsWith("\"") && value.endsWith("\"")) {
|
||||
JSONObject().apply { }.let {
|
||||
// Use JSONObject's string decoding: wrap in an object.
|
||||
JSONObject("""{"v":$value}""").optString("v", "")
|
||||
}
|
||||
} else {
|
||||
value
|
||||
}
|
||||
} catch (_: Exception) {
|
||||
value
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package io.reasonix.mobile.app
|
||||
|
||||
import android.content.Context
|
||||
import android.net.ConnectivityManager
|
||||
import android.net.LinkProperties
|
||||
import android.os.Build
|
||||
|
||||
/**
|
||||
* The phone's DNS servers, as the kernel needs them.
|
||||
*
|
||||
* The kernel is a CGO-free Go binary, and Go's own source explains why that
|
||||
* matters:
|
||||
*
|
||||
* ```
|
||||
* // DNS requests don't work on Android, so prefer the cgo resolver.
|
||||
* // Issue #10714.
|
||||
* case "android":
|
||||
* ```
|
||||
*
|
||||
* A normal Android program resolves names through `getaddrinfo`, which reaches
|
||||
* the `netd` daemon and gets the servers of the network that is actually up.
|
||||
* A static binary has no cgo resolver, so Go reads `/etc/resolv.conf` instead —
|
||||
* and Android has no such file. Go then falls back to `127.0.0.1:53`, where
|
||||
* nothing listens, and every lookup fails: the setup screen reports
|
||||
* "连接失败" for every provider, including the official ones, on a phone whose
|
||||
* browser reaches them fine.
|
||||
*
|
||||
* So they are read here and handed to the kernel through `REASONIX_DNS`. This is
|
||||
* the same source the system itself uses, so a VPN, a private DNS hostname, or a
|
||||
* network that changed since boot is all reflected without this code knowing
|
||||
* anything about them.
|
||||
*/
|
||||
object Dns {
|
||||
|
||||
/**
|
||||
* The active network's servers, comma separated, or "" when there is none.
|
||||
*
|
||||
* Read from the *active* network rather than from every link, because a phone
|
||||
* commonly has several at once — Wi-Fi, mobile data, a VPN — and the servers
|
||||
* of a network that is not carrying traffic are the wrong ones to ask.
|
||||
*/
|
||||
fun servers(context: Context): String {
|
||||
val manager = context.getSystemService(Context.CONNECTIVITY_SERVICE) as? ConnectivityManager
|
||||
?: return ""
|
||||
val properties: LinkProperties = runCatching { manager.activeNetwork }.getOrNull()
|
||||
?.let { network -> runCatching { manager.getLinkProperties(network) }.getOrNull() }
|
||||
?: return ""
|
||||
return properties.dnsServers
|
||||
.mapNotNull { address -> address.hostAddress }
|
||||
.filter { it.isNotBlank() }
|
||||
.filter { routable(it) }
|
||||
.sortedBy { preference(it) }
|
||||
.joinToString(",")
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an address can actually carry a query.
|
||||
*
|
||||
* A link-local address (fe80::/10) without its zone cannot: the packet has
|
||||
* no interface to leave by, so it is dropped, nothing answers, and the
|
||||
* lookup hangs until the kernel's own timeout expires — which is reported as
|
||||
* a timeout rather than as a failure to resolve, and so reads like the
|
||||
* network is slow instead of like the address is unusable.
|
||||
*
|
||||
* Android reports the zone as a `%wlan0` suffix on `hostAddress`, and the
|
||||
* address is only usable with it kept. Nothing here trims it: an earlier
|
||||
* version called `substringBefore('%')`, which turned a usable link-local
|
||||
* server into an unroutable one.
|
||||
*/
|
||||
private fun routable(host: String): Boolean =
|
||||
!(host.startsWith("fe80:", ignoreCase = true) && !host.contains('%'))
|
||||
|
||||
/**
|
||||
* IPv4 first, then IPv6.
|
||||
*
|
||||
* A phone commonly reports both, and its IPv6 entry is often a router's
|
||||
* link-local address — reachable on one network and not the next. The
|
||||
* kernel tries the servers in the order it is given them, and stops at the
|
||||
* first that answers; putting the address family that shares the network's
|
||||
* own numbering first means its timeout is spent on a server that works.
|
||||
*/
|
||||
private fun preference(host: String): Int = if (host.contains(':')) 1 else 0
|
||||
|
||||
/**
|
||||
* The private-DNS setting, for the one case the address list cannot answer.
|
||||
*
|
||||
* Since Android 9 a phone may use DNS-over-TLS, and then `dnsServers` holds a
|
||||
* name rather than an address — a name this kernel cannot resolve *because*
|
||||
* resolving it is the thing that is broken. There is no way for the shell to
|
||||
* do that lookup for it, so this reports the situation instead of handing
|
||||
* over an unusable value: the caller says so in the log, and the person can
|
||||
* turn private DNS off or set it to automatic.
|
||||
*/
|
||||
fun looksLikeDot(servers: String): Boolean =
|
||||
servers.isNotBlank() && servers.split(',').any { part ->
|
||||
val host = part.trim()
|
||||
host.isNotEmpty() && host.none { it == ':' } && !host.all { it.isDigit() || it == '.' }
|
||||
}
|
||||
|
||||
/** Whether the system requires a VPN or private DNS, which changes nothing
|
||||
* here but is worth saying in a bug report. */
|
||||
fun describe(context: Context): String {
|
||||
val manager = context.getSystemService(Context.CONNECTIVITY_SERVICE) as? ConnectivityManager
|
||||
?: return "no connectivity service"
|
||||
val network = runCatching { manager.activeNetwork }.getOrNull() ?: return "no active network"
|
||||
val properties = runCatching { manager.getLinkProperties(network) }.getOrNull()
|
||||
?: return "no link properties"
|
||||
// The DHCP server arrived in API 30, not 23: asked for below that it is a
|
||||
// NoSuchMethodError on a device this app supports (minSdk 26), and the
|
||||
// answer is only ever read out of this string by a person debugging.
|
||||
val dhcp = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
|
||||
properties.dhcpServerAddress?.hostAddress
|
||||
} else {
|
||||
null
|
||||
}
|
||||
return "iface=${properties.interfaceName} dns=${properties.dnsServers.joinToString(",") { it.hostAddress ?: "?" }}" +
|
||||
(dhcp?.let { " dhcp=$it" } ?: "")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package io.reasonix.mobile.app
|
||||
|
||||
import android.content.Context
|
||||
import android.net.Uri
|
||||
import android.webkit.CookieManager
|
||||
import android.widget.Toast
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.FileNotFoundException
|
||||
import java.net.HttpURLConnection
|
||||
import java.net.URL
|
||||
|
||||
/**
|
||||
* Putting a file the page produced onto the phone's disk.
|
||||
*
|
||||
* A WebView has no downloads shelf and cannot write to the system's document
|
||||
* picker itself, so the two halves are joined here: the page asks, the picker
|
||||
* answers with a `content://` the app has been granted, and this copies the
|
||||
* bytes across.
|
||||
*
|
||||
* The request carries the WebView's own cookie, read out of the jar the page
|
||||
* authenticated in. That is the only reason this is not a plain `openStream`:
|
||||
* the kernel's download route is behind the session the page established, and a
|
||||
* shell that fetched without it would get a 403 and no explanation.
|
||||
*/
|
||||
object Downloader {
|
||||
|
||||
suspend fun run(
|
||||
context: Context,
|
||||
url: String,
|
||||
userAgent: String,
|
||||
mime: String,
|
||||
name: String,
|
||||
target: Uri,
|
||||
): Result<Unit> = withContext(Dispatchers.IO) {
|
||||
runCatching {
|
||||
val connection = (URL(url).openConnection() as HttpURLConnection).apply {
|
||||
connectTimeout = 15_000
|
||||
readTimeout = 60_000
|
||||
requestMethod = "GET"
|
||||
if (userAgent.isNotBlank()) setRequestProperty("User-Agent", userAgent)
|
||||
// The download route answers with the same session cookie the
|
||||
// page is using; without it the kernel refuses the request.
|
||||
cookieFor(url)?.let { setRequestProperty("Cookie", it) }
|
||||
}
|
||||
try {
|
||||
val code = connection.responseCode
|
||||
if (code !in 200..299) {
|
||||
throw IllegalStateException(
|
||||
context.getString(R.string.download_failed_status, code),
|
||||
)
|
||||
}
|
||||
val resolver = context.contentResolver
|
||||
val output = resolver.openOutputStream(target, "w")
|
||||
?: throw FileNotFoundException(target.toString())
|
||||
output.use { out ->
|
||||
connection.inputStream.use { input -> input.copyTo(out) }
|
||||
}
|
||||
// `copyTo` answers with a byte count; what the caller wants to
|
||||
// know is that there were no bytes missing, not how many.
|
||||
Unit
|
||||
} finally {
|
||||
connection.disconnect()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The cookie the page authenticated with, if any.
|
||||
*
|
||||
* `CookieManager` is the WebView's own store, so this asks the exact jar the
|
||||
* request will be checked against rather than a copy this shell kept.
|
||||
*/
|
||||
private fun cookieFor(url: String): String? =
|
||||
runCatching { CookieManager.getInstance().getCookie(url) }.getOrNull()?.takeIf { it.isNotBlank() }
|
||||
|
||||
/** A short receipt for a tap that otherwise does nothing visible. */
|
||||
fun report(context: Context, name: String, result: Result<Unit>) {
|
||||
val message = result.fold(
|
||||
onSuccess = { context.getString(R.string.download_saved, name) },
|
||||
onFailure = { failure ->
|
||||
context.getString(R.string.download_failed, failure.message ?: "")
|
||||
},
|
||||
)
|
||||
Toast.makeText(context, message, Toast.LENGTH_LONG).show()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,741 @@
|
||||
package io.reasonix.mobile.app
|
||||
|
||||
import android.content.Context
|
||||
import android.content.res.AssetManager
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
import android.util.Log
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import java.net.HttpURLConnection
|
||||
import java.net.URL
|
||||
import java.security.SecureRandom
|
||||
import java.util.Locale
|
||||
|
||||
/**
|
||||
* The Reasonix kernel, as this app runs it: one child process serving loopback,
|
||||
* plus the two things it needs on disk to start — the page it serves and a
|
||||
* credential for this launch.
|
||||
*
|
||||
* The shape is the desktop host's (`cmd/reasonix-studio-host`), narrowed to what
|
||||
* a phone does not have. That host binds a socket, mints a credential, speaks a
|
||||
* handshake down a pipe and holds a lease; here the credential goes through
|
||||
* `--token-file` and the bound address through `--port-file`, because a phone's
|
||||
* app process has no pipe to hand a child and no reason to care which port was
|
||||
* free. Everything after that — the kernel's own gate, its auth middleware, its
|
||||
* CSRF check — is untouched, so the security of the surface is the one upstream
|
||||
* tested, not one this shell invented.
|
||||
*
|
||||
* One instance lives in the app process. The activity asks it for a URL; the
|
||||
* foreground service asks it to stay alive. Both see the same process, so there
|
||||
* is nothing to synchronise beyond a lock on start/stop.
|
||||
*/
|
||||
class Kernel(private val context: Context) {
|
||||
|
||||
/** Where everything the kernel owns lives. Nothing here is backed up. */
|
||||
private val home = File(context.filesDir, "home")
|
||||
|
||||
/**
|
||||
* The folder the agent works in, which is not the folder the kernel lives in.
|
||||
*
|
||||
* These used to be the same directory, and that is what made the workbench
|
||||
* unreadable: the kernel's home holds `config.toml`, `projects/`, `cache/`,
|
||||
* `stats/` and the rest of its own bookkeeping, and a workspace pointed at
|
||||
* it shows all of that as if the person had put it there. Reported as "why
|
||||
* does the default workspace have so many files, the desktop version does
|
||||
* not generate these" — the desktop does generate them, in a different
|
||||
* directory, because its workspace is whatever folder it was opened on.
|
||||
*
|
||||
* Split apart, the workbench shows what the agent produced and nothing else.
|
||||
* `write_roots` follows the workspace, so the file tools stay confined to
|
||||
* the folder the person actually works in rather than to the kernel's own
|
||||
* home — which is the stricter answer, and the one the desktop already has.
|
||||
*/
|
||||
private val workspaceDir = File(context.filesDir, "workspace")
|
||||
|
||||
/**
|
||||
* The directory the kernel calls `REASONIX_HOME`, for the one caller that
|
||||
* has to put a file where the kernel can read it — a private key the person
|
||||
* picked from their Documents folder.
|
||||
*/
|
||||
fun homeDir(): File = home
|
||||
|
||||
/** The folder the agent works in. */
|
||||
fun workspaceDir(): File = workspaceDir
|
||||
|
||||
/** The built page, unpacked once per version. */
|
||||
private val pageDir = File(context.filesDir, "page")
|
||||
|
||||
private val tokenFile = File(home, "remote/launch.token")
|
||||
private val portFile = File(home, "remote/serve.port")
|
||||
private val pidFile = File(home, "remote/serve.pid")
|
||||
|
||||
private val lock = Any()
|
||||
|
||||
@Volatile
|
||||
private var process: Process? = null
|
||||
|
||||
/** The port the shell's browser HTTP server listens on, set by the
|
||||
* activity before `start()` is called. Zero means no server is up, and
|
||||
* the kernel leaves its browser tools suspended (the correct degradation
|
||||
* on a phone with no Chromium). */
|
||||
@Volatile
|
||||
var browserShellPort: Int = 0
|
||||
|
||||
/** The bearer credential for that server, set beside the port. Sent on
|
||||
* every browser call so the server can refuse anything that did not come
|
||||
* from the kernel this shell started — loopback on a phone is reachable
|
||||
* by every other app. Empty when there is no server. */
|
||||
@Volatile
|
||||
var browserShellToken: String = ""
|
||||
|
||||
/** What the WebView should load, once the kernel has bound a port. */
|
||||
@Volatile
|
||||
var origin: String? = null
|
||||
private set
|
||||
|
||||
/**
|
||||
* The launch credential, as the URL fragment the page trades for a cookie.
|
||||
* Kept in memory only: the file on disk belongs to the kernel's own
|
||||
* bookkeeping, and a phone's app data is not a place to leave a second copy.
|
||||
*/
|
||||
@Volatile
|
||||
private var token: String? = null
|
||||
|
||||
/**
|
||||
* The URL the WebView loads first. It carries the credential in the
|
||||
* fragment, which the page exchanges for an HttpOnly cookie before it makes
|
||||
* any request — the same path a shared `reasonix web` link takes, so the
|
||||
* token never enters a request line, a history entry or a log.
|
||||
*/
|
||||
fun entryUrl(): String? {
|
||||
val origin = origin ?: return null
|
||||
val token = token ?: return null
|
||||
return "$origin/#token=$token"
|
||||
}
|
||||
|
||||
/** The plain origin, for reloads once the cookie is set. */
|
||||
fun plainUrl(): String? = origin
|
||||
|
||||
fun isRunning(): Boolean = process?.isAlive == true
|
||||
|
||||
/**
|
||||
* Bring the kernel up, or answer the URL of the one already up.
|
||||
*
|
||||
* Starting is idempotent by intent: the activity and the service both call
|
||||
* this, and whichever arrives second joins the running kernel instead of
|
||||
* racing it for the port.
|
||||
*/
|
||||
suspend fun start(): Result<String> = withContext(Dispatchers.IO) {
|
||||
synchronized(lock) {
|
||||
origin?.let { return@withContext Result.success(it) }
|
||||
|
||||
runCatching {
|
||||
val binary = kernelBinary()
|
||||
publishPage()
|
||||
prepareHome()
|
||||
val launchToken = mintToken()
|
||||
token = launchToken
|
||||
|
||||
val child = launch(binary, launchToken)
|
||||
|
||||
// The port is the kernel's answer, not this shell's guess. It
|
||||
// writes the file after binding, so waiting on the file is
|
||||
// waiting on the bind.
|
||||
val bound = awaitPort(child)
|
||||
val url = "http://$bound"
|
||||
|
||||
// A kernel that bound a port and then died would look healthy
|
||||
// here and broken in the WebView; the two facts are checked
|
||||
// together, and only a kernel that is answering is adopted.
|
||||
awaitServing(child, url)
|
||||
origin = url
|
||||
process = child
|
||||
url
|
||||
}.onFailure { origin = null; process = null }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the kernel.
|
||||
*
|
||||
* The graceful signal first: the kernel drains its listeners and closes the
|
||||
* runtimes it opened, which is what lets a later launch adopt the sessions
|
||||
* rather than find them locked. `destroyForcibly` is the second attempt, and
|
||||
* only after a wait — a phone that is being shut down should not keep a
|
||||
* half-killed agent.
|
||||
*/
|
||||
fun stop() {
|
||||
synchronized(lock) {
|
||||
val p = process ?: return
|
||||
process = null
|
||||
origin = null
|
||||
token = null
|
||||
runCatching {
|
||||
p.destroy()
|
||||
if (!p.waitFor(3, java.util.concurrent.TimeUnit.SECONDS)) {
|
||||
p.destroyForcibly()
|
||||
}
|
||||
}
|
||||
portFile.delete()
|
||||
pidFile.delete()
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the two inputs that ship inside the APK -------------------------
|
||||
|
||||
/**
|
||||
* The kernel binary.
|
||||
*
|
||||
* It is packaged as a native library, so Android has already extracted it to
|
||||
* a directory it allows execution from and given it the executable bit —
|
||||
* which is the one thing a file under app storage does not get for free.
|
||||
*/
|
||||
private fun kernelBinary(): File {
|
||||
val dir = File(context.applicationInfo.nativeLibraryDir)
|
||||
val binary = File(dir, "libreasonix.so")
|
||||
if (!binary.exists()) {
|
||||
throw IllegalStateException(
|
||||
"内核未随 APK 安装：${binary.path}\n这是一次构建问题，请重新安装完整 APK。",
|
||||
)
|
||||
}
|
||||
return binary
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the bundled page where the kernel can serve it.
|
||||
*
|
||||
* It goes to app storage rather than being served out of the APK because the
|
||||
* kernel's `--page` is a directory on a real filesystem; there is no
|
||||
* archive-aware variant, and adding one would be a change to the kernel for
|
||||
* this shell's convenience alone.
|
||||
*
|
||||
* Unpacking is versioned: the marker holds the app's version name, so an
|
||||
* update replaces the page and a relaunch does not. A partially written
|
||||
* directory — the app killed mid-copy — is caught by the same marker never
|
||||
* having been written, so the next launch rewrites it.
|
||||
*/
|
||||
private fun publishPage() {
|
||||
val marker = File(pageDir, ".unpacked")
|
||||
val stamp = "${BuildConfig.VERSION_NAME}:${BuildConfig.VERSION_CODE}"
|
||||
if (marker.isFile && marker.readText().trim() == stamp) return
|
||||
|
||||
pageDir.deleteRecursively()
|
||||
pageDir.mkdirs()
|
||||
copyAssetTree(context.assets, "page", pageDir)
|
||||
|
||||
if (!File(pageDir, "index.html").isFile) {
|
||||
throw IllegalStateException("随 APK 分发的页面不完整：缺少 index.html")
|
||||
}
|
||||
marker.writeText(stamp)
|
||||
}
|
||||
|
||||
/**
|
||||
* A directory tree out of assets.
|
||||
*
|
||||
* `AssetManager.list` on a file answers with an empty array, so a "file" is
|
||||
* told from a directory by trying to open it — which is the only signal the
|
||||
* API offers and the reason this is not a one-liner.
|
||||
*/
|
||||
private fun copyAssetTree(assets: AssetManager, from: String, to: File) {
|
||||
val children = assets.list(from).orEmpty()
|
||||
if (children.isEmpty()) {
|
||||
// A leaf: open it as a stream. A directory that happens to be empty
|
||||
// is indistinguishable and harmless to skip.
|
||||
runCatching {
|
||||
assets.open(from).use { input ->
|
||||
FileOutputStream(to).use { output -> input.copyTo(output) }
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
to.mkdirs()
|
||||
for (child in children) {
|
||||
copyAssetTree(assets, "$from/$child", File(to, child))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare the kernel's home.
|
||||
*
|
||||
* `REASONIX_HOME` is what every path in the kernel hangs off — config,
|
||||
* sessions, the remote state directory its token file lands in — so pointing
|
||||
* it at app storage is what keeps a phone's Reasonix separate from anything
|
||||
* else and away from global storage entirely.
|
||||
*/
|
||||
private fun prepareHome() {
|
||||
File(home, "remote").mkdirs()
|
||||
// The workspace directory not existing yet is what marks the first run
|
||||
// after this build split it from the kernel's home. That is the one
|
||||
// moment a migration is due, and doing it here means it happens before
|
||||
// the kernel is asked for anything.
|
||||
val firstSplit = !workspaceDir.exists()
|
||||
workspaceDir.mkdirs()
|
||||
if (firstSplit) {
|
||||
adoptSessionsFromHome()
|
||||
}
|
||||
// Every launch, not only the first. The rewrite is cheap and
|
||||
// idempotent, and an entry that survived the first pass would otherwise
|
||||
// survive forever: `firstSplit` is spent the moment the directory
|
||||
// exists, so a build that cleaned the list once and missed leaves the
|
||||
// home on the sidebar for good. Running it unconditionally costs one
|
||||
// small read and makes the whole thing self-correcting.
|
||||
forgetHomeFromWorkspaceList()
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the kernel's own home from the sidebar's remembered projects.
|
||||
*
|
||||
* The list is written whenever a pane opens, and until this build the pane
|
||||
* opened on the home itself — so an upgraded install carries a "project"
|
||||
* that is really the kernel's own directory. Leaving it there would put the
|
||||
* person back where they started: one tap on it and the workbench is full
|
||||
* of `config.toml`, `cache/` and `projects/` again, which is the complaint
|
||||
* this whole change answers. Only that one path is removed; every folder
|
||||
* someone added on purpose stays.
|
||||
*
|
||||
* Parsed rather than pattern-matched, because the file's two shapes — a bare
|
||||
* array of paths and an object with `paths` and `launch` — both have to keep
|
||||
* whatever else they hold.
|
||||
*/
|
||||
private fun forgetHomeFromWorkspaceList() {
|
||||
val file = File(home, "serve-workspaces.json")
|
||||
if (!file.isFile) return
|
||||
val text = runCatching { file.readText() }.getOrNull() ?: return
|
||||
val wrong = runCatching { homeNames() }.getOrNull() ?: return
|
||||
val rewritten = runCatching { withoutHome(text, wrong) }.getOrNull() ?: return
|
||||
if (!rewritten.dropped) return
|
||||
runCatching { file.writeText(rewritten.text) }
|
||||
.onSuccess { Log.i(TAG, "dropped the kernel home from the project list") }
|
||||
.onFailure { Log.w(TAG, "could not rewrite the project list: ${it.message}") }
|
||||
}
|
||||
|
||||
/**
|
||||
* Every name of the kernel's home on this device — not just the one this
|
||||
* process happens to write.
|
||||
*
|
||||
* Android shows one directory under two names: `/data/data/<pkg>` and
|
||||
* `/data/user/0/<pkg>` are two mounts of the same tree, the same inode
|
||||
* either way, and which name a path carries depends on how it was written
|
||||
* rather than on where it points. The list is written by the kernel, whose
|
||||
* working directory this shell set from a path it built itself; a lookup
|
||||
* that compared against only one spelling would silently keep the entry
|
||||
* this exists to remove. Both names are compared, so either spelling is
|
||||
* recognised.
|
||||
*/
|
||||
private fun homeNames(): Set<String> {
|
||||
val path = home.absolutePath
|
||||
val names = mutableSetOf(path)
|
||||
val user = "/data/user/0/"
|
||||
val data = "/data/data/"
|
||||
when {
|
||||
path.startsWith(user) -> names.add(data + path.removePrefix(user))
|
||||
path.startsWith(data) -> names.add(user + path.removePrefix(data))
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
/** What a rewrite produced, and whether it actually removed anything. */
|
||||
private class Rewrite(val text: String, val dropped: Boolean)
|
||||
|
||||
/**
|
||||
* The list without the home, in whichever shape it arrived, and whether
|
||||
* anything was actually dropped from it.
|
||||
*
|
||||
* The flag matters because `JSONObject` re-serialises: it escapes every
|
||||
* `/` and does not preserve key order, so a round trip never reproduces
|
||||
* the original bytes. Deciding to write on "the text differs" alone would
|
||||
* therefore rewrite the kernel's file on every single launch, log a drop
|
||||
* that never happened, and race the kernel for a file it also writes.
|
||||
*/
|
||||
private fun withoutHome(text: String, wrong: Set<String>): Rewrite {
|
||||
if (text.trimStart().startsWith("[")) {
|
||||
val paths = JSONArray(text)
|
||||
val kept = JSONArray()
|
||||
var dropped = false
|
||||
for (i in 0 until paths.length()) {
|
||||
val path = paths.optString(i)
|
||||
if (path in wrong) {
|
||||
dropped = true
|
||||
continue
|
||||
}
|
||||
kept.put(path)
|
||||
}
|
||||
return if (dropped) Rewrite(kept.toString(), true) else Rewrite(text, false)
|
||||
}
|
||||
val list = JSONObject(text)
|
||||
val kept = JSONArray()
|
||||
var dropped = false
|
||||
val paths = list.optJSONArray("paths")
|
||||
if (paths != null) {
|
||||
for (i in 0 until paths.length()) {
|
||||
val path = paths.optString(i)
|
||||
if (path in wrong) {
|
||||
dropped = true
|
||||
continue
|
||||
}
|
||||
kept.put(path)
|
||||
}
|
||||
if (dropped) list.put("paths", kept)
|
||||
}
|
||||
if (list.optString("launch") in wrong) {
|
||||
list.remove("launch")
|
||||
dropped = true
|
||||
}
|
||||
return if (dropped) Rewrite(list.toString(), true) else Rewrite(text, false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Move the conversations that were filed under the kernel's own home.
|
||||
*
|
||||
* The transcript store is keyed by the workspace root — one directory per
|
||||
* project, under `<state>/projects/<slug>/sessions` — and until this build
|
||||
* the root was the home itself. Splitting the two therefore moves every
|
||||
* existing conversation to a slug nothing looks under, which reads as "my
|
||||
* history is gone" rather than as a reorganisation. So the old directory is
|
||||
* renamed onto the new slug once.
|
||||
*
|
||||
* Renaming the project directory rather than just `sessions/` carries the
|
||||
* rest of it too: `memory/` is filed the same way, and a project's memory
|
||||
* lost while its transcripts were kept would be a stranger failure than
|
||||
* either alone.
|
||||
*
|
||||
* The slug is the absolute path with its separators flattened, which is
|
||||
* what `config.WorkspaceSlug` does on the kernel's side. Only the home's own
|
||||
* slug is touched: a slug for some other folder is left exactly where it is,
|
||||
* because those are projects someone added on purpose.
|
||||
*/
|
||||
private fun adoptSessionsFromHome() {
|
||||
val projects = File(home, "projects")
|
||||
if (!projects.isDirectory) return
|
||||
val from = projects.resolve(slugOf(home.absolutePath))
|
||||
if (!from.isDirectory) return
|
||||
val to = projects.resolve(slugOf(workspaceDir.absolutePath))
|
||||
// A target already there means a previous run half-finished, or the
|
||||
// person opened this folder before. Either way its contents win:
|
||||
// merging two transcript directories is not a thing this can do safely,
|
||||
// and the old one is still on disk to be looked at.
|
||||
if (to.exists()) {
|
||||
Log.i(TAG, "${to.name} already exists; leaving ${from.name} in place")
|
||||
return
|
||||
}
|
||||
if (!from.renameTo(to)) {
|
||||
Log.w(TAG, "could not move ${from.name} to ${to.name}; history stays under the old slug")
|
||||
} else {
|
||||
Log.i(TAG, "moved conversations from ${from.name} to ${to.name}")
|
||||
}
|
||||
}
|
||||
|
||||
/** The kernel's `config.WorkspaceSlug`, for the two paths that matter here. */
|
||||
private fun slugOf(path: String): String = path.replace('/', '-').replace('\\', '-').replace(':', '-')
|
||||
|
||||
/**
|
||||
* The launch credential.
|
||||
*
|
||||
* Minted here rather than read from configuration: a credential a person can
|
||||
* set is one anything that can read their files can present. 32 bytes, hex,
|
||||
* the same shape the desktop host mints.
|
||||
*/
|
||||
private fun mintToken(): String {
|
||||
val bytes = ByteArray(32)
|
||||
SecureRandom().nextBytes(bytes)
|
||||
val token = bytes.joinToString("") { "%02x".format(it) }
|
||||
tokenFile.parentFile?.mkdirs()
|
||||
tokenFile.writeText("$token\n")
|
||||
// 0600 by intent, and the app's own private directory means it is
|
||||
// already unreachable to every other app on the device.
|
||||
runCatching {
|
||||
tokenFile.setReadable(false, false)
|
||||
tokenFile.setReadable(true, true)
|
||||
tokenFile.setWritable(false, false)
|
||||
tokenFile.setWritable(true, true)
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the child.
|
||||
*
|
||||
* `--addr 127.0.0.1:0` lets the kernel pick a free port; `--port-file`
|
||||
* makes it announce which. `--auth token` is the mode a phone needs: the
|
||||
* page authenticates, and a state-changing request without the credential
|
||||
* is refused. `--no-open` is belt and braces — a serve never opens a
|
||||
* browser, and a phone has no browser to open.
|
||||
*/
|
||||
private fun launch(binary: File, token: String): Process {
|
||||
portFile.delete()
|
||||
pidFile.delete()
|
||||
// The folder the agent works in, resolved before the command line is
|
||||
// built so it can be named on it. See the note under the environment
|
||||
// block for why naming it is not the same as setting the directory.
|
||||
val workspace = launchWorkspaceDir()
|
||||
val command = listOf(
|
||||
binary.absolutePath,
|
||||
"serve",
|
||||
"--addr", "127.0.0.1:0",
|
||||
"--auth", "token",
|
||||
"--token-file", tokenFile.absolutePath,
|
||||
"--port-file", portFile.absolutePath,
|
||||
"--pid-file", pidFile.absolutePath,
|
||||
"--no-open",
|
||||
"-page", pageDir.absolutePath,
|
||||
// The project root, said outright. Without this the kernel derives
|
||||
// it from its working directory, and on this phone the two did not
|
||||
// agree: the sidebar named the shared folder, `write_roots` confined
|
||||
// the file tools to the app's own directory, and every write into the
|
||||
// folder the person was looking at came back
|
||||
// `workspace.write_outside_scope`. A flag is a fact the kernel was
|
||||
// told rather than one it inferred from a mount it sees differently.
|
||||
"--dir", workspace.absolutePath,
|
||||
)
|
||||
val builder = ProcessBuilder(command)
|
||||
builder.environment().apply {
|
||||
put("REASONIX_HOME", home.absolutePath)
|
||||
put("HOME", home.absolutePath)
|
||||
// The kernel's own language, for its tool failures, approval
|
||||
// prompts and error sentences. Without this it detects from the
|
||||
// POSIX locale variables, and Android sets none of them for an
|
||||
// app: the kernel then answers in English while the page in front
|
||||
// of the person is Chinese, and every sentence added for a Chinese
|
||||
// reader was dead text. The tag, not the bare language, so a
|
||||
// Traditional Chinese device resolves to zh-TW rather than zh.
|
||||
// An explicit `language` in the kernel's config.toml still wins:
|
||||
// DetectLanguage applies the configured value over this.
|
||||
val languageTag = Locale.getDefault().toLanguageTag()
|
||||
if (languageTag.isNotBlank() && languageTag != "und") {
|
||||
put("REASONIX_LANG", languageTag)
|
||||
}
|
||||
// Android has no /tmp. Everything that wants a scratch directory —
|
||||
// the shell tool, tool output spooling, sqlite journals — reads this.
|
||||
put("TMPDIR", context.cacheDir.absolutePath)
|
||||
// The phone's own DNS servers. A CGO-free Go binary cannot read them
|
||||
// for itself — Android keeps them in `netd`, and Go looks for a
|
||||
// `/etc/resolv.conf` that does not exist — so without this the kernel
|
||||
// resolves nothing and every provider probe reports "连接失败" on a
|
||||
// device whose browser reaches the same address fine.
|
||||
val dns = Dns.servers(context)
|
||||
if (dns.isNotBlank()) {
|
||||
put("REASONIX_DNS", dns)
|
||||
Log.i(TAG, "kernel DNS: $dns (${Dns.describe(context)})")
|
||||
if (Dns.looksLikeDot(dns)) {
|
||||
// Private DNS (DNS-over-TLS) is on, and it hands back a
|
||||
// hostname this kernel cannot resolve by definition. Saying
|
||||
// so here is what turns "连不上" into something actionable.
|
||||
Log.w(TAG, "kernel DNS looks like a private-DNS hostname; turn private DNS off or set it to automatic")
|
||||
}
|
||||
} else {
|
||||
Log.w(TAG, "kernel DNS: none reported by the system")
|
||||
}
|
||||
// The shell's browser server, when one is listening. The kernel
|
||||
// reads this to bind its browser tools to the shell's WebView
|
||||
// instead of looking for a Chromium binary that does not exist on
|
||||
// a phone. When absent (zero), the variable is not set and the
|
||||
// kernel keeps the browser tools suspended — the correct
|
||||
// degradation for a phone with no engine to drive.
|
||||
val shellPort = browserShellPort
|
||||
if (shellPort > 0) {
|
||||
put("REASONIX_SHELL_BROWSER_PORT", shellPort.toString())
|
||||
// The credential the server checks. Set only alongside the
|
||||
// port: a token with nothing listening would be a variable the
|
||||
// kernel carries for no reason.
|
||||
val shellToken = browserShellToken
|
||||
if (shellToken.isNotBlank()) {
|
||||
put("REASONIX_SHELL_BROWSER_TOKEN", shellToken)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The agent's folder, not the kernel's. `REASONIX_HOME` above is still
|
||||
// where the config, the sessions and the transcripts live; this is what
|
||||
// the workspace root resolves to, because the kernel reads the working
|
||||
// directory for it.
|
||||
//
|
||||
// The folder the person last opened, when it is still usable — the same
|
||||
// answer the desktop host gives (`launchWorkspace`), and the reason a
|
||||
// phone session used to work in the app's own `files/workspace` while
|
||||
// the sidebar said it was in the shared folder: the launch entry in
|
||||
// `serve-workspaces.json` was written and then ignored, and `write_roots`
|
||||
// is derived from the working directory, so the file tools were confined
|
||||
// to the app's private folder and every write into the shared one was
|
||||
// refused. `workspaceDir` is still the answer for a first run and for a
|
||||
// folder since deleted, but the choice is never silent — see
|
||||
// [launchWorkspaceDir]. `workspace` was resolved above so it could also
|
||||
// be named on the command line.
|
||||
builder.directory(workspace)
|
||||
// `PWD` is what the kernel's own `os.Getwd` prefers: Go uses it whenever
|
||||
// it names the same directory as `.`, which is exactly this case, and it
|
||||
// is the only way to make the kernel resolve its root to the string this
|
||||
// shell chose. Without it `os.Getwd` falls back to the kernel's view of
|
||||
// the mount, and on Android `/storage/emulated/0/...` and the path the
|
||||
// FUSE layer reports for it need not be spelled the same way. The kernel
|
||||
// derives `write_roots` from that root, so the two spellings have to be
|
||||
// the same one.
|
||||
builder.environment().put("PWD", workspace.absolutePath)
|
||||
builder.redirectErrorStream(true)
|
||||
val process = builder.start()
|
||||
|
||||
// The kernel's own log is the only place a startup failure says why, and
|
||||
// a phone has no terminal to read. It is drained on a thread and kept
|
||||
// ring-buffered so the About screen can show the last of it.
|
||||
Thread({ drain(process) }, "kernel-log").apply { isDaemon = true }.start()
|
||||
return process
|
||||
}
|
||||
|
||||
/**
|
||||
* The folder the person last opened, or null when there is not one.
|
||||
*
|
||||
* `serve-workspaces.json` is the kernel's own record of the projects the
|
||||
* sidebar lists, and its `launch` entry is which of them a session starts
|
||||
* in. Both shapes the file has are read — a bare array of paths, and an
|
||||
* object with `paths` and `launch` — because both are written in the wild
|
||||
* and a reader that only knew one would silently fall back.
|
||||
*
|
||||
* The entry has to be one of the listed paths, which is the same rule the
|
||||
* kernel applies when it reads this file (`workspacelist.Read` blanks a
|
||||
* `launch` that is not in `paths`). It matters more here than there: the
|
||||
* working directory decides `write_roots`, so whatever this answers is
|
||||
* where the agent's file tools may write — and on a phone that permission
|
||||
* reaches all of shared storage. A file with a `launch` outside its own
|
||||
* list is one this shell did not write, and it gets the listed answer.
|
||||
*
|
||||
* Null for anything unusable: no file, unreadable JSON, a launch that is
|
||||
* not among the listed folders, an entry that has since been deleted, or a
|
||||
* path that is now a file. Every one of those means the same thing to the
|
||||
* caller — start where this shell always started.
|
||||
*/
|
||||
private fun rememberedLaunchDir(): File? {
|
||||
val file = File(home, "serve-workspaces.json")
|
||||
if (!file.isFile) return null
|
||||
val text = runCatching { file.readText() }.getOrNull() ?: return null
|
||||
val launch = runCatching {
|
||||
// A bare array is the file's other shape — the one older kernels
|
||||
// wrote, and therefore the one a person upgrading is most likely to
|
||||
// have. Without this branch `JSONObject(text)` throws on it and the
|
||||
// whole function falls back, which is the same mismatch (the
|
||||
// sidebar naming one folder, the session working in another) this
|
||||
// exists to fix. `withoutHome` below reads both shapes the same way.
|
||||
if (text.trimStart().startsWith("[")) {
|
||||
val arr = JSONArray(text)
|
||||
if (arr.length() == 0) return@runCatching null
|
||||
arr.optString(0)
|
||||
} else {
|
||||
val json = JSONObject(text)
|
||||
val paths = json.optJSONArray("paths")
|
||||
?.let { arr -> (0 until arr.length()).mapNotNull { arr.optString(it).takeIf(String::isNotBlank) } }
|
||||
.orEmpty()
|
||||
if (paths.isEmpty()) return@runCatching null
|
||||
json.optString("launch").takeIf { it.isNotBlank() && it in paths } ?: paths.first()
|
||||
}
|
||||
}.getOrNull()?.takeIf { it.isNotBlank() } ?: return null
|
||||
val dir = runCatching { File(launch) }.getOrNull() ?: return null
|
||||
if (!dir.isDirectory) {
|
||||
// A launch entry this process cannot stat is exactly the failure the
|
||||
// caller exists to avoid, and falling back silently is what made it
|
||||
// take three releases to notice: the session opened in the app's own
|
||||
// folder while the sidebar named the shared one. A phone has no
|
||||
// terminal, so the log is the only place the reason can be said —
|
||||
// and `exists`/`readable` tell apart "deleted" from "not permitted",
|
||||
// which need different answers from the person.
|
||||
Log.w(
|
||||
TAG,
|
||||
"launch entry $launch is not a usable directory " +
|
||||
"(exists=${dir.exists()}, readable=${dir.canRead()})",
|
||||
)
|
||||
return null
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
/**
|
||||
* The folder the kernel should start in.
|
||||
*
|
||||
* Either the launch entry — the folder the person last opened, which is
|
||||
* also the one the sidebar shows — or the app's own folder when there is
|
||||
* none. Whichever it is, it is logged, because the two disagreeing is the
|
||||
* bug this exists to fix and a silent choice is what let it survive.
|
||||
*/
|
||||
private fun launchWorkspaceDir(): File {
|
||||
val remembered = rememberedLaunchDir()
|
||||
if (remembered != null) {
|
||||
Log.i(TAG, "kernel workspace: ${remembered.absolutePath} (the remembered launch project)")
|
||||
return remembered
|
||||
}
|
||||
Log.i(TAG, "kernel workspace: ${workspaceDir.absolutePath} (no remembered launch project)")
|
||||
return workspaceDir
|
||||
}
|
||||
|
||||
private val logLines = ArrayDeque<String>()
|
||||
|
||||
private fun drain(process: Process) {
|
||||
runCatching {
|
||||
process.inputStream.bufferedReader().useLines { lines ->
|
||||
lines.forEach { line ->
|
||||
Log.i(TAG, line)
|
||||
synchronized(logLines) {
|
||||
logLines.addLast(line)
|
||||
while (logLines.size > LOG_KEEP) logLines.removeFirst()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The kernel's tail, newest last, for a bug report or the About screen. */
|
||||
fun recentLog(): String = synchronized(logLines) { logLines.joinToString("\n") }
|
||||
|
||||
private fun awaitPort(process: Process): String {
|
||||
val deadline = System.currentTimeMillis() + START_TIMEOUT_MS
|
||||
while (System.currentTimeMillis() < deadline) {
|
||||
if (!process.isAlive) {
|
||||
throw IllegalStateException("内核启动即退出。\n${recentLog().takeLast(2000)}")
|
||||
}
|
||||
val text = if (portFile.isFile) runCatching { portFile.readText().trim() }.getOrNull() else null
|
||||
// The file is written by the kernel on the way to serving, and the
|
||||
// kernel rewrites it with the real port when it was given :0.
|
||||
if (!text.isNullOrBlank() && text.contains(":")) return text
|
||||
Thread.sleep(POLL_MS)
|
||||
}
|
||||
throw IllegalStateException("内核在 ${START_TIMEOUT_MS / 1000}s 内没有报告监听地址。\n${recentLog().takeLast(2000)}")
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait until the bound port answers.
|
||||
*
|
||||
* A port that is open but not yet serving is the normal first second of a
|
||||
* kernel's life; a WebView pointed at it would show a connection error and
|
||||
* cache the failure, so the wait belongs here rather than in the page.
|
||||
*/
|
||||
private fun awaitServing(process: Process, origin: String) {
|
||||
val deadline = System.currentTimeMillis() + START_TIMEOUT_MS
|
||||
while (System.currentTimeMillis() < deadline) {
|
||||
if (!process.isAlive) {
|
||||
throw IllegalStateException("内核在开始服务前退出。\n${recentLog().takeLast(2000)}")
|
||||
}
|
||||
if (probe("$origin/")) return
|
||||
Thread.sleep(POLL_MS)
|
||||
}
|
||||
throw IllegalStateException("内核已监听 $origin，但页面没有响应。\n${recentLog().takeLast(2000)}")
|
||||
}
|
||||
|
||||
private fun probe(url: String): Boolean = runCatching {
|
||||
(URL(url).openConnection() as HttpURLConnection).run {
|
||||
connectTimeout = 1000
|
||||
readTimeout = 1000
|
||||
requestMethod = "GET"
|
||||
try {
|
||||
responseCode in 200..399
|
||||
} finally {
|
||||
disconnect()
|
||||
}
|
||||
}
|
||||
}.getOrDefault(false)
|
||||
|
||||
private companion object {
|
||||
const val TAG = "ReasonixKernel"
|
||||
const val LOG_KEEP = 400
|
||||
const val POLL_MS = 120L
|
||||
const val START_TIMEOUT_MS = 30_000L
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package io.reasonix.mobile.app
|
||||
|
||||
import android.Manifest
|
||||
import android.app.Notification
|
||||
import android.app.PendingIntent
|
||||
import android.app.Service
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.content.pm.ServiceInfo
|
||||
import android.os.Build
|
||||
import android.os.IBinder
|
||||
import androidx.core.app.NotificationCompat
|
||||
import androidx.core.app.NotificationManagerCompat
|
||||
import androidx.core.app.ServiceCompat
|
||||
import androidx.core.content.ContextCompat
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.launch
|
||||
|
||||
/**
|
||||
* Keeps the kernel alive while the app is not on screen.
|
||||
*
|
||||
* The reason this exists at all: an agent turn that takes minutes is a normal
|
||||
* turn, and Android reclaims a backgrounded app's child process within seconds
|
||||
* of the screen going off. Without the service, pocketing the phone would kill
|
||||
* a running task — the one failure mode that makes a phone a bad client for
|
||||
* something that works on its own.
|
||||
*
|
||||
* A foreground service needs a notification the person can see, which is also
|
||||
* the honest thing to show: the kernel is running and what it is doing lives on
|
||||
* another machine.
|
||||
*/
|
||||
class KernelService : Service() {
|
||||
|
||||
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main)
|
||||
|
||||
override fun onBind(intent: Intent?): IBinder? = null
|
||||
|
||||
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
||||
// Promoted before `start()`: a service that starts work first and
|
||||
// notifies after has a window in which the system may kill it, and the
|
||||
// work it would have been protecting is exactly what is being started.
|
||||
startForegroundNotification()
|
||||
|
||||
val kernel = (application as App).kernel
|
||||
scope.launch {
|
||||
kernel.start().onFailure { failure ->
|
||||
// A kernel that will not start is a fact the person needs, and a
|
||||
// notification that only ever said "running" would be a lie.
|
||||
notify(SERVICE_NOTICE, getString(R.string.notify_failed_title), failure.message.orEmpty())
|
||||
stopSelf()
|
||||
}
|
||||
}
|
||||
// Sticky, so a kernel killed under memory pressure comes back when there
|
||||
// is room — which is what a person who left a task running expects.
|
||||
return START_STICKY
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
scope.cancel()
|
||||
super.onDestroy()
|
||||
}
|
||||
|
||||
private fun startForegroundNotification() {
|
||||
val notification = buildNotification(
|
||||
getString(R.string.notify_running_title),
|
||||
getString(R.string.notify_running_body),
|
||||
)
|
||||
// `dataSync` matches the manifest's declaration. On 34+ the type is
|
||||
// mandatory and a mismatch throws, so it is named in both places.
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {
|
||||
ServiceCompat.startForeground(
|
||||
this,
|
||||
SERVICE_NOTICE,
|
||||
notification,
|
||||
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC,
|
||||
)
|
||||
} else {
|
||||
startForeground(SERVICE_NOTICE, notification)
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildNotification(title: String, body: String): Notification {
|
||||
val open = PendingIntent.getActivity(
|
||||
this,
|
||||
0,
|
||||
Intent(this, MainActivity::class.java).apply {
|
||||
flags = Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP
|
||||
},
|
||||
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
|
||||
)
|
||||
return NotificationCompat.Builder(this, App.CHANNEL_KERNEL)
|
||||
.setSmallIcon(R.drawable.ic_notification)
|
||||
.setContentTitle(title)
|
||||
.setContentText(body)
|
||||
.setContentIntent(open)
|
||||
// The notification is a receipt, not an alarm; swiping it away
|
||||
// leaves the kernel running, which is what the person meant by
|
||||
// swiping it.
|
||||
.setOngoing(false)
|
||||
.setSilent(true)
|
||||
.setPriority(NotificationCompat.PRIORITY_MIN)
|
||||
.build()
|
||||
}
|
||||
|
||||
private fun notify(id: Int, title: String, body: String) {
|
||||
// The notification permission is granted at runtime from Android 13 and
|
||||
// the person can refuse it. Without this check the call throws
|
||||
// SecurityException and takes the service down with it — the opposite of
|
||||
// what a notification that could not be shown should do. Below 13 the
|
||||
// permission does not exist and posting always succeeds.
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU &&
|
||||
ContextCompat.checkSelfPermission(this, Manifest.permission.POST_NOTIFICATIONS) !=
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
) {
|
||||
return
|
||||
}
|
||||
runCatching {
|
||||
NotificationManagerCompat.from(this).notify(id, buildNotification(title, body))
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val SERVICE_NOTICE = 1
|
||||
|
||||
/** Idempotent: called on every start of the activity, and by the boot
|
||||
* path, and safe to repeat. */
|
||||
fun start(context: Context) {
|
||||
ContextCompat.startForegroundService(context, Intent(context, KernelService::class.java))
|
||||
}
|
||||
|
||||
fun stop(context: Context) {
|
||||
context.stopService(Intent(context, KernelService::class.java))
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,744 @@
|
||||
package io.reasonix.mobile.app
|
||||
|
||||
import android.content.ClipData
|
||||
import android.content.ClipboardManager
|
||||
import android.content.ContentValues
|
||||
import android.content.Context
|
||||
import android.content.res.Configuration
|
||||
import android.net.Uri
|
||||
import android.os.Build
|
||||
import android.os.Environment
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.provider.MediaStore
|
||||
import android.webkit.JavascriptInterface
|
||||
import android.webkit.MimeTypeMap
|
||||
import java.io.File
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
|
||||
/**
|
||||
* What the page can ask the phone for.
|
||||
*
|
||||
* This is the whole surface, and it is deliberately tiny. Android's
|
||||
* `@JavascriptInterface` is a bridge into the app's process from a WebView whose
|
||||
* content is a web page: every method here is reachable by anything that can run
|
||||
* script in that page, so each one is written as a verb with no address
|
||||
* parameter it does not need, and none of them can name a path outside the
|
||||
* kernel's own home.
|
||||
*
|
||||
* It is deliberately *not* the Electron preload bridge. That one exposes window
|
||||
* management, a filesystem walker, an editor launcher and a browser view host —
|
||||
* all of which are meaningless on a phone, and the two that were not (open a URL,
|
||||
* save a file) already have a way to happen here: `WebViewClient` routes external
|
||||
* links to the browser, and the page's own download reaches the WebView's
|
||||
* `DownloadListener`.
|
||||
*
|
||||
* The one thing a phone must add is `importIntoHome`: the kernel runs on this
|
||||
* device and can only read paths under its own home, so a private key the person
|
||||
* has in their Documents folder has to be copied across. Nothing else the page
|
||||
* wants is out of its reach.
|
||||
*/
|
||||
class NativeBridge(
|
||||
private val context: Context,
|
||||
private val kernel: Kernel,
|
||||
private val onOpen: (String) -> Unit,
|
||||
) {
|
||||
|
||||
/**
|
||||
* Whether the document calling may use this bridge.
|
||||
*
|
||||
* `addJavascriptInterface` makes the object reachable from whatever page the
|
||||
* WebView holds, so a link the model produced — or a cleartext page on the
|
||||
* LAN — would otherwise get `importIntoHome`, `pickFolder`, `hostCall` and
|
||||
* the clipboard. The activity answers by comparing the main frame's origin
|
||||
* against the kernel's and the one paired computer the person confirmed.
|
||||
*
|
||||
* Anything that reaches the device checks this first. The five methods that
|
||||
* do not — version, systemTheme, safeArea, setLayerOpen, setRunning — report
|
||||
* back what the page already knows about its own window and grant nothing.
|
||||
*/
|
||||
var trusted: () -> Boolean = { false }
|
||||
|
||||
/** The version the page reads to decide whether this bridge is the one it
|
||||
* knows. Bumped when a method's contract changes, not when one is added. */
|
||||
@JavascriptInterface
|
||||
fun version(): String = BRIDGE_VERSION
|
||||
|
||||
/**
|
||||
* Copy a file the person picks into the kernel's home, answering with its
|
||||
* path there — or "" when they dismissed the picker.
|
||||
*
|
||||
* `kind` names the directory inside the home, because that is a fact about
|
||||
* the kernel's layout and not about the file: an SSH private key belongs in
|
||||
* `.ssh`, and a dotenv the kernel reads for passphrases belongs at the root
|
||||
* of the home where `config.Load` looks for it.
|
||||
*
|
||||
* The path that comes back is *inside the home*, never the document
|
||||
* provider's URI. That is the point: the kernel is a child process with no
|
||||
* access to the provider, so a `content://` would be a path it cannot open.
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun importIntoHome(kind: String, name: String): String {
|
||||
if (!trusted()) return ""
|
||||
val dir = when (kind) {
|
||||
"identity" -> File(kernel.homeDir(), ".ssh")
|
||||
"env" -> kernel.homeDir()
|
||||
else -> return ""
|
||||
}
|
||||
val safeName = sanitise(name)
|
||||
if (safeName.isEmpty()) return ""
|
||||
|
||||
val picked = pickFile() ?: return ""
|
||||
return try {
|
||||
dir.mkdirs()
|
||||
val target = File(dir, safeName)
|
||||
context.contentResolver.openInputStream(picked)?.use { input ->
|
||||
target.outputStream().use { output -> input.copyTo(output) }
|
||||
} ?: return ""
|
||||
// A private key with group or world read is one OpenSSH refuses to
|
||||
// use, and the refusal reads as "bad permissions" from a long way
|
||||
// away. Setting it here is what makes the file usable at all.
|
||||
if (kind == "identity") {
|
||||
runCatching {
|
||||
target.setReadable(false, false)
|
||||
target.setReadable(true, true)
|
||||
target.setWritable(false, false)
|
||||
target.setWritable(true, true)
|
||||
}
|
||||
}
|
||||
target.absolutePath
|
||||
} catch (_: Exception) {
|
||||
// A copy that failed leaves nothing behind; the caller sees "" and
|
||||
// the person can try again.
|
||||
""
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a URL in the device's browser.
|
||||
*
|
||||
* The page asks for this when a link points somewhere outside the kernel —
|
||||
* a provider's console, a documentation page — and loading it in place would
|
||||
* strand the person in a window with no address bar and no way back.
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun openExternal(url: String) {
|
||||
if (!trusted()) return
|
||||
// Only what a browser can open. A `file://` or `content://` here would
|
||||
// be the page reaching into the device through a method named for
|
||||
// leaving it.
|
||||
if (!url.startsWith("http://") && !url.startsWith("https://")) return
|
||||
onOpen(url)
|
||||
}
|
||||
|
||||
/** Put text on the clipboard. A WebView's own `navigator.clipboard` write is
|
||||
* gated behind a permission Android grants only to a focused document, so
|
||||
* the one thing the page actually offers to copy — a token, a command, a
|
||||
* path — has no other way out.
|
||||
*
|
||||
* Answers whether the write landed. A platform that refuses it raises
|
||||
* rather than answering, and an exception must not cross
|
||||
* `@JavascriptInterface` as a silent success — the page would say 已复制
|
||||
* over an empty clipboard. Caught here, the answer is the one the caller
|
||||
* can act on: false, and the button says 复制失败 instead. */
|
||||
@JavascriptInterface
|
||||
fun copyText(text: String): Boolean = if (!trusted()) false else runCatching {
|
||||
val clipboard = context.getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager
|
||||
clipboard.setPrimaryClip(ClipData.newPlainText("reasonix", text))
|
||||
}.isSuccess
|
||||
|
||||
/**
|
||||
* Write text where the person picks, and answer with the path it went to.
|
||||
*
|
||||
* The page's own route to a file — a blob URL on an `<a download>` — reaches
|
||||
* nothing here: a WebView has no downloads shelf, and its `DownloadListener`
|
||||
* fires only for a real navigation, not for a blob the page just built. So
|
||||
* the text crosses the bridge and the shell opens the system's document
|
||||
* picker, which is also the only way to reach a folder the person can find
|
||||
* again afterwards.
|
||||
*
|
||||
* "" is a dismissal, the same answer `importIntoHome` gives, and the page
|
||||
* reads the two the same way.
|
||||
*/
|
||||
/**
|
||||
* Put a copy of a file where a phone's own file manager will show it.
|
||||
*
|
||||
* `saveText` asks the person where to put it, which is the right question
|
||||
* on a desktop and the wrong one here: the kernel's workspace is
|
||||
* app-private storage, so the file someone wants is in a place no file
|
||||
* manager can open, and a picker that starts at "recent" is another place
|
||||
* to lose it. This answers the actual question — "where is it" — by
|
||||
* putting it in Downloads, in a folder named after the app, under a name
|
||||
* that is the file's own.
|
||||
*
|
||||
* The answer is the path relative to shared storage, so the page can say
|
||||
* where the file went rather than only that it worked. "" means the write
|
||||
* failed or the platform refused it.
|
||||
*
|
||||
* MediaStore on Android 10 and later, a plain file before that. The two
|
||||
* are not interchangeable: from 10 the app may not write another app's
|
||||
* files directly, and MediaStore is the only way in; before it there is no
|
||||
* MediaStore collection for downloads and the public directory is writable
|
||||
* with the storage permission this app does not ask for, so that branch is
|
||||
* best-effort and says so by answering "".
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun exportToDownloads(name: String, content: String): String {
|
||||
if (!trusted()) return ""
|
||||
val safeName = sanitise(name).ifEmpty { "reasonix.txt" }
|
||||
return exportFile(safeName, content).orEmpty()
|
||||
}
|
||||
|
||||
@JavascriptInterface
|
||||
fun saveText(name: String, content: String): String {
|
||||
if (!trusted()) return ""
|
||||
val safeName = sanitise(name).ifEmpty { "reasonix.txt" }
|
||||
return saveFile(safeName, content) ?: ""
|
||||
}
|
||||
|
||||
/**
|
||||
* Hand text to whatever the device shares with — a chat app, the mail
|
||||
* client, a note app.
|
||||
*
|
||||
* The share sheet rather than a second save dialog: "where is that file"
|
||||
* is the question this answers, and on a phone a document someone wants to
|
||||
* keep or send usually goes through one of those apps rather than a folder.
|
||||
*
|
||||
* Shared as text with the filename as its subject, which is what every
|
||||
* target understands. A file attachment would need a `FileProvider` and a
|
||||
* cache copy for the same words to arrive.
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun shareText(name: String, content: String): Boolean {
|
||||
if (!trusted()) return false
|
||||
val safeName = sanitise(name).ifEmpty { "reasonix.txt" }
|
||||
val posted = java.util.concurrent.CountDownLatch(1)
|
||||
var ok = false
|
||||
main.post {
|
||||
ok = runCatching { askForShare(safeName, content) }.getOrDefault(false)
|
||||
posted.countDown()
|
||||
}
|
||||
runCatching { posted.await(SHARE_TIMEOUT_SECONDS, TimeUnit.SECONDS) }
|
||||
return ok
|
||||
}
|
||||
|
||||
/**
|
||||
* The device's own light/dark answer.
|
||||
*
|
||||
* The page resolves its theme from `localStorage` first and only asks the
|
||||
* system when nothing is saved, so this is what makes a fresh install match
|
||||
* the phone instead of defaulting to light.
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun systemTheme(): String {
|
||||
val night = context.resources.configuration.uiMode and Configuration.UI_MODE_NIGHT_MASK
|
||||
return if (night == Configuration.UI_MODE_NIGHT_YES) "dark" else "light"
|
||||
}
|
||||
|
||||
/**
|
||||
* The window's real safe-area insets, in CSS pixels, as
|
||||
* `top,right,bottom,left`.
|
||||
*
|
||||
* A WebView does not implement `env(safe-area-inset-*)`: those are an iOS
|
||||
* Safari feature, and on Android every one of them resolves to `0`. So a
|
||||
* page that pads itself with them — which `mobile.css` does — is un-padded
|
||||
* on exactly the platform this shell exists for, and the first line of the
|
||||
* interface sits under the status bar.
|
||||
*
|
||||
* The values have to come from here for a second reason too: from Android 15
|
||||
* the window is edge-to-edge whether the app asks for it or not, so the bars
|
||||
* really are drawn over the page and there is no window-manager switch that
|
||||
* could move the content back down.
|
||||
*
|
||||
* Answered as a string because that is what crosses the bridge; the page
|
||||
* sets it as a CSS custom property and `mobile.css` reads it.
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun safeArea(): String {
|
||||
val insets = onInsets?.invoke() ?: return "0,0,0,0"
|
||||
return "${insets.left},${insets.top},${insets.right},${insets.bottom}"
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the pairing code a computer is showing.
|
||||
*
|
||||
* A phone cannot type what a computer renders as a QR code — it is a
|
||||
* one-time code inside a URL, and the whole reason it is a picture is that
|
||||
* nobody reads one aloud. So the page asks for the camera and gets the text
|
||||
* back, which is the same direction the computer's own dialog describes.
|
||||
*
|
||||
* The answer is the code's own text, not a parsed result: what a pairing
|
||||
* fragment means is the page's to decide, and it already knows how — the
|
||||
* `#pair=` handling ships with the interface.
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun scanPairingCode(): String = if (!trusted()) "" else scanWithCamera()
|
||||
|
||||
/**
|
||||
* Load a URL in this window instead of handing it to the browser.
|
||||
*
|
||||
* The page asks for this once, right after a scan: the code it read points at
|
||||
* the computer's own Studio, and the whole point of scanning is to end up
|
||||
* looking at that computer. Going out to a browser would leave the app on one
|
||||
* screen and the paired machine on another, with the back gesture an exit
|
||||
* rather than a return.
|
||||
*
|
||||
* Only http and https are accepted. The page is this repository's own build
|
||||
* and is trusted, but a scanner reads whatever is held in front of it, so the
|
||||
* one value that arrives here from outside is checked: a `file://` or
|
||||
* `javascript:` that happened to be on a printed code must not become a
|
||||
* navigation.
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun openInApp(url: String) {
|
||||
if (!trusted()) return
|
||||
val trimmed = url.trim()
|
||||
if (!trimmed.startsWith("http://") && !trimmed.startsWith("https://")) return
|
||||
main.post { onOpenInApp?.invoke(trimmed) }
|
||||
}
|
||||
|
||||
/** How the activity loads a URL in the window it owns. Set by the activity. */
|
||||
var onOpenInApp: ((String) -> Unit)? = null
|
||||
|
||||
// ---- the native browser view host bridge ----
|
||||
//
|
||||
// The page's `host.ts` looks for `window.reasonixHost` with Promise-based
|
||||
// methods (showBrowserView, hideBrowserView, etc.). `@JavascriptInterface`
|
||||
// is synchronous, so the round trip is: JS generates an id, calls
|
||||
// `hostCall(id, method, args)`, Kotlin does the work asynchronously, then
|
||||
// pushes the result back via `evaluateJavascript` calling
|
||||
// `window.__reasonixHostReply(id, json)`. The injected script (in
|
||||
// MainActivity) wires each Promise to that reply channel.
|
||||
|
||||
/** How the activity handles an async host call. Set by the activity. */
|
||||
var onHostCall: ((id: String, method: String, args: String) -> Unit)? = null
|
||||
|
||||
/**
|
||||
* Entry point for the page's `window.reasonixHost` Promise methods.
|
||||
*
|
||||
* `id` is a unique call identifier the JS side generated; `method` is the
|
||||
* method name (e.g. "showBrowserView"); `args` is the JSON-encoded argument
|
||||
* array. The activity handles the call asynchronously and replies through
|
||||
* `evaluateJavascript` with `window.__reasonixHostReply(id, resultJson)`.
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun hostCall(id: String, method: String, args: String) {
|
||||
if (!trusted()) return
|
||||
onHostCall?.invoke(id, method, args)
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the page has a panel up that the back gesture should close first.
|
||||
*
|
||||
* Written by the page, read by the activity's back handler. The page owns the
|
||||
* answer — it is the only thing that knows what is open, and a second
|
||||
* implementation of that question in the shell would be one that drifts — so
|
||||
* the page reports the fact and the shell keeps the action.
|
||||
*
|
||||
* Volatile because the two ends are on different threads: the page sets it
|
||||
* from the WebView's JavaScript thread, the back gesture reads it on the main
|
||||
* one, and a stale read here is a panel that does not close.
|
||||
*/
|
||||
@Volatile
|
||||
var layerOpen: Boolean = false
|
||||
private set
|
||||
|
||||
@JavascriptInterface
|
||||
fun setLayerOpen(open: Boolean) {
|
||||
layerOpen = open
|
||||
}
|
||||
|
||||
// ---- the picker, which is an Activity result in a synchronous world ----
|
||||
|
||||
private val pending = AtomicReference<Uri?>(null)
|
||||
private val latch = AtomicReference<CountDownLatch?>(null)
|
||||
|
||||
/** The code the camera read, held between the activity's answer and the
|
||||
* wait below — the same round trip the file picker takes. */
|
||||
private val scanned = AtomicReference<String?>(null)
|
||||
private val scanLatch = AtomicReference<CountDownLatch?>(null)
|
||||
|
||||
/** The main thread, because starting an Activity result must happen there. */
|
||||
private val main = Handler(Looper.getMainLooper())
|
||||
|
||||
/**
|
||||
* How to ask for a file. Set by the activity, which owns the Activity Result
|
||||
* API; it answers whether the ask was dispatched, not whether the person
|
||||
* picked anything — that arrives through `deliver`.
|
||||
*/
|
||||
var askForFile: () -> Boolean = { false }
|
||||
|
||||
/**
|
||||
* How the activity reports the window's insets. Set by the activity, which is
|
||||
* the only thing that can see them, and asked on every call rather than
|
||||
* cached — a rotation, a fold, or the keyboard opening all change them, and a
|
||||
* stale answer here is a stripe of interface under the status bar.
|
||||
*/
|
||||
var onInsets: (() -> Insets?)? = null
|
||||
|
||||
/**
|
||||
* How to open the camera. Set by the activity, which owns the Activity
|
||||
* Result API; like `askForFile` it answers whether the ask was dispatched,
|
||||
* and the code itself arrives through `deliverScan`.
|
||||
*/
|
||||
var askForScan: () -> Boolean = { false }
|
||||
|
||||
/**
|
||||
* How to put a file where the person picks. Set by the activity, which owns
|
||||
* the Activity Result API; it answers whether the ask was dispatched, and
|
||||
* the chosen place arrives through `deliverSave`.
|
||||
*/
|
||||
var askForSave: (name: String, content: String) -> Boolean = { _, _ -> false }
|
||||
|
||||
/**
|
||||
* How to open the share sheet. Set by the activity, which is the only thing
|
||||
* that can start an Intent; it answers whether anything on the device took
|
||||
* the text.
|
||||
*/
|
||||
var askForShare: (name: String, content: String) -> Boolean = { _, _ -> false }
|
||||
|
||||
/** The place the save dialog answered with, held between the two halves. */
|
||||
private val saveTarget = AtomicReference<Uri?>(null)
|
||||
private val saveLatch = AtomicReference<CountDownLatch?>(null)
|
||||
|
||||
/**
|
||||
* Ask where to put a file and write it there.
|
||||
*
|
||||
* The same latch round trip as `pickFile`, and for the same reason:
|
||||
* `@JavascriptInterface` cannot suspend, so the WebView's thread waits while
|
||||
* the picker is open — a modal panel the person is looking at.
|
||||
*/
|
||||
private fun saveFile(name: String, content: String): String? {
|
||||
val gate = CountDownLatch(1)
|
||||
saveLatch.set(gate)
|
||||
saveTarget.set(null)
|
||||
|
||||
val posted = CountDownLatch(1)
|
||||
var dispatched = false
|
||||
main.post {
|
||||
dispatched = runCatching { askForSave(name, content) }.getOrDefault(false)
|
||||
posted.countDown()
|
||||
}
|
||||
val ready = runCatching { posted.await(PICK_TIMEOUT_SECONDS, TimeUnit.SECONDS) }.getOrDefault(false)
|
||||
if (!ready || !dispatched) {
|
||||
saveLatch.set(null)
|
||||
return null
|
||||
}
|
||||
return try {
|
||||
if (!gate.await(PICK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) null
|
||||
else saveTarget.get()?.toString()
|
||||
} catch (_: InterruptedException) {
|
||||
null
|
||||
} finally {
|
||||
saveLatch.set(null)
|
||||
}
|
||||
}
|
||||
|
||||
/** Called by the activity once the bytes are written. A null uri is a dismissal. */
|
||||
fun deliverSave(uri: Uri?) {
|
||||
saveTarget.set(uri)
|
||||
saveLatch.get()?.countDown()
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the pairing code a computer is showing, and return it.
|
||||
*
|
||||
* The same shape as `pickFile` and for the same reason: a
|
||||
* `@JavascriptInterface` method is called on the WebView's thread and must
|
||||
* return a value, so the camera is opened on the main thread and this one
|
||||
* waits. The block lasts as long as the person is aiming, which is why the
|
||||
* timeout is generous — a scan that gave up after a few seconds would be
|
||||
* one nobody could complete.
|
||||
*
|
||||
* An empty string is "no code", which is what a refusal, a cancellation and
|
||||
* a timeout all are. The caller acts on the text, so a distinct error would
|
||||
* only be a fourth way to say the same thing.
|
||||
*/
|
||||
private fun scanWithCamera(): String {
|
||||
val gate = CountDownLatch(1)
|
||||
scanLatch.set(gate)
|
||||
scanned.set(null)
|
||||
|
||||
val posted = CountDownLatch(1)
|
||||
var dispatched = false
|
||||
main.post {
|
||||
dispatched = runCatching { askForScan() }.getOrDefault(false)
|
||||
posted.countDown()
|
||||
}
|
||||
val ready = runCatching { posted.await(SCAN_TIMEOUT_SECONDS, TimeUnit.SECONDS) }.getOrDefault(false)
|
||||
if (!ready || !dispatched) {
|
||||
scanLatch.set(null)
|
||||
return ""
|
||||
}
|
||||
return try {
|
||||
if (!gate.await(SCAN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) "" else scanned.get().orEmpty()
|
||||
} catch (_: InterruptedException) {
|
||||
""
|
||||
} finally {
|
||||
scanLatch.set(null)
|
||||
}
|
||||
}
|
||||
|
||||
/** Called by the activity when the scanner answers. Null is a dismissal. */
|
||||
fun deliverScan(text: String?) {
|
||||
scanned.set(text)
|
||||
scanLatch.get()?.countDown()
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask for one file and wait for the answer.
|
||||
*
|
||||
* `@JavascriptInterface` methods cannot suspend: the WebView calls them on
|
||||
* its own thread and expects a value back. So the ask is posted to the main
|
||||
* thread and this one waits on a latch. The cost is a blocked JS thread for
|
||||
* as long as the picker is open — which is a modal panel the person is
|
||||
* looking at, with nothing else in the page running.
|
||||
*
|
||||
* A timeout is set rather than waiting forever: a picker that never answers
|
||||
* (the activity destroyed underneath it) would otherwise pin a WebView
|
||||
* thread until the process died.
|
||||
*/
|
||||
private fun pickFile(): Uri? {
|
||||
val gate = CountDownLatch(1)
|
||||
latch.set(gate)
|
||||
pending.set(null)
|
||||
|
||||
val posted = CountDownLatch(1)
|
||||
var dispatched = false
|
||||
main.post {
|
||||
dispatched = runCatching { askForFile() }.getOrDefault(false)
|
||||
posted.countDown()
|
||||
}
|
||||
val ready = runCatching { posted.await(PICK_TIMEOUT_SECONDS, TimeUnit.SECONDS) }.getOrDefault(false)
|
||||
if (!ready || !dispatched) {
|
||||
latch.set(null)
|
||||
return null
|
||||
}
|
||||
|
||||
return try {
|
||||
if (!gate.await(PICK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) null else pending.get()
|
||||
} catch (_: InterruptedException) {
|
||||
null
|
||||
} finally {
|
||||
latch.set(null)
|
||||
}
|
||||
}
|
||||
|
||||
/** Called by the activity when the picker answers. A null uri is a dismissal. */
|
||||
fun deliver(uri: Uri?) {
|
||||
pending.set(uri)
|
||||
latch.get()?.countDown()
|
||||
}
|
||||
|
||||
/** The folder under Downloads that this app's exports land in. */
|
||||
private fun exportFolder(): String = "Reasonix"
|
||||
|
||||
/** The MIME type this name declares, or null when the platform does not know
|
||||
* it.
|
||||
*
|
||||
* Where the type is written, the name and the type agree, and that agreement
|
||||
* is the whole fix: `.md` is answered `text/markdown` here, so MediaStore
|
||||
* has nothing to correct and the export stays `报告.md` instead of being
|
||||
* renamed `报告.md.txt` by the `text/plain` it used to be given. Where the
|
||||
* type is unknown the field is left off, which leaves MediaStore to derive
|
||||
* the same `application/octet-stream` it would have been told — the name
|
||||
* still stands on its own, which is all this is for. */
|
||||
private fun mimeFor(name: String): String? {
|
||||
val ext = name.substringAfterLast('.', "").lowercase()
|
||||
if (ext.isEmpty() || ext == name.lowercase()) return null
|
||||
return runCatching { MimeTypeMap.getSingleton().getMimeTypeFromExtension(ext) }.getOrNull()
|
||||
}
|
||||
|
||||
private fun exportFile(name: String, content: String): String? = runCatching {
|
||||
val bytes = content.toByteArray(Charsets.UTF_8)
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
val resolver = context.contentResolver
|
||||
val values = ContentValues().apply {
|
||||
put(MediaStore.Downloads.DISPLAY_NAME, name)
|
||||
// The type is derived here rather than declared, and it is
|
||||
// written only when the name really has an extension that maps
|
||||
// to one.
|
||||
//
|
||||
// It used to be `text/plain` for everything, and MediaStore
|
||||
// answers a name whose extension disagrees with the declared
|
||||
// type by appending its own: exporting `报告.md` landed in
|
||||
// Downloads as `报告.md.txt`, and any source file came out with
|
||||
// the wrong second extension. Deriving the type from the name
|
||||
// instead keeps the two in agreement — which is what stops the
|
||||
// rename — and a name the platform has no type for is left off
|
||||
// entirely rather than declared as `application/octet-stream`,
|
||||
// so nothing is corrected there either.
|
||||
mimeFor(name)?.let { put(MediaStore.Downloads.MIME_TYPE, it) }
|
||||
put(
|
||||
MediaStore.Downloads.RELATIVE_PATH,
|
||||
Environment.DIRECTORY_DOWNLOADS + "/" + exportFolder(),
|
||||
)
|
||||
// Hidden until the bytes are all there, so a file manager that
|
||||
// reads the folder mid-write cannot show half a document.
|
||||
put(MediaStore.Downloads.IS_PENDING, 1)
|
||||
}
|
||||
val uri = resolver.insert(MediaStore.Downloads.EXTERNAL_CONTENT_URI, values)
|
||||
?: return@runCatching null
|
||||
resolver.openOutputStream(uri)?.use { it.write(bytes) } ?: return@runCatching null
|
||||
values.clear()
|
||||
values.put(MediaStore.Downloads.IS_PENDING, 0)
|
||||
resolver.update(uri, values, null, null)
|
||||
"${Environment.DIRECTORY_DOWNLOADS}/${exportFolder()}/$name"
|
||||
} else {
|
||||
val dir = File(
|
||||
Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS),
|
||||
exportFolder(),
|
||||
)
|
||||
if (!dir.exists() && !dir.mkdirs()) return@runCatching null
|
||||
val target = File(dir, name)
|
||||
target.writeBytes(bytes)
|
||||
target.absolutePath
|
||||
}
|
||||
}.getOrNull()
|
||||
|
||||
private fun sanitise(name: String): String {
|
||||
// A name from the page is used as a filename, so it is reduced to one:
|
||||
// no separators, no traversal, and nothing a filesystem treats as
|
||||
// special. The default is the name the field it came from suggests.
|
||||
val trimmed = name.trim().substringAfterLast('/').substringAfterLast('\\')
|
||||
if (trimmed.isEmpty() || trimmed == "." || trimmed == "..") return ""
|
||||
return trimmed.filter { it.isLetterOrDigit() || it in "._-" }.take(64)
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a turn is running, so the screen can be kept awake for one.
|
||||
*
|
||||
* The page owns the answer — it is the only thing that knows if the kernel
|
||||
* is working — and the shell owns the flag, because a window flag is not
|
||||
* something a page can set. Nothing is asked at launch: a window that is
|
||||
* merely open is one the person may be reading, and holding the screen on
|
||||
* for that is holding it on for nothing.
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun setRunning(running: Boolean) {
|
||||
onRunning?.invoke(running)
|
||||
}
|
||||
|
||||
/** How the activity applies that flag. Set by the activity. */
|
||||
var onRunning: ((Boolean) -> Unit)? = null
|
||||
|
||||
/**
|
||||
* Whether the app may read every file, which is what lets the kernel open a
|
||||
* folder on this phone as a workspace.
|
||||
*
|
||||
* Asked rather than assumed: the permission is granted by hand in system
|
||||
* settings, so the page has to be able to say "not yet" rather than fail on
|
||||
* the first `stat`. Answered `true` below API 30, where ordinary read
|
||||
* permission is enough and the concept does not exist.
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun hasAllFilesAccess(): Boolean = filesAccess?.invoke() ?: true
|
||||
|
||||
var filesAccess: (() -> Boolean)? = null
|
||||
|
||||
/** Open the system page where that permission is granted. The only place
|
||||
* Android offers it — there is no dialog for this one. */
|
||||
@JavascriptInterface
|
||||
fun requestAllFilesAccess() {
|
||||
if (!trusted()) return
|
||||
askForFilesAccess?.invoke()
|
||||
}
|
||||
|
||||
var askForFilesAccess: (() -> Unit)? = null
|
||||
|
||||
/**
|
||||
* Whether this app is exempt from the system's battery optimisation.
|
||||
*
|
||||
* A turn runs in a child process under a foreground service. Under doze the
|
||||
* whole app group is frozen anyway, so the turn stops making progress while
|
||||
* the screen is off — which is the state the person left it in on purpose.
|
||||
* Unrestricted is the answer, and it is a question Android answers.
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun batteryUnrestricted(): Boolean = if (!trusted()) false else (batteryFree?.invoke() ?: true)
|
||||
|
||||
var batteryFree: (() -> Boolean)? = null
|
||||
|
||||
/** Ask for that exemption. Unlike the storage one, this has a real dialog. */
|
||||
@JavascriptInterface
|
||||
fun requestBatteryUnrestricted() {
|
||||
if (!trusted()) return
|
||||
askForBattery?.invoke()
|
||||
}
|
||||
|
||||
var askForBattery: (() -> Unit)? = null
|
||||
|
||||
/**
|
||||
* Ask for a folder on this phone and answer with its real filesystem path.
|
||||
*
|
||||
* "" is a dismissal, the same answer `importIntoHome` gives — and, unlike
|
||||
* that one, an empty string is a normal outcome here rather than a failure.
|
||||
* The `startIn` hint is a path the kernel knows; the picker may ignore it,
|
||||
* which is the picker's business.
|
||||
*
|
||||
* The round trip is the same latch as `pickFile`, for the same reason: a
|
||||
* document-tree picker is another activity, and this call cannot suspend.
|
||||
*/
|
||||
@JavascriptInterface
|
||||
fun pickFolder(startIn: String): String {
|
||||
if (!trusted()) return ""
|
||||
val gate = CountDownLatch(1)
|
||||
folderLatch.set(gate)
|
||||
folderPending.set(null)
|
||||
|
||||
val posted = CountDownLatch(1)
|
||||
var dispatched = false
|
||||
main.post {
|
||||
dispatched = runCatching { askForFolder(startIn) }.getOrDefault(false)
|
||||
posted.countDown()
|
||||
}
|
||||
val ready = runCatching { posted.await(PICK_TIMEOUT_SECONDS, TimeUnit.SECONDS) }.getOrDefault(false)
|
||||
if (!ready || !dispatched) {
|
||||
folderLatch.set(null)
|
||||
return ""
|
||||
}
|
||||
return try {
|
||||
if (!gate.await(PICK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) "" else folderPending.get().orEmpty()
|
||||
} catch (_: InterruptedException) {
|
||||
""
|
||||
} finally {
|
||||
folderLatch.set(null)
|
||||
}
|
||||
}
|
||||
|
||||
var askForFolder: (String) -> Boolean = { false }
|
||||
|
||||
private val folderPending = AtomicReference<String?>(null)
|
||||
private val folderLatch = AtomicReference<CountDownLatch?>(null)
|
||||
|
||||
/** Called by the activity with the folder's real path, or null on dismissal. */
|
||||
fun deliverFolder(path: String?) {
|
||||
folderPending.set(path ?: "")
|
||||
folderLatch.get()?.countDown()
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val BRIDGE_VERSION = "4"
|
||||
private const val PICK_TIMEOUT_SECONDS = 5L * 60
|
||||
|
||||
/** Long, because this is however long someone takes to aim. A shorter
|
||||
* one would turn "still lining it up" into a failed scan. */
|
||||
private const val SCAN_TIMEOUT_SECONDS = 5L * 60
|
||||
|
||||
/** Only as long as the sheet takes to appear: the answer is "something
|
||||
* took it", which is known the moment the Intent resolves. */
|
||||
private const val SHARE_TIMEOUT_SECONDS = 15L
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The window's insets, in CSS pixels.
|
||||
*
|
||||
* A named type rather than four numbers because the page reads them as one
|
||||
* thing: half an inset is a layout that is wrong in one direction, and a caller
|
||||
* that passed the wrong pair would be indistinguishable from a device with no
|
||||
* cutout at all.
|
||||
*/
|
||||
data class Insets(val left: Int, val top: Int, val right: Int, val bottom: Int)
|
||||
@@ -0,0 +1,66 @@
|
||||
package io.reasonix.mobile.app
|
||||
|
||||
import androidx.camera.core.ImageProxy
|
||||
import com.google.zxing.BarcodeFormat
|
||||
import com.google.zxing.BinaryBitmap
|
||||
import com.google.zxing.DecodeHintType
|
||||
import com.google.zxing.MultiFormatReader
|
||||
import com.google.zxing.PlanarYUVLuminanceSource
|
||||
import com.google.zxing.common.HybridBinarizer
|
||||
|
||||
/**
|
||||
* Reading a QR code out of one camera frame.
|
||||
*
|
||||
* ZXing core and nothing else: it is pure Java, so it runs identically on a
|
||||
* phone with Google Play Services and on one whose HarmonyOS compatibility
|
||||
* layer has none. A decoder that leaned on ML Kit would work on the machine it
|
||||
* was written on and fail on the one that needs it.
|
||||
*
|
||||
* The frame arrives as YUV_420_888, whose Y plane is already the greyscale
|
||||
* bitmap ZXing wants — no conversion, no colour maths, just the bytes. That is
|
||||
* the whole reason a phone can decode 30 frames a second without warming up.
|
||||
*/
|
||||
object Qr {
|
||||
|
||||
private val reader = MultiFormatReader().apply {
|
||||
setHints(
|
||||
mapOf(
|
||||
// Only the form the computer actually renders. Leaving the
|
||||
// reader open to every 1D barcode makes each frame slower for
|
||||
// a case that cannot occur, and a slower frame is a code that
|
||||
// has already left the viewfinder.
|
||||
DecodeHintType.POSSIBLE_FORMATS to listOf(BarcodeFormat.QR_CODE),
|
||||
DecodeHintType.TRY_HARDER to true,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/** The code's text, or null when this frame holds none. */
|
||||
fun decode(image: ImageProxy): String? {
|
||||
val plane = image.planes.firstOrNull() ?: return null
|
||||
val buffer = plane.buffer
|
||||
val data = ByteArray(buffer.remaining())
|
||||
buffer.get(data)
|
||||
// `rowStride` is padded to a hardware alignment and is usually wider
|
||||
// than the picture. Passing the width as the row length is what tells
|
||||
// ZXing where each row actually ends; getting it wrong shears the image
|
||||
// and every code reads as garbage.
|
||||
val source = PlanarYUVLuminanceSource(
|
||||
data,
|
||||
plane.rowStride,
|
||||
image.height,
|
||||
0,
|
||||
0,
|
||||
image.width,
|
||||
image.height,
|
||||
false,
|
||||
)
|
||||
return runCatching {
|
||||
reader.decodeWithState(BinaryBitmap(HybridBinarizer(source))).text
|
||||
}.getOrNull().also {
|
||||
// The reader keeps per-frame state between calls; resetting is what
|
||||
// stops a partial match in one frame from corrupting the next.
|
||||
reader.reset()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package io.reasonix.mobile.app
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.os.Bundle
|
||||
import android.view.Gravity
|
||||
import android.view.ViewGroup
|
||||
import android.widget.FrameLayout
|
||||
import android.widget.TextView
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.appcompat.app.AppCompatActivity
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.camera.core.ImageAnalysis
|
||||
import androidx.camera.core.ImageProxy
|
||||
import androidx.camera.core.Preview
|
||||
import androidx.camera.lifecycle.ProcessCameraProvider
|
||||
import androidx.camera.view.PreviewView
|
||||
import androidx.core.content.ContextCompat
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* The camera, held open just long enough to read one pairing code.
|
||||
*
|
||||
* A phone that wants to drive a computer cannot type the code that computer
|
||||
* shows: it is a base64 fragment inside a URL, and the whole point of showing it
|
||||
* as a QR code is that nobody reads one aloud. So this is the way in — the same
|
||||
* gesture the computer's own dialog asks for, from the other side.
|
||||
*
|
||||
* The decode is ZXing rather than ML Kit on purpose. ML Kit needs Google Play
|
||||
* Services, and this app installs on phones whose HarmonyOS compatibility layer
|
||||
* has none: a GMS-backed scanner would work on a developer's phone and fail on
|
||||
* the device that needs it. ZXing core is pure Java, so the scanner behaves the
|
||||
* same on both.
|
||||
*
|
||||
* It returns the first code it reads and finishes. A pairing code is
|
||||
* single-use and short-lived, so "keep scanning after a hit" is not a feature —
|
||||
* it is a way to spend a code that a later frame would have discarded.
|
||||
*/
|
||||
class ScanActivity : AppCompatActivity() {
|
||||
|
||||
private lateinit var preview: PreviewView
|
||||
private lateinit var hint: TextView
|
||||
private val executor = Executors.newSingleThreadExecutor()
|
||||
|
||||
/** Set once a code is found, so the frames in flight behind it do not each
|
||||
* try to finish the activity. */
|
||||
@Volatile
|
||||
private var settled = false
|
||||
|
||||
private val requestCamera = registerForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
|
||||
if (granted) start() else refuse()
|
||||
}
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
val root = FrameLayout(this)
|
||||
preview = PreviewView(this).apply {
|
||||
layoutParams = FrameLayout.LayoutParams(
|
||||
ViewGroup.LayoutParams.MATCH_PARENT,
|
||||
ViewGroup.LayoutParams.MATCH_PARENT,
|
||||
)
|
||||
// The frames are analysed, not composed: filling the view and
|
||||
// cropping is right here and would be wrong for a camera app.
|
||||
scaleType = PreviewView.ScaleType.FILL_CENTER
|
||||
}
|
||||
hint = TextView(this).apply {
|
||||
text = getString(R.string.scan_hint)
|
||||
setTextColor(0xFFFFFFFF.toInt())
|
||||
textSize = 14f
|
||||
gravity = Gravity.CENTER
|
||||
val pad = (16 * resources.displayMetrics.density).toInt()
|
||||
setPadding(pad, pad, pad, pad)
|
||||
// Readable over whatever the camera is pointing at, which is the
|
||||
// one background a design system cannot choose.
|
||||
setBackgroundColor(0x99000000.toInt())
|
||||
layoutParams = FrameLayout.LayoutParams(
|
||||
ViewGroup.LayoutParams.MATCH_PARENT,
|
||||
ViewGroup.LayoutParams.WRAP_CONTENT,
|
||||
Gravity.BOTTOM,
|
||||
)
|
||||
}
|
||||
root.addView(preview)
|
||||
root.addView(hint)
|
||||
setContentView(root)
|
||||
|
||||
if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
) {
|
||||
start()
|
||||
} else {
|
||||
requestCamera.launch(Manifest.permission.CAMERA)
|
||||
}
|
||||
}
|
||||
|
||||
private fun refuse() {
|
||||
// No dialog of our own: the permission prompt already said no, and a
|
||||
// second window repeating it is one more tap to dismiss. The caller
|
||||
// gets the same answer as a cancelled scan.
|
||||
setResult(RESULT_CANCELED)
|
||||
finish()
|
||||
}
|
||||
|
||||
private fun start() {
|
||||
val providerFuture = ProcessCameraProvider.getInstance(this)
|
||||
providerFuture.addListener({
|
||||
val provider = runCatching { providerFuture.get() }.getOrNull() ?: return@addListener refuse()
|
||||
val previewUseCase = Preview.Builder().build().also { it.setSurfaceProvider(preview.surfaceProvider) }
|
||||
val analysis = ImageAnalysis.Builder()
|
||||
// Only the newest frame matters: a stale one describes where the
|
||||
// camera was, and a code that has already left the viewfinder
|
||||
// must not win over one that is in it now.
|
||||
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
|
||||
.build()
|
||||
.also { it.setAnalyzer(executor, ::analyse) }
|
||||
runCatching {
|
||||
provider.unbindAll()
|
||||
provider.bindToLifecycle(this, CameraSelector.DEFAULT_BACK_CAMERA, previewUseCase, analysis)
|
||||
}.onFailure { refuse() }
|
||||
}, ContextCompat.getMainExecutor(this))
|
||||
}
|
||||
|
||||
private fun analyse(image: ImageProxy) {
|
||||
try {
|
||||
if (settled) return
|
||||
val text = Qr.decode(image) ?: return
|
||||
if (settled) return
|
||||
settled = true
|
||||
setResult(RESULT_OK, Intent().putExtra(EXTRA_TEXT, text))
|
||||
// Finishing from the analysis thread is the whole reason `settled`
|
||||
// exists; the activity teardown has to happen on the main one.
|
||||
runOnUiThread { finish() }
|
||||
} finally {
|
||||
image.close()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
super.onDestroy()
|
||||
executor.shutdown()
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val EXTRA_TEXT = "text"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- The mark, drawn rather than shipped as a bitmap: one R in the app's own ink
|
||||
on the palette's ground, which scales to every launcher's mask without a
|
||||
density bucket per size. The hole in the R is the glyph's own counter, cut
|
||||
with evenOdd rather than painted over. -->
|
||||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="108dp"
|
||||
android:height="108dp"
|
||||
android:viewportWidth="108"
|
||||
android:viewportHeight="108">
|
||||
<path
|
||||
android:fillColor="#F4F1EA"
|
||||
android:fillType="evenOdd"
|
||||
android:pathData="M39,30 L39,78 L47,78 L47,60 L54.5,60 L65,78 L74.5,78 L62.5,58 C68.6,55.8 72.6,50.6 72.6,43.6 C72.6,35.4 66.8,30 57.6,30 Z M47,37 L56.4,37 C61.2,37 64.4,39.4 64.4,43.8 C64.4,48.2 61.2,50.9 56.4,50.9 L47,50.9 Z" />
|
||||
<path
|
||||
android:fillColor="#A6680C"
|
||||
android:pathData="M62,66 L74,66 L74,78 L62,78 Z" />
|
||||
</vector>
|
||||
@@ -0,0 +1,14 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- The status-bar glyph, not the launcher icon: Android renders a notification
|
||||
icon as a mask, so anything that is not a solid silhouette on transparency
|
||||
comes out as a grey blob. -->
|
||||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
android:viewportWidth="24"
|
||||
android:viewportHeight="24"
|
||||
android:tint="#FFFFFF">
|
||||
<path
|
||||
android:fillColor="#FFFFFF"
|
||||
android:pathData="M3,4h18v2H3zM3,9h18v2H3zM3,14h12v2H3zM3,19h12v2H3zM17.5,13.2l1.6,2.6 1.6,-2.6h2.3l-2.8,4.3V21h-2.2v-3.5L15.2,13.2z" />
|
||||
</vector>
|
||||
@@ -0,0 +1,62 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!--
|
||||
The screen between the tap and the page.
|
||||
|
||||
It exists for a reason that is not cosmetic: the kernel has to be unpacked and
|
||||
started before there is a page to show, and on a cold start that is about a
|
||||
second. A blank window for a second reads as a broken app; a line of text that
|
||||
says what is happening does not.
|
||||
|
||||
The progress bar is indeterminate and driven by nothing: the WebView's own
|
||||
progress is mirrored onto it while the page loads, and before that it simply
|
||||
turns, which is the honest description of "we do not know how long".
|
||||
-->
|
||||
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="match_parent"
|
||||
android:background="@color/page"
|
||||
android:orientation="vertical"
|
||||
android:gravity="center"
|
||||
android:paddingStart="32dp"
|
||||
android:paddingEnd="32dp">
|
||||
|
||||
<TextView
|
||||
android:id="@+id/splash_wordmark"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="@string/app_name"
|
||||
android:textColor="#8A857B"
|
||||
android:textSize="22sp"
|
||||
android:letterSpacing="0.08"
|
||||
android:fontFamily="sans-serif-medium" />
|
||||
|
||||
<ProgressBar
|
||||
android:id="@+id/splash_bar"
|
||||
style="?android:attr/progressBarStyleHorizontal"
|
||||
android:layout_width="132dp"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="26dp"
|
||||
android:indeterminate="true" />
|
||||
|
||||
<TextView
|
||||
android:id="@+id/splash_text"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="18dp"
|
||||
android:gravity="center"
|
||||
android:text="@string/splash_connecting"
|
||||
android:textColor="#8A857B"
|
||||
android:textSize="13sp"
|
||||
android:lineSpacingMultiplier="1.5" />
|
||||
|
||||
<Button
|
||||
android:id="@+id/splash_retry"
|
||||
style="?android:attr/borderlessButtonStyle"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_marginTop="14dp"
|
||||
android:minHeight="44dp"
|
||||
android:text="@string/splash_retry"
|
||||
android:textColor="#A6680C"
|
||||
android:visibility="gone" />
|
||||
</LinearLayout>
|
||||
@@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- The adaptive launcher icon.
|
||||
|
||||
minSdk is 26, which is the release that introduced adaptive icons, so this
|
||||
is the only launcher asset the app needs: there is no device running it
|
||||
that would ask for a bitmap fallback. -->
|
||||
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
<background android:drawable="@color/ic_launcher_background" />
|
||||
<!-- The safe zone is the middle 66dp of 108dp; the glyph is drawn inside it
|
||||
so no launcher's mask can clip it. -->
|
||||
<foreground android:drawable="@drawable/ic_launcher_foreground" />
|
||||
<monochrome android:drawable="@drawable/ic_launcher_foreground" />
|
||||
</adaptive-icon>
|
||||
@@ -0,0 +1,26 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<resources>
|
||||
<string name="app_name">Reasonix</string>
|
||||
|
||||
<string name="channel_kernel">Kernel running</string>
|
||||
<string name="channel_kernel_desc">Shown while the Reasonix kernel runs in the background. Can be muted on its own in system settings.</string>
|
||||
|
||||
<string name="notify_running_title">Reasonix is running</string>
|
||||
<string name="notify_running_body">A task is still going. Switch back to see it.</string>
|
||||
<string name="notify_failed_title">Reasonix could not start</string>
|
||||
|
||||
<string name="err_kernel_unreachable">Could not reach the kernel. Reopen the app from your home screen.</string>
|
||||
|
||||
<string name="download_default_name">reasonix-download</string>
|
||||
<string name="download_saved">Saved: %1$s</string>
|
||||
<string name="download_failed">Save failed: %1$s</string>
|
||||
<string name="download_failed_status">Save failed: the server answered %1$d</string>
|
||||
|
||||
<string name="splash_connecting">Starting the kernel…</string>
|
||||
<string name="splash_retry">Retry</string>
|
||||
|
||||
<string name="scan_title">Scan the code on the computer</string>
|
||||
<string name="scan_hint">Point at the QR code in the computer\'s device panel to connect to it.</string>
|
||||
<string name="scan_button">Scan to connect</string>
|
||||
<string name="scan_denied">No camera permission, so scanning is unavailable. Allow it in system settings.</string>
|
||||
</resources>
|
||||
@@ -0,0 +1,7 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- The dark palette's page colour. `DayNight` on the theme picks this up when
|
||||
the system is dark, which is what keeps the frame before the WebView from
|
||||
flashing light at someone using a dark phone at night. -->
|
||||
<resources>
|
||||
<color name="page">#0E1014</color>
|
||||
</resources>
|
||||
@@ -0,0 +1,9 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- The light palette's page colour, and only what the window needs before the
|
||||
WebView paints. The interface's own colours are `tokens.css`'s and are the
|
||||
page's business; these two exist so the first frame is the app's ground
|
||||
rather than the platform default white. -->
|
||||
<resources>
|
||||
<color name="page">#F4F1EA</color>
|
||||
<color name="boot">#07080C</color>
|
||||
</resources>
|
||||
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- The launcher ground: the interface's dark page colour. It is the same in
|
||||
both themes because a launcher icon is not themed by the app. -->
|
||||
<resources>
|
||||
<color name="ic_launcher_background">#14161C</color>
|
||||
</resources>
|
||||
@@ -0,0 +1,35 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Every string the shell itself shows. The interface's own text belongs to
|
||||
the page and lives in its i18n bundle; these are only the few lines that
|
||||
exist before it, or when it cannot load. -->
|
||||
<resources>
|
||||
<string name="app_name">Reasonix</string>
|
||||
|
||||
<string name="channel_kernel">内核运行中</string>
|
||||
<string name="channel_kernel_desc">Reasonix 内核在后台运行时显示，可在系统设置中单独关闭。</string>
|
||||
|
||||
<string name="notify_running_title">Reasonix 正在运行</string>
|
||||
<string name="notify_running_body">任务继续进行，切回应用即可查看。</string>
|
||||
<string name="notify_failed_title">Reasonix 无法启动</string>
|
||||
|
||||
<string name="err_kernel_unreachable">连接内核失败。请返回桌面重新打开应用。</string>
|
||||
|
||||
<string name="download_default_name">reasonix-download</string>
|
||||
<string name="download_saved">已保存：%1$s</string>
|
||||
<string name="download_failed">保存失败：%1$s</string>
|
||||
<string name="download_failed_status">保存失败：服务器返回 %1$d</string>
|
||||
|
||||
<string name="splash_connecting">正在启动内核…</string>
|
||||
<string name="splash_retry">重试</string>
|
||||
|
||||
<string name="wsperm_title">需要「所有文件访问」权限</string>
|
||||
<string name="wsperm_body">要打开手机上的文件夹作为工作区，需要先允许本应用访问所有文件。点「去设置」，在列表里打开 Reasonix 的开关，然后返回这里。</string>
|
||||
<string name="wsperm_go">去设置</string>
|
||||
<string name="wsperm_later">以后再说</string>
|
||||
|
||||
<string name="scan_title">扫描电脑上的二维码</string>
|
||||
<string name="scan_hint">对准电脑上「设备访问」里的二维码，识别后即可连接这台电脑。</string>
|
||||
<string name="scan_button">扫码连接电脑</string>
|
||||
<string name="scan_denied">没有相机权限，无法扫码。请在系统设置里允许。</string>
|
||||
<string name="share_title">分享文件</string>
|
||||
</resources>
|
||||
@@ -0,0 +1,28 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- The shell's own furniture: a window background matching the page's light
|
||||
palette before the WebView paints, and a transparent system bar so the
|
||||
page's own safe-area padding is what the notch meets. The page themes
|
||||
itself; this exists only so the first frame is not a white flash. -->
|
||||
<resources xmlns:tools="http://schemas.android.com/tools">
|
||||
|
||||
<style name="Theme.Reasonix" parent="Theme.AppCompat.DayNight.NoActionBar">
|
||||
<item name="android:windowBackground">@color/page</item>
|
||||
<item name="android:statusBarColor">@android:color/transparent</item>
|
||||
<item name="android:navigationBarColor">@android:color/transparent</item>
|
||||
<!-- The page draws its own edge-to-edge layout and pads itself with the
|
||||
safe-area insets from mobile.css. Laying the window out edge to edge
|
||||
while leaving the bars transparent is what makes those insets real
|
||||
numbers instead of zero. -->
|
||||
<item name="android:windowLayoutInDisplayCutoutMode" tools:targetApi="p">shortEdges</item>
|
||||
<item name="android:enforceStatusBarContrast" tools:targetApi="q">false</item>
|
||||
<item name="android:enforceNavigationBarContrast" tools:targetApi="q">false</item>
|
||||
</style>
|
||||
|
||||
<!-- The camera. A black ground because the preview has not drawn yet and a
|
||||
light window behind a dark viewfinder is a flash in the dark; the bars
|
||||
stay transparent so the preview reaches the edges, which is what makes
|
||||
a code near the top of the screen scannable. -->
|
||||
<style name="Theme.Reasonix.Scan" parent="Theme.Reasonix">
|
||||
<item name="android:windowBackground">@android:color/black</item>
|
||||
</style>
|
||||
</resources>
|
||||
@@ -0,0 +1,31 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!--
|
||||
Backups are off, and this file says why rather than leaving it implicit.
|
||||
|
||||
A Reasonix home holds provider keys, an account token and every session
|
||||
transcript. Android's auto-backup would copy all of it to the person's cloud
|
||||
drive and, on a device-to-device transfer, to a new phone — which is a
|
||||
credential leaving the device by a route nobody chose. `allowBackup="false"`
|
||||
in the manifest turns that off; these two stanzas are the belt to its braces,
|
||||
excluding everything explicitly so that a later change to `allowBackup` cannot
|
||||
quietly start copying credentials.
|
||||
|
||||
What a person wants to move between phones, the kernel already moves: its own
|
||||
export writes a file the person hands to the system's picker.
|
||||
-->
|
||||
<data-extraction-rules>
|
||||
<cloud-backup>
|
||||
<exclude domain="root" />
|
||||
<exclude domain="file" />
|
||||
<exclude domain="database" />
|
||||
<exclude domain="sharedpref" />
|
||||
<exclude domain="external" />
|
||||
</cloud-backup>
|
||||
<device-transfer>
|
||||
<exclude domain="root" />
|
||||
<exclude domain="file" />
|
||||
<exclude domain="database" />
|
||||
<exclude domain="sharedpref" />
|
||||
<exclude domain="external" />
|
||||
</device-transfer>
|
||||
</data-extraction-rules>
|
||||
@@ -0,0 +1,27 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!--
|
||||
Cleartext HTTP is permitted, and the platform has no way to permit less.
|
||||
|
||||
Two kinds of page load over plain HTTP: the kernel's own page on
|
||||
`127.0.0.1:<port>`, and the same page served by a paired computer on whatever
|
||||
address its router handed out. The port is chosen when the kernel starts and
|
||||
the LAN address when a code is scanned, so neither can be written here — and a
|
||||
`domain-config` matches literals and domains, never a subnet.
|
||||
|
||||
So the boundary is not this file. `MainActivity` decides which cleartext origin
|
||||
may become the window: `shouldOverrideUrlLoading` sends every http(s) origin
|
||||
that is not the kernel's or the one just paired out to the browser, and the
|
||||
native bridge answers only those two. A plaintext page on the LAN therefore
|
||||
cannot become the document holding the bridge, which is what the reviewer's
|
||||
"deliberate path for LAN pairing" asks for.
|
||||
|
||||
HTTPS is untouched: nothing here downgrades a request, and `trust-anchors`
|
||||
keeps the system roots for every TLS connection that is made.
|
||||
-->
|
||||
<network-security-config>
|
||||
<base-config cleartextTrafficPermitted="true">
|
||||
<trust-anchors>
|
||||
<certificates src="system" />
|
||||
</trust-anchors>
|
||||
</base-config>
|
||||
</network-security-config>
|
||||
@@ -0,0 +1,15 @@
|
||||
// The build itself lives in app/build.gradle; this file declares the plugin
|
||||
// versions once so the app module can be read on its own.
|
||||
//
|
||||
// AGP 8.11.1 is the pairing for the Gradle 8.14 toolchain this machine has:
|
||||
// AGP 8.7 and older refuse a Gradle that new, and AGP 8.13+ wants a Gradle this
|
||||
// one does not have. Both numbers are pinned rather than ranged, because a
|
||||
// build that silently moved either one would be a different build.
|
||||
plugins {
|
||||
id "com.android.application" version "8.11.1" apply false
|
||||
id "org.jetbrains.kotlin.android" version "2.1.20" apply false
|
||||
}
|
||||
|
||||
tasks.register("clean", Delete) {
|
||||
delete rootProject.layout.buildDirectory
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
# AndroidX is the only copy of the support libraries this project uses; the
|
||||
# non-transitive R class keeps the app's own resources from aliasing library
|
||||
# ones, which is what makes `R.string.*` unambiguous here.
|
||||
android.useAndroidX=true
|
||||
android.nonTransitiveRClass=true
|
||||
|
||||
# The daemon is what keeps a second `gradle assembleDebug` from re-reading the
|
||||
# whole SDK, and the heap is sized for the R8 pass over the WebView and the
|
||||
# packaged page rather than for this handful of Kotlin files.
|
||||
org.gradle.jvmargs=-Xmx3g -Dfile.encoding=UTF-8
|
||||
org.gradle.daemon=true
|
||||
org.gradle.parallel=true
|
||||
org.gradle.caching=true
|
||||
|
||||
kotlin.code.style=official
|
||||
File diff suppressed because it is too large.
Load diff
BIN
Binary file not shown.
@@ -0,0 +1,12 @@
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
|
||||
# Pinning the distribution's checksum is what makes the download verifiable: a
|
||||
# mirror that served a different zip would fail here instead of being unpacked
|
||||
# and run. The value is Gradle's own, from
|
||||
# https://services.gradle.org/distributions/gradle-8.14.3-bin.zip.sha256
|
||||
distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531
|
||||
networkTimeout=10000
|
||||
validateDistributionUrl=true
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
+251
@@ -0,0 +1,251 @@
|
||||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright © 2015-2021 the original authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
##############################################################################
|
||||
#
|
||||
# Gradle start up script for POSIX generated by Gradle.
|
||||
#
|
||||
# Important for running:
|
||||
#
|
||||
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
|
||||
# noncompliant, but you have some other compliant shell such as ksh or
|
||||
# bash, then to run this script, type that shell name before the whole
|
||||
# command line, like:
|
||||
#
|
||||
# ksh Gradle
|
||||
#
|
||||
# Busybox and similar reduced shells will NOT work, because this script
|
||||
# requires all of these POSIX shell features:
|
||||
# * functions;
|
||||
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
|
||||
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
|
||||
# * compound commands having a testable exit status, especially «case»;
|
||||
# * various built-in commands including «command», «set», and «ulimit».
|
||||
#
|
||||
# Important for patching:
|
||||
#
|
||||
# (2) This script targets any POSIX shell, so it avoids extensions provided
|
||||
# by Bash, Ksh, etc; in particular arrays are avoided.
|
||||
#
|
||||
# The "traditional" practice of packing multiple parameters into a
|
||||
# space-separated string is a well documented source of bugs and security
|
||||
# problems, so this is (mostly) avoided, by progressively accumulating
|
||||
# options in "$@", and eventually passing that to Java.
|
||||
#
|
||||
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
|
||||
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
|
||||
# see the in-line comments for details.
|
||||
#
|
||||
# There are tweaks for specific operating systems such as AIX, CygWin,
|
||||
# Darwin, MinGW, and NonStop.
|
||||
#
|
||||
# (3) This script is generated from the Groovy template
|
||||
# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
|
||||
# within the Gradle project.
|
||||
#
|
||||
# You can find Gradle at https://github.com/gradle/gradle/.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
# Attempt to set APP_HOME
|
||||
|
||||
# Resolve links: $0 may be a link
|
||||
app_path=$0
|
||||
|
||||
# Need this for daisy-chained symlinks.
|
||||
while
|
||||
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
|
||||
[ -h "$app_path" ]
|
||||
do
|
||||
ls=$( ls -ld "$app_path" )
|
||||
link=${ls#*' -> '}
|
||||
case $link in #(
|
||||
/*) app_path=$link ;; #(
|
||||
*) app_path=$APP_HOME$link ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# This is normally unused
|
||||
# shellcheck disable=SC2034
|
||||
APP_BASE_NAME=${0##*/}
|
||||
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD=maximum
|
||||
|
||||
warn () {
|
||||
echo "$*"
|
||||
} >&2
|
||||
|
||||
die () {
|
||||
echo
|
||||
echo "$*"
|
||||
echo
|
||||
exit 1
|
||||
} >&2
|
||||
|
||||
# OS specific support (must be 'true' or 'false').
|
||||
cygwin=false
|
||||
msys=false
|
||||
darwin=false
|
||||
nonstop=false
|
||||
case "$( uname )" in #(
|
||||
CYGWIN* ) cygwin=true ;; #(
|
||||
Darwin* ) darwin=true ;; #(
|
||||
MSYS* | MINGW* ) msys=true ;; #(
|
||||
NONSTOP* ) nonstop=true ;;
|
||||
esac
|
||||
|
||||
CLASSPATH="\\\"\\\""
|
||||
|
||||
|
||||
# Determine the Java command to use to start the JVM.
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD=$JAVA_HOME/jre/sh/java
|
||||
else
|
||||
JAVACMD=$JAVA_HOME/bin/java
|
||||
fi
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
else
|
||||
JAVACMD=java
|
||||
if ! command -v java >/dev/null 2>&1
|
||||
then
|
||||
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
||||
case $MAX_FD in #(
|
||||
max*)
|
||||
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
MAX_FD=$( ulimit -H -n ) ||
|
||||
warn "Could not query maximum file descriptor limit"
|
||||
esac
|
||||
case $MAX_FD in #(
|
||||
'' | soft) :;; #(
|
||||
*)
|
||||
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
ulimit -n "$MAX_FD" ||
|
||||
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||
esac
|
||||
fi
|
||||
|
||||
# Collect all arguments for the java command, stacking in reverse order:
|
||||
# * args from the command line
|
||||
# * the main class name
|
||||
# * -classpath
|
||||
# * -D...appname settings
|
||||
# * --module-path (only if needed)
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
|
||||
|
||||
# For Cygwin or MSYS, switch paths to Windows format before running java
|
||||
if "$cygwin" || "$msys" ; then
|
||||
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
|
||||
CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
|
||||
|
||||
JAVACMD=$( cygpath --unix "$JAVACMD" )
|
||||
|
||||
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||
for arg do
|
||||
if
|
||||
case $arg in #(
|
||||
-*) false ;; # don't mess with options #(
|
||||
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
|
||||
[ -e "$t" ] ;; #(
|
||||
*) false ;;
|
||||
esac
|
||||
then
|
||||
arg=$( cygpath --path --ignore --mixed "$arg" )
|
||||
fi
|
||||
# Roll the args list around exactly as many times as the number of
|
||||
# args, so each arg winds up back in the position where it started, but
|
||||
# possibly modified.
|
||||
#
|
||||
# NB: a `for` loop captures its iteration list before it begins, so
|
||||
# changing the positional parameters here affects neither the number of
|
||||
# iterations, nor the values presented in `arg`.
|
||||
shift # remove old arg
|
||||
set -- "$@" "$arg" # push replacement arg
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Collect all arguments for the java command:
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
|
||||
# and any embedded shellness will be escaped.
|
||||
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
|
||||
# treated as '${Hostname}' itself on the command line.
|
||||
|
||||
set -- \
|
||||
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||
-classpath "$CLASSPATH" \
|
||||
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
|
||||
"$@"
|
||||
|
||||
# Stop when "xargs" is not available.
|
||||
if ! command -v xargs >/dev/null 2>&1
|
||||
then
|
||||
die "xargs is not available"
|
||||
fi
|
||||
|
||||
# Use "xargs" to parse quoted args.
|
||||
#
|
||||
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
|
||||
#
|
||||
# In Bash we could simply go:
|
||||
#
|
||||
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
|
||||
# set -- "${ARGS[@]}" "$@"
|
||||
#
|
||||
# but POSIX shell has neither arrays nor command substitution, so instead we
|
||||
# post-process each arg (as a line of input to sed) to backslash-escape any
|
||||
# character that might be a shell metacharacter, then use eval to reverse
|
||||
# that process (while maintaining the separation between arguments), and wrap
|
||||
# the whole thing up as a single "set" statement.
|
||||
#
|
||||
# This will of course break if any of these variables contains a newline or
|
||||
# an unmatched quote.
|
||||
#
|
||||
|
||||
eval "set -- $(
|
||||
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
|
||||
xargs -n1 |
|
||||
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
|
||||
tr '\n' ' '
|
||||
)" '"$@"'
|
||||
|
||||
exec "$JAVACMD" "$@"
|
||||
Vendored
+94
@@ -0,0 +1,94 @@
|
||||
@rem
|
||||
@rem Copyright 2015 the original author or authors.
|
||||
@rem
|
||||
@rem Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@rem you may not use this file except in compliance with the License.
|
||||
@rem You may obtain a copy of the License at
|
||||
@rem
|
||||
@rem https://www.apache.org/licenses/LICENSE-2.0
|
||||
@rem
|
||||
@rem Unless required by applicable law or agreed to in writing, software
|
||||
@rem distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@rem See the License for the specific language governing permissions and
|
||||
@rem limitations under the License.
|
||||
@rem
|
||||
@rem SPDX-License-Identifier: Apache-2.0
|
||||
@rem
|
||||
|
||||
@if "%DEBUG%"=="" @echo off
|
||||
@rem ##########################################################################
|
||||
@rem
|
||||
@rem Gradle startup script for Windows
|
||||
@rem
|
||||
@rem ##########################################################################
|
||||
|
||||
@rem Set local scope for the variables with windows NT shell
|
||||
if "%OS%"=="Windows_NT" setlocal
|
||||
|
||||
set DIRNAME=%~dp0
|
||||
if "%DIRNAME%"=="" set DIRNAME=.
|
||||
@rem This is normally unused
|
||||
set APP_BASE_NAME=%~n0
|
||||
set APP_HOME=%DIRNAME%
|
||||
|
||||
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
|
||||
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
|
||||
|
||||
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
|
||||
|
||||
@rem Find java.exe
|
||||
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||
|
||||
set JAVA_EXE=java.exe
|
||||
%JAVA_EXE% -version >NUL 2>&1
|
||||
if %ERRORLEVEL% equ 0 goto execute
|
||||
|
||||
echo. 1>&2
|
||||
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
|
||||
echo. 1>&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||
echo location of your Java installation. 1>&2
|
||||
|
||||
goto fail
|
||||
|
||||
:findJavaFromJavaHome
|
||||
set JAVA_HOME=%JAVA_HOME:"=%
|
||||
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
||||
|
||||
if exist "%JAVA_EXE%" goto execute
|
||||
|
||||
echo. 1>&2
|
||||
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
|
||||
echo. 1>&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||
echo location of your Java installation. 1>&2
|
||||
|
||||
goto fail
|
||||
|
||||
:execute
|
||||
@rem Setup the command line
|
||||
|
||||
set CLASSPATH=
|
||||
|
||||
|
||||
@rem Execute Gradle
|
||||
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %*
|
||||
|
||||
:end
|
||||
@rem End local scope for the variables with windows NT shell
|
||||
if %ERRORLEVEL% equ 0 goto mainEnd
|
||||
|
||||
:fail
|
||||
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
||||
rem the _cmd.exe /c_ return code!
|
||||
set EXIT_CODE=%ERRORLEVEL%
|
||||
if %EXIT_CODE% equ 0 set EXIT_CODE=1
|
||||
if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
|
||||
exit /b %EXIT_CODE%
|
||||
|
||||
:mainEnd
|
||||
if "%OS%"=="Windows_NT" endlocal
|
||||
|
||||
:omega
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Build the Reasonix kernel for Android and place it where the APK packager
|
||||
# expects a native library.
|
||||
#
|
||||
# Why a shell script and not a Gradle task: the kernel is a Go module that lives
|
||||
# outside this Gradle build, with its own toolchain version pinned in go.mod. A
|
||||
# Gradle task that invoked `go build` would make the Android build depend on a Go
|
||||
# toolchain being configured for it, and would rebuild a 21MB binary on every
|
||||
# `assembleDebug` even when nothing in the kernel changed. This runs when you
|
||||
# mean it to, and the APK task only checks that the result is there.
|
||||
#
|
||||
# Output: app/src/main/jniLibs/arm64-v8a/libreasonix.so
|
||||
# The name is not cosmetic. Android extracts files under `jniLibs/` to
|
||||
# `nativeLibraryDir` and permits execution *there*; a binary shipped in
|
||||
# `assets/` would have to be copied to app storage and chmod'd, which fails on
|
||||
# a device whose data partition is mounted noexec. `lib*.so` is the shape the
|
||||
# packager recognises as a native library, so the kernel wears it.
|
||||
set -euo pipefail
|
||||
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
android="$here/.."
|
||||
upstream="$android/.."
|
||||
out="$android/app/src/main/jniLibs/arm64-v8a/libreasonix.so"
|
||||
|
||||
if [ ! -d "$upstream" ]; then
|
||||
echo "kernel source not found: $upstream" >&2
|
||||
echo "run this from a checkout: the parent of android/ must be the kernel source." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# GOOS=android needs nothing else: the kernel is CGO-free, which is what makes a
|
||||
# single static binary possible in the first place. `-s -w` drops the symbol and
|
||||
# DWARF tables — about a third of the file — and nothing on a phone reads them.
|
||||
export GOOS=android
|
||||
export GOARCH=arm64
|
||||
export CGO_ENABLED=0
|
||||
|
||||
# The app version, read from the module's own build.gradle so the two cannot
|
||||
# drift. The official build stamps these through `-X` (see upstream's Makefile);
|
||||
# without them the kernel reports `version=dev revision=unknown`, which a phone
|
||||
# then shows next to the documentation it was built with and makes a bug report
|
||||
# from that build impossible to place. There is no `.git` here, so the revision
|
||||
# names the package the corpus shipped in rather than a commit hash — the honest
|
||||
# answer when there is no commit to name.
|
||||
app_gradle="$android/app/build.gradle"
|
||||
version_field() {
|
||||
sed -n "s/^def $1 *= *\([0-9]*\).*/\1/p" "$app_gradle" | head -n1
|
||||
}
|
||||
app_version="$(version_field versionMajor).$(version_field versionMinor).$(version_field versionPatch)"
|
||||
build_time_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
ldflags="-s -w"
|
||||
ldflags="$ldflags -X main.version=v$app_version"
|
||||
ldflags="$ldflags -X main.buildTimeUTC=$build_time_utc"
|
||||
ldflags="$ldflags -X reasonix/internal/tools/productdocs.linkedVersion=v$app_version"
|
||||
ldflags="$ldflags -X reasonix/internal/tools/productdocs.linkedRevision=apk-v$app_version"
|
||||
|
||||
echo "building the kernel (android/arm64, v$app_version) ..."
|
||||
mkdir -p "$(dirname "$out")"
|
||||
cd "$upstream"
|
||||
go build -trimpath -ldflags="$ldflags" -o "$out" ./cmd/reasonix
|
||||
|
||||
size=$(wc -c < "$out")
|
||||
echo "kernel ready: $out ($((size / 1024 / 1024)) MiB)"
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# One command from a clean checkout to an installable APK.
|
||||
#
|
||||
# The order matters and is the whole content of this script: the kernel and the
|
||||
# page are inputs to packaging, not build steps inside it, so they are produced
|
||||
# first and the Gradle task only assembles what is already there. Running
|
||||
# `gradle assembleRelease` directly on a fresh checkout fails on purpose (see
|
||||
# `requireBuildInputs` in app/build.gradle) rather than producing an APK with no
|
||||
# kernel in it.
|
||||
#
|
||||
# Usage: ./scripts/build.sh [debug|release] (default: release)
|
||||
set -euo pipefail
|
||||
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
android="$here/.."
|
||||
variant="${1:-release}"
|
||||
|
||||
case "$variant" in
|
||||
debug|release) ;;
|
||||
*) echo "usage: $0 [debug|release]" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
"$here/build-kernel.sh"
|
||||
"$here/sync-page.sh"
|
||||
|
||||
cd "$android"
|
||||
|
||||
# The wrapper is committed, jar and all, and it is the only thing that runs the
|
||||
# build. Generating one here would tie the dependency versions to whichever
|
||||
# `gradle` happened to be on PATH, which is the opposite of a pinned build.
|
||||
if [ ! -f gradlew ]; then
|
||||
echo "gradlew is missing: this repository commits the Gradle wrapper, so the checkout is incomplete" >&2
|
||||
echo "(a release build does not use `gradle` from PATH: that would tie dependency versions to the machine)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
task="assemble$([ "$variant" = release ] && echo Release || echo Debug)"
|
||||
echo "packaging the APK ($variant) ..."
|
||||
./gradlew "$task" --no-daemon
|
||||
|
||||
apk_dir="$android/app/build/outputs/apk/$variant"
|
||||
echo
|
||||
echo "done. Installable artifacts:"
|
||||
find "$apk_dir" -name '*.apk' -maxdepth 1 -print -exec ls -lh {} \;
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Install the built APK on a connected phone and show its log.
|
||||
#
|
||||
# `adb` comes from the Android SDK rather than this project, because the SDK is
|
||||
# already how the app is built. Point ANDROID_HOME at it, or have `adb` on PATH.
|
||||
#
|
||||
# Usage: ./scripts/run.sh [debug|release]
|
||||
set -euo pipefail
|
||||
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
android="$here/.."
|
||||
variant="${1:-release}"
|
||||
|
||||
adb="${ADB:-${ANDROID_HOME:-$HOME/AndroidSDK}/platform-tools/adb}"
|
||||
if ! command -v "$adb" >/dev/null 2>&1; then
|
||||
adb="adb"
|
||||
fi
|
||||
|
||||
apk_dir="$android/app/build/outputs/apk/$variant"
|
||||
apk=$(find "$apk_dir" -name '*.apk' -maxdepth 1 2>/dev/null | head -n1)
|
||||
if [ -z "$apk" ]; then
|
||||
echo "no $variant APK found; run ./scripts/build.sh $variant first" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "installing $apk ..."
|
||||
"$adb" install -r "$apk"
|
||||
|
||||
# The device count is checked before the app is started so an emulator-less
|
||||
# machine gets "no devices" rather than a launch failure that reads like a bug.
|
||||
if [ "$("$adb" devices | grep -c 'device$')" -eq 0 ]; then
|
||||
echo "no device attached; skipping the launch." >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
package="io.reasonix.mobile.app"
|
||||
[ "$variant" = debug ] && package="io.reasonix.mobile.app.debug"
|
||||
"$adb" shell monkey -p "$package" -c android.intent.category.LAUNCHER 1 >/dev/null
|
||||
|
||||
echo
|
||||
echo "following the log (Ctrl-C to stop) ..."
|
||||
"$adb" logcat -v color "ReasonixKernel:V" "AndroidRuntime:E" "*:S"
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Build the interface and copy it where the APK will carry it.
|
||||
#
|
||||
# The page that ships is the same build the desktop Studio serves — that is the
|
||||
# whole reason the mobile layout lives in `desktop/frontend-next` rather than in
|
||||
# an Android project's own web root. Nothing is edited on the way through: the
|
||||
# only thing this script does that `pnpm build` does not is copy `dist/` into the
|
||||
# app's assets.
|
||||
#
|
||||
# Output: app/src/main/assets/page/
|
||||
set -euo pipefail
|
||||
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
android="$here/.."
|
||||
frontend="$android/../desktop/frontend-next"
|
||||
dest="$android/app/src/main/assets/page"
|
||||
|
||||
if [ ! -d "$frontend" ]; then
|
||||
echo "frontend project not found: $frontend" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "$frontend"
|
||||
|
||||
# pnpm is the package manager the lockfile is written for; npm would resolve a
|
||||
# different tree than the one that was tested.
|
||||
if ! command -v pnpm >/dev/null 2>&1; then
|
||||
echo "pnpm 10 is required: npm install -g pnpm@10" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d node_modules ]; then
|
||||
echo "installing frontend dependencies ..."
|
||||
pnpm install --frozen-lockfile
|
||||
fi
|
||||
|
||||
echo "building the frontend ..."
|
||||
pnpm build
|
||||
|
||||
rm -rf "$dest"
|
||||
mkdir -p "$dest"
|
||||
cp -R dist/. "$dest/"
|
||||
|
||||
if [ ! -f "$dest/index.html" ]; then
|
||||
echo "the build produced no index.html; the frontend build probably failed." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
count=$(find "$dest" -type f | wc -l)
|
||||
size=$(du -sk "$dest" | cut -f1)
|
||||
echo "page ready: $dest ($count files, $((size / 1024)) MiB)"
|
||||
|
||||
# MIT requires the copyright notice and the licence text to travel with a binary
|
||||
# distribution, so the APK carries it. Generated from the repository's own
|
||||
# `LICENSE` rather than committed a second time: a checked-in copy is one more
|
||||
# thing that drifts when the root file is updated.
|
||||
root_license="$android/../LICENSE"
|
||||
if [ ! -f "$root_license" ]; then
|
||||
echo "LICENSE not found: $root_license" >&2
|
||||
exit 1
|
||||
fi
|
||||
cp "$root_license" "$android/app/src/main/assets/LICENSE"
|
||||
echo "licence: app/src/main/assets/LICENSE (copied from the repository's LICENSE)"
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Reproduce what `Kernel.kt` does, on this machine, against the page that is
|
||||
# actually packaged into the APK.
|
||||
#
|
||||
# This is the closest thing to an end-to-end test that runs without a phone: the
|
||||
# launch flags, the token file, the port file and the `--page` directory are the
|
||||
# same ones the Kotlin shell passes, and the page is read out of
|
||||
# `app/src/main/assets/page` rather than the frontend's own `dist/`. If this
|
||||
# serves a working interface, the only untested part is Android itself.
|
||||
set -euo pipefail
|
||||
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
android="$here/.."
|
||||
page="$android/app/src/main/assets/page"
|
||||
kernel="${KERNEL_BIN:-$android/../reasonix-verify.exe}"
|
||||
|
||||
work="${VERIFY_DIR:-/tmp/reasonix-verify}"
|
||||
rm -rf "$work"
|
||||
mkdir -p "$work/home/remote"
|
||||
|
||||
if [ ! -f "$kernel" ]; then
|
||||
echo "no runnable kernel; build one first:" >&2
|
||||
echo " (cd .. && go build -o reasonix-verify.exe ./cmd/reasonix)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$page/index.html" ]; then
|
||||
echo "no packaged page: $page/index.html" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 32 random bytes as hex — the same shape `Kernel.mintToken` writes.
|
||||
token=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n')
|
||||
printf '%s\n' "$token" > "$work/home/remote/launch.token"
|
||||
|
||||
export REASONIX_HOME="$work/home"
|
||||
export HOME="$work/home"
|
||||
export TMPDIR="$work"
|
||||
|
||||
echo "kernel: $kernel"
|
||||
echo "page: $page"
|
||||
echo "token: ${token:0:8}... (${#token} chars)"
|
||||
|
||||
"$kernel" serve \
|
||||
--addr 127.0.0.1:0 \
|
||||
--auth token \
|
||||
--token-file "$work/home/remote/launch.token" \
|
||||
--port-file "$work/home/remote/serve.port" \
|
||||
--pid-file "$work/home/remote/serve.pid" \
|
||||
--no-open \
|
||||
-page "$page" \
|
||||
> "$work/kernel.log" 2>&1 &
|
||||
echo $! > "$work/kernel.pid"
|
||||
|
||||
# Wait for the port file the kernel writes after binding, which is exactly the
|
||||
# wait `Kernel.awaitPort` performs.
|
||||
for _ in $(seq 1 150); do
|
||||
if [ -s "$work/home/remote/serve.port" ]; then break; fi
|
||||
if ! kill -0 "$(cat "$work/kernel.pid")" 2>/dev/null; then
|
||||
echo "the kernel exited; log:" >&2
|
||||
cat "$work/kernel.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 0.2
|
||||
done
|
||||
|
||||
if [ ! -s "$work/home/remote/serve.port" ]; then
|
||||
echo "the kernel reported no listening address; log:" >&2
|
||||
cat "$work/kernel.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
port=$(cat "$work/home/remote/serve.port")
|
||||
echo "listening on: $port"
|
||||
|
||||
# The two things the WebView will do: fetch the page, and exist on the origin.
|
||||
probe=$(curl -s -o /dev/null -w '%{http_code}' "http://$port/")
|
||||
echo "GET / -> $probe"
|
||||
if [ "$probe" != "200" ]; then
|
||||
echo "the page did not answer; log:" >&2
|
||||
cat "$work/kernel.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat > "$work/entry.txt" <<EOF
|
||||
origin=http://$port
|
||||
entry=http://$port/#token=$token
|
||||
token=$token
|
||||
EOF
|
||||
|
||||
echo "entry written: $work/entry.txt"
|
||||
echo "kernel log (first 10 lines):"
|
||||
head -10 "$work/kernel.log"
|
||||
@@ -0,0 +1,31 @@
|
||||
// Reasonix Mobile — the Android shell around the Reasonix kernel.
|
||||
//
|
||||
// The shell is deliberately thin: it unpacks the kernel and the built page that
|
||||
// ship inside the APK, starts `reasonix serve` on a loopback port, and points a
|
||||
// WebView at it. Everything the person sees and does is the same page the
|
||||
// desktop Studio and `reasonix web` serve, because it is literally the same
|
||||
// build of `desktop/frontend-next` — which is what keeps one interface honest
|
||||
// across the three surfaces, and why the page is packaged rather than fetched.
|
||||
|
||||
pluginManagement {
|
||||
repositories {
|
||||
// The official repositories only. A mirror listed here would be an
|
||||
// unverified trust anchor for anything that ends up in a signed build:
|
||||
// the artifact it serves is whichever copy landed there. A machine that
|
||||
// needs a mirror belongs on a user-level init script, where the choice
|
||||
// stays with the person building rather than travelling with the source.
|
||||
google()
|
||||
mavenCentral()
|
||||
gradlePluginPortal()
|
||||
}
|
||||
}
|
||||
|
||||
dependencyResolutionManagement {
|
||||
repositories {
|
||||
google()
|
||||
mavenCentral()
|
||||
}
|
||||
}
|
||||
|
||||
rootProject.name = "ReasonixMobile"
|
||||
include(":app")
|
||||
@@ -103,6 +103,10 @@ func newHarness(t *testingDir, p provider.Provider, window int, rec *callRecorde
|
||||
ContextWindow: window,
|
||||
ArchiveDir: filepath.Join(t.dir, "archive"),
|
||||
SessionPath: path,
|
||||
RecentKeep: 4,
|
||||
// boot's default when cfg.Agent.Keep is unset; without it the bench
|
||||
// would measure a configuration no real session runs.
|
||||
KeepPolicy: agent.KeepErrors,
|
||||
}, rec.sink())
|
||||
return &harness{sess: sess, agentA: a, path: path, calls: rec}
|
||||
}
|
||||
|
||||
@@ -108,6 +108,7 @@ func newAgent(p provider.Provider, sess *sessionstore.Session, path string, sink
|
||||
return agent.New(p, tool.NewRegistry(), sess, agent.Options{
|
||||
ContextWindow: windowTokens,
|
||||
CompactRatio: compactRatio,
|
||||
RecentKeep: 2,
|
||||
SessionPath: path,
|
||||
WorkspaceID: "cm-e2e",
|
||||
ModelRef: "deepseek/" + model,
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
# Context retrieval bench
|
||||
|
||||
Two questions about long-horizon memory, and the harness that makes their
|
||||
answers trustworthy.
|
||||
|
||||
-mode=preflight build all 18 tasks under every arm and check the contract
|
||||
-mode=snippet does a rank-1 hit hand over the whole answer?
|
||||
-mode=adversarial send a model after the answer through every host surface
|
||||
-mode=run-search Experiment R: is searchable canonical a real capability?
|
||||
-mode=run-boundary Experiment I, paired: the 6 efficiency tasks across their boundary
|
||||
-mode=run-index Experiment I, full: the 6 efficiency tasks x 4 budgets
|
||||
-mode=calibrate find each index task's PlantAfterGen for its cue tier
|
||||
|
||||
`-dry` drives any run mode with a scripted provider, for nothing. Real runs need
|
||||
`DEEPSEEK_API_KEY`.
|
||||
|
||||
## Status
|
||||
|
||||
**Searchable canonical recall: proven.** Six of six recovered with search on,
|
||||
three of six with it off, and the successes without it came from enumerating
|
||||
addresses by hand at roughly nine times the page-in cost (139 vs 1292
|
||||
tokens/task). Measured after three separate leaks were closed, so the numbers
|
||||
are from a clean environment.
|
||||
|
||||
**Fold index marginal utility: direction reproduced, size still conditional.** On the v2
|
||||
substrate the direction holds in six of six tasks and in the pooled paired batch: with the
|
||||
cue visible the index addresses the target, so the same task is searched less and pages in
|
||||
less (0.67 searches and 70 recall tokens per task, against 1.50 and 120 with the cue
|
||||
absent).
|
||||
|
||||
A size is visible but is not yet quotable. On the four tasks whose absent-cue arm stayed
|
||||
out of a loop, the index cost 64,952 prompt tokens against 115,551 without, 44% less
|
||||
(`i106-budgetseal` -35,147, `i70-ingestfloor` -8,222, `i37-dispatchlag` -7,599, and
|
||||
`i112-lagquota` +369, the prefix rent a task that never searches pays). That is a
|
||||
**post-hoc subset**: `i118-fencelag` and `i47-tenantlease` are excluded because their
|
||||
absent-cue arm looped in this batch (five and three retrieval rounds, three escapes each;
|
||||
together 80% of the absent-cue total), and the qualification above does not screen for
|
||||
stopping behaviour — holding on both boundary arms across three passes shows a stable cue
|
||||
boundary, not a stopping arm. Quoting 44% as the index's effect would mean choosing the
|
||||
sample after seeing the result; a batch screened on stopping stability is what would make
|
||||
a size quotable, and one arm also reverses either way (`i70-ingestfloor` default, +11,135
|
||||
prompt tokens over three escapes).
|
||||
|
||||
Batch facts, because the counts differ: 36 runs were executed (12 paired, 24 index), and
|
||||
the token totals here cover the 24-run index batch — the paired batch ran before the
|
||||
per-run usage line existed.
|
||||
|
||||
**Things that turned out not to be problems**, each after being measured rather
|
||||
than argued about:
|
||||
|
||||
| Suspected | Measured |
|
||||
| --- | --- |
|
||||
| Model writes poor queries (2.17 searches/task) | 22/23 found the target on query 1; the extra searches are one round's parallel fan-out |
|
||||
| Model reaches for the workspace before memory (7/11) | Linear event order misread; by model round it is MemoryFirst 4/6 |
|
||||
| Snippets too short for multi-value answers | 240 runes covers 12/12 tasks at 100%; records are 30-179 runes |
|
||||
| A general stopping failure | 37/44 runs issued no search after the answer was in hand; all 7 that did belong to 2 tasks |
|
||||
|
||||
## Measurement distinctions this bench had to learn
|
||||
|
||||
Each of these reversed a conclusion once. They are enforced in the schema now,
|
||||
not left to whoever reads the numbers next.
|
||||
|
||||
- **TargetHit is not EvidenceSufficient.** A rank-1 hit can return a window too
|
||||
short to answer with. `FirstHitCoverage` records how much of the scored answer
|
||||
the first hit actually handed over.
|
||||
- **Linear event order is not model-round order.** Two calls in one round are a
|
||||
parallel fan-out, not a preference. Routing compares rounds.
|
||||
- **An isolated workdir is not an isolated host.** The sandbox mounts the host
|
||||
read-only by design. Three leaks were found and closed: the corpus source, the
|
||||
fixture transcript, and its event log sidecar.
|
||||
- **A count is not its evidence.** Query text, snippet contents and trajectories
|
||||
are all persisted now, because three analyses in a row needed a paid re-run to
|
||||
ask a question of data already collected.
|
||||
|
||||
## How a run stays honest
|
||||
|
||||
The corpus holds templates, never answers: every scored literal is a `{{var}}`
|
||||
instantiated per run, so grepping this directory reveals the question and not
|
||||
the answer. `TestNoAnswerLiteralExistsInTheRepository` asserts it with
|
||||
`git grep`. Preflight rebuilds every arm and checks the answer is unreadable in
|
||||
the real `provider.Request`, the probe query ranks the target within five, and
|
||||
an index task's cue sits at exactly the scales its tier names. The fixture is
|
||||
removed from disk once the agent holds it in memory, and the directory is
|
||||
scanned for answer literals before the first provider request and again after
|
||||
the turn. Any answer reaching the model through a tool that is not `recall`
|
||||
marks the run contaminated and takes it out of every statistic.
|
||||
|
||||
## Index corpus v2
|
||||
|
||||
The efficiency substrate is six tasks that hold on both arms: `i37-dispatchlag`
|
||||
and `i118-fencelag` (quarter), `i106-budgetseal` and `i70-ingestfloor` (half),
|
||||
`i47-tenantlease` and `i112-lagquota` (default).
|
||||
|
||||
They came out of five batches of candidates, each screened the same way: one
|
||||
`-mode=run-index` pass over the batch, then two more only for the candidates that
|
||||
were clean on both boundary arms, and only the candidates that held on both arms
|
||||
in all three passes stayed. A candidate that does not hold is dropped, not
|
||||
promoted. The numbers are candidates, per tier:
|
||||
|
||||
| batch | quarter | half | default |
|
||||
| --- | --- | --- | --- |
|
||||
| first pool, `i31`-`i54` | 1 of 8 | 0 of 8 | 2 of 8 |
|
||||
| second pool, `i61`-`i84` | 1 of 4 screened | 4 of 8 | 0 of 8 |
|
||||
| third pool, `i85`-`i108` | 2 of 8 | 8 of 16 | none authored |
|
||||
| replacement pool, `i109`-`i116` | - | - | 4 of 8 |
|
||||
| replacement pool, `i92`, `i117`-`i124` | 3 of 9 | - | - |
|
||||
|
||||
That table counts candidates that held on both arms in all three passes, which is
|
||||
not the size of the substrate. 89 candidates were authored, 77 were screened (the
|
||||
second pool's quarter tier stopped at 4 of 8 and its default tier was never run),
|
||||
and 25 held: 7 quarter, 12 half, 6 default. The substrate keeps two per tier, so
|
||||
the other 19 survivors are spares rather than corpus members. Candidates from the
|
||||
earlier round are not counted here.
|
||||
|
||||
| task | tier | cue present | cue absent |
|
||||
| --- | --- | ---: | ---: |
|
||||
| `i37-dispatchlag` | quarter | 3/3 | 3/3 |
|
||||
| `i118-fencelag` | quarter | 3/3 | 3/3 |
|
||||
| `i106-budgetseal` | half | 3/3 | 3/3 |
|
||||
| `i70-ingestfloor` | half | 3/3 | 3/3 |
|
||||
| `i47-tenantlease` | default | 3/3 | 3/3 |
|
||||
| `i112-lagquota` | default | 3/3 | 3/3 |
|
||||
|
||||
Across their 36 judged cells: `SnippetStop` 23, `SnippetThenRead` 10 and
|
||||
`DefensiveRead` 3 - every cell clean, nothing post-sufficient and nothing
|
||||
`NeverSufficient`.
|
||||
|
||||
Four of the first six were flagged in review for sharing answer values, probe
|
||||
queries and prompt wording within a tier. Separating them and re-screening those
|
||||
four changed two of them: `i37` and `i47` held, `i62` and `i51` did not, so they
|
||||
were replaced - `i90-recoverylag` was tried first and did not hold either, and
|
||||
`i118-fencelag` and `i112-lagquota` are what did. The separation was not
|
||||
cosmetic: the shared wording had been carrying two of the six.
|
||||
|
||||
Nothing in the task shape separates the survivors from the rest: within the half
|
||||
tier, plain-number answers survived 0 of 8 in the first pool and 4 of 8 in the
|
||||
second, and codename answers survived 8 of 16 in the third. An earlier claim here
|
||||
that the answer kind decides whether a candidate holds does not survive this
|
||||
table and is withdrawn. The dirty class was `PostSufficientRetrieval` in most
|
||||
failures - the model re-checking a value it already had - but which tasks hit it
|
||||
was not predictable from their shape at this sample.
|
||||
|
||||
`i01`-`i06` are the frozen study's six tasks, kept as `roleStopping`. Measured
|
||||
once each with `-mode=run-index -task <id>`, per task and per arm:
|
||||
|
||||
| task | tier | cue present | cue absent |
|
||||
| --- | --- | --- | --- |
|
||||
| `i01-transport-fallback` | quarter | SnippetStop | DefensiveRead |
|
||||
| `i02-scheduler-handoff` | quarter | PostSufficientRetrieval | PostSufficientRetrieval |
|
||||
| `i03-coalescing` | half | PostSufficientRetrieval | PostSufficientRunaway |
|
||||
| `i04-recovery-fence` | half | DefensiveRead | PostSufficientRunaway |
|
||||
| `i05-cache-probe` | default | SnippetThenRead | PostSufficientRetrieval |
|
||||
| `i06-dispatch-handoff` | default | SnippetStop | PostSufficientRetrieval |
|
||||
|
||||
Five of the six show post-sufficient behaviour in at least one arm (`i02`-`i06`),
|
||||
and `i01` is clean on both. The cue-present arm is clean for four of them
|
||||
(`i01`, `i04`, `i05`, `i06`), which is why they stay in the corpus as the
|
||||
stopping side rather than being retitled. All six are excluded from `run-index`
|
||||
and `run-boundary` batches and stay measurable one at a time with `-task <id>`. A
|
||||
task added to the corpus declares a role, so it cannot arrive without one.
|
||||
|
||||
A task qualifies only if it is clean on both of its boundary arms - the
|
||||
cue-present run and the cue-absent run - in every run of a screening batch. Clean
|
||||
means a `stopping_class` other than `PostSufficientRetrieval` or
|
||||
`PostSufficientRunaway`: `SnippetStop`, `SnippetThenRead` and `DefensiveRead`
|
||||
all qualify, and a defensive read is deliberately not counted as waste. The
|
||||
class comes from the three counts in that report — `searches_after_sufficient`,
|
||||
`reads_after_sufficient` and `escapes_after_sufficient` — together with how
|
||||
sufficiency was reached and how many rounds followed it. The remaining
|
||||
requirements still apply — successful tool call, answer in the output rather
|
||||
than the cue, rank 1, `FirstHitCoverage` complete, one or two simple values,
|
||||
nothing in the workspace.
|
||||
|
||||
`-mode=calibrate` is not a screening tool: it builds and reads back the
|
||||
fixtures, and reads nothing else, so its cue-visibility profile is the same for
|
||||
every task.
|
||||
|
||||
Then repeats rather than levels: cue-present against cue-absent on each task's
|
||||
own boundary, three runs per cell. What is unknown is the variance, and four
|
||||
budgets sampled once each cannot show it.
|
||||
@@ -0,0 +1,86 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// runBlockedBatch runs one dry index batch where every arm's runOne fails
|
||||
// before any model turn: the task's own directory is a regular file, so the
|
||||
// fixture cannot be built. In production the same early return happens when the
|
||||
// fixture build, the session load, or the answer-on-disk guard fails.
|
||||
func runBlockedBatch(t *testing.T) string {
|
||||
t.Helper()
|
||||
tk := indexTasks()[0]
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, tk.ID), []byte("not a directory\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code := runExperiment("run-index", experimentIndex, root, true, []contextTask{tk}); code != 0 {
|
||||
t.Fatalf("runExperiment exit = %d, want 0 (every run failed, but the batch finished)", code)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func readResultLines(t *testing.T, root string) ([]contextMetrics, []string) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(root, "results.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var runs []contextMetrics
|
||||
var lines []string
|
||||
for text := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") {
|
||||
var m contextMetrics
|
||||
if err := json.Unmarshal([]byte(text), &m); err != nil {
|
||||
t.Fatalf("results.jsonl: %v", err)
|
||||
}
|
||||
runs = append(runs, m)
|
||||
lines = append(lines, text)
|
||||
}
|
||||
return runs, lines
|
||||
}
|
||||
|
||||
// A run that failed before the model turn produced no measurement. Recording it
|
||||
// as a zero-valued contextMetrics puts it in the funnel's Scored denominator,
|
||||
// in "Where it broke", and in the dose curve as a real zero.
|
||||
func TestBatchDoesNotRecordAFailedRunAsAResult(t *testing.T) {
|
||||
root := runBlockedBatch(t)
|
||||
runs, lines := readResultLines(t, root)
|
||||
if len(runs) == 0 {
|
||||
t.Fatal("no runs recorded at all")
|
||||
}
|
||||
for i, m := range runs {
|
||||
if m.Task == "" || m.Arm == "" || m.FailureStage == "" {
|
||||
t.Fatalf("results.jsonl line %d records a run that never ran as a result "+
|
||||
"(task=%q arm=%q failure_stage=%q):\n %s", i+1, m.Task, m.Arm, m.FailureStage, lines[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The manifest is the artefact that says which runs counted, so a failed run
|
||||
// must not be added to CompletedRuns or to the task id list.
|
||||
func TestBatchManifestDoesNotReportFailedRunsAsCompleted(t *testing.T) {
|
||||
root := runBlockedBatch(t)
|
||||
var manifest batchManifest
|
||||
data, err := os.ReadFile(filepath.Join(root, "batch.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range manifest.TaskIDs {
|
||||
if id == "" {
|
||||
t.Fatalf("manifest lists an empty task id: task_ids=%q (completed=%d planned=%d)",
|
||||
manifest.TaskIDs, manifest.CompletedRuns, manifest.PlannedRuns)
|
||||
}
|
||||
}
|
||||
if manifest.CompletedRuns == manifest.PlannedRuns {
|
||||
t.Fatalf("manifest reports every planned run completed although none produced evidence: "+
|
||||
"completed=%d planned=%d task_ids=%q", manifest.CompletedRuns, manifest.PlannedRuns, manifest.TaskIDs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// batchInfo is what the mode itself knows before any run happens. Everything
|
||||
// else in the manifest is derived from the runs that were written, so a figure
|
||||
// in it cannot drift from the artifact beside it.
|
||||
type batchInfo struct {
|
||||
Mode string
|
||||
Planned int
|
||||
Started time.Time
|
||||
Dry bool
|
||||
// Repo is the checkout the batch ran from, captured before the harness
|
||||
// isolates its workspace: git cannot describe a directory that is gone.
|
||||
Repo string
|
||||
}
|
||||
|
||||
// batchManifest is written beside results.jsonl so a later reader does not have
|
||||
// to reconstruct what the batch was from a terminal scrollback.
|
||||
type batchManifest struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Mode string `json:"mode"`
|
||||
StartedAt string `json:"started_at"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
GitCommit string `json:"git_commit"`
|
||||
GitDirty *bool `json:"git_dirty,omitempty"`
|
||||
Model string `json:"model"`
|
||||
Window int `json:"window"`
|
||||
Generations int `json:"generations"`
|
||||
TaskIDs []string `json:"task_ids"`
|
||||
Arms []string `json:"arms"`
|
||||
PlannedRuns int `json:"planned_runs"`
|
||||
CompletedRuns int `json:"completed_runs"`
|
||||
// FailedRuns counts the planned runs that ended before a transcript existed.
|
||||
// They are rows in results.jsonl, never part of CompletedRuns, and the field
|
||||
// is absent when there were none so an intact batch is unchanged.
|
||||
FailedRuns int `json:"failed_runs,omitempty"`
|
||||
// Usage totals are absent when nothing reported any: an unmeasured cost must
|
||||
// not read as a measured zero. UsageReportedRuns stays, because zero there
|
||||
// is a fact about the batch.
|
||||
UsageReportedRuns int `json:"usage_reported_runs"`
|
||||
UsageReportedRequests int `json:"usage_reported_requests,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
CacheHitTokens int `json:"cache_hit_tokens,omitempty"`
|
||||
CacheMissTokens int `json:"cache_miss_tokens,omitempty"`
|
||||
UsageEstimated bool `json:"usage_estimated,omitempty"`
|
||||
ResultsSHA256 string `json:"results_sha256"`
|
||||
TrajectoriesSHA256 string `json:"trajectories_sha256"`
|
||||
}
|
||||
|
||||
// gitCommitAndDirty reports the checkout the batch ran from. Unknown is written
|
||||
// rather than omitted, so an absent field is never mistaken for a clean tree.
|
||||
func gitCommitAndDirty(repo string) (commit string, dirty *bool) {
|
||||
commit = "unknown"
|
||||
if repo == "" {
|
||||
return commit, nil
|
||||
}
|
||||
in := func(args ...string) ([]byte, error) {
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = repo
|
||||
return cmd.Output()
|
||||
}
|
||||
if out, err := in("rev-parse", "--short", "HEAD"); err == nil {
|
||||
commit = strings.TrimSpace(string(out))
|
||||
}
|
||||
if out, err := in("status", "--porcelain"); err == nil {
|
||||
state := strings.TrimSpace(string(out)) != ""
|
||||
dirty = &state
|
||||
}
|
||||
return commit, dirty
|
||||
}
|
||||
|
||||
func fileSHA256(path string) (string, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%x", sha256.Sum256(data)), nil
|
||||
}
|
||||
|
||||
func sortedKeys(set map[string]bool) []string {
|
||||
out := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// writeBatchManifest records the batch beside the runs it describes. It runs
|
||||
// last, so its digests are of what is already on disk.
|
||||
func writeBatchManifest(root string, info batchInfo, all []contextMetrics) error {
|
||||
ids, arms := map[string]bool{}, map[string]bool{}
|
||||
var prompt, completion, cacheHit, cacheMiss, requests, reported, failed int
|
||||
estimated := false
|
||||
for _, m := range all {
|
||||
// A failed run has no task, arm or evidence to contribute: it is counted
|
||||
// as a failure below and is not part of what the batch measured.
|
||||
if m.Failed {
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
ids[m.Task] = true
|
||||
arms[m.Arm] = true
|
||||
if m.UsageReportedRequests > 0 {
|
||||
reported++
|
||||
requests += m.UsageReportedRequests
|
||||
prompt += m.PromptTokens
|
||||
completion += m.CompletionTokens
|
||||
cacheHit += m.CacheHitTokens
|
||||
cacheMiss += m.CacheMissTokens
|
||||
estimated = estimated || m.UsageEstimated
|
||||
}
|
||||
}
|
||||
commit, dirty := gitCommitAndDirty(info.Repo)
|
||||
model := runModel
|
||||
if info.Dry {
|
||||
model = "scripted"
|
||||
}
|
||||
manifest := batchManifest{
|
||||
SchemaVersion: 1, Mode: info.Mode,
|
||||
StartedAt: info.Started.UTC().Format(time.RFC3339),
|
||||
FinishedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
GitCommit: commit, GitDirty: dirty,
|
||||
Model: model, Window: fixtureWindow, Generations: fixtureGenerations,
|
||||
TaskIDs: sortedKeys(ids), Arms: sortedKeys(arms),
|
||||
PlannedRuns: info.Planned, CompletedRuns: len(all) - failed, FailedRuns: failed,
|
||||
UsageReportedRuns: reported, UsageReportedRequests: requests,
|
||||
PromptTokens: prompt, CompletionTokens: completion,
|
||||
CacheHitTokens: cacheHit, CacheMissTokens: cacheMiss, UsageEstimated: estimated,
|
||||
}
|
||||
var err error
|
||||
if manifest.ResultsSHA256, err = fileSHA256(filepath.Join(root, "results.jsonl")); err != nil {
|
||||
return err
|
||||
}
|
||||
if manifest.TrajectoriesSHA256, err = fileSHA256(filepath.Join(root, "trajectories.jsonl")); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(filepath.Join(root, "batch.json"), append(data, '\n'), 0o644)
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func seedArtifacts(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
for _, name := range []string{"results.jsonl", "trajectories.jsonl"} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte("{}\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readManifest(t *testing.T, root string) (batchManifest, string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(root, "batch.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got batchManifest
|
||||
if err := json.Unmarshal(data, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got, string(data)
|
||||
}
|
||||
|
||||
func TestBatchManifestRecordsWhatTheBatchWas(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
seedArtifacts(t, root)
|
||||
all := []contextMetrics{
|
||||
{Task: "b", Arm: "index-off"},
|
||||
{Task: "a", Arm: "index-off"},
|
||||
{Task: "a", Arm: "index-half", UsageReportedRequests: 2, PromptTokens: 100,
|
||||
CompletionTokens: 10, CacheHitTokens: 60, CacheMissTokens: 40},
|
||||
}
|
||||
if err := writeBatchManifest(root, batchInfo{Mode: "run-index", Planned: 4, Started: time.Now().Add(-time.Minute)}, all); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := readManifest(t, root)
|
||||
if got.SchemaVersion != 1 || got.Mode != "run-index" || got.PlannedRuns != 4 || got.CompletedRuns != 3 {
|
||||
t.Fatalf("manifest header = %+v", got)
|
||||
}
|
||||
if strings.Join(got.TaskIDs, ",") != "a,b" || strings.Join(got.Arms, ",") != "index-half,index-off" {
|
||||
t.Fatalf("ids/arms = %v / %v, want sorted and unique", got.TaskIDs, got.Arms)
|
||||
}
|
||||
if got.UsageReportedRuns != 1 || got.UsageReportedRequests != 2 || got.PromptTokens != 100 ||
|
||||
got.CompletionTokens != 10 || got.CacheHitTokens != 60 || got.CacheMissTokens != 40 {
|
||||
t.Fatalf("usage totals = %+v", got)
|
||||
}
|
||||
// No repo means git could not describe the checkout: unknown commit, and no
|
||||
// dirty flag at all, because "clean" is a claim this batch cannot make.
|
||||
if got.GitCommit != "unknown" || got.GitDirty != nil {
|
||||
t.Fatalf("git state = %q / %v, want unknown and no dirty flag", got.GitCommit, got.GitDirty)
|
||||
}
|
||||
want := fmt.Sprintf("%x", sha256.Sum256([]byte("{}\n")))
|
||||
if got.ResultsSHA256 != want || got.TrajectoriesSHA256 != want {
|
||||
t.Fatalf("digests = %q / %q, want %q", got.ResultsSHA256, got.TrajectoriesSHA256, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A dry batch pays nothing, so its cost fields must be absent rather than zero:
|
||||
// an unmeasured spend must not read as a measured one.
|
||||
func TestBatchManifestLeavesUnreportedUsageAbsent(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
seedArtifacts(t, root)
|
||||
all := []contextMetrics{{Task: "a", Arm: "index-off"}, {Task: "a", Arm: "index-half"}}
|
||||
if err := writeBatchManifest(root, batchInfo{Mode: "run-index", Planned: 2, Started: time.Now(), Dry: true}, all); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, raw := readManifest(t, root)
|
||||
if got.UsageReportedRuns != 0 {
|
||||
t.Fatalf("usage_reported_runs = %d, want 0 reported", got.UsageReportedRuns)
|
||||
}
|
||||
if got.Model != "scripted" {
|
||||
t.Fatalf("dry batch model = %q, want scripted", got.Model)
|
||||
}
|
||||
for _, absent := range []string{"usage_reported_requests", "prompt_tokens", "completion_tokens", "cache_hit_tokens"} {
|
||||
if strings.Contains(raw, absent) {
|
||||
t.Fatalf("a dry batch wrote %q:\n%s", absent, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// boundaryRuns builds the input reportBoundaries reads: one scored run for an
|
||||
// index task's cue side, and one failed run for another index task's cue side.
|
||||
// The runs are constructed because the report is a pure function of the slice;
|
||||
// the batch path that produces them is covered by the failed-batch tests.
|
||||
func boundaryRuns(t *testing.T) (all []contextMetrics, scoredTask, failedTask string) {
|
||||
t.Helper()
|
||||
index := indexTasks()
|
||||
if len(index) < 2 {
|
||||
t.Fatalf("need at least two index tasks, have %d", len(index))
|
||||
}
|
||||
scored, failed := index[0], index[1]
|
||||
cue, noCue := boundaryPair(scored.CueTier)
|
||||
if cue == "" || noCue == "" {
|
||||
t.Fatalf("task %s has an empty boundary pair (%q, %q)", scored.ID, cue, noCue)
|
||||
}
|
||||
failedCue, _ := boundaryPair(failed.CueTier)
|
||||
if failedCue == "" {
|
||||
t.Fatalf("task %s has no cue side", failed.ID)
|
||||
}
|
||||
all = []contextMetrics{
|
||||
{Task: scored.ID, Arm: "index-" + cue, AnswerRecovered: true, SearchCalls: 1, RecallReturnedTokens: 318},
|
||||
{Task: scored.ID, Arm: "index-" + noCue, AnswerRecovered: false},
|
||||
{Task: failed.ID, Arm: "index-" + failedCue, Failed: true, FailureStage: stageNoEvidence},
|
||||
}
|
||||
return all, scored.ID, failed.ID
|
||||
}
|
||||
|
||||
// TestBoundaryReportExcludesFailedRuns: a run that never produced a transcript
|
||||
// must not appear in the cue-boundary table, which was the last reporting exit
|
||||
// that still folded it in as a zero measurement.
|
||||
func TestBoundaryReportExcludesFailedRuns(t *testing.T) {
|
||||
all, scoredTask, failedTask := boundaryRuns(t)
|
||||
out := captureStdout(t, func() { reportBoundaries(all) })
|
||||
if strings.Contains(out, failedTask) {
|
||||
t.Fatalf("the boundary table printed the failed task %q as a measurement:\n%s", failedTask, out)
|
||||
}
|
||||
if !strings.Contains(out, scoredTask) {
|
||||
t.Fatalf("the boundary table dropped the scored task %q, so the exclusion is not specific to failures:\n%s", scoredTask, out)
|
||||
}
|
||||
if !strings.Contains(out, "cue-present") || !strings.Contains(out, "cue-absent") {
|
||||
t.Fatalf("the pooled comparison disappeared:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBoundaryReportSaysNothingToCompareWhenEveryRunFailed: with no scored run
|
||||
// the report must say so, rather than printing a comparison of zeros.
|
||||
func TestBoundaryReportSaysNothingToCompareWhenEveryRunFailed(t *testing.T) {
|
||||
all, _, _ := boundaryRuns(t)
|
||||
failedOnly := make([]contextMetrics, 0, len(all))
|
||||
for _, m := range all {
|
||||
m.Failed = true
|
||||
m.FailureStage = stageNoEvidence
|
||||
failedOnly = append(failedOnly, m)
|
||||
}
|
||||
out := captureStdout(t, func() { reportBoundaries(failedOnly) })
|
||||
if !strings.Contains(out, "scored=0") || !strings.Contains(out, "failed=") {
|
||||
t.Fatalf("an all-failed batch did not report scored=0 with a failed count:\n%s", out)
|
||||
}
|
||||
if strings.Contains(out, "cue-present") || strings.Contains(out, "cue-absent") {
|
||||
t.Fatalf("an all-failed batch still printed a pooled comparison of zeros:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// capture.go — what the agent actually hands a provider.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/assembly/boot"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/runtime/agent"
|
||||
_ "reasonix/internal/tools/builtin"
|
||||
)
|
||||
|
||||
// A projection is what the agent believes; a provider.Request is what the model
|
||||
// is told. Rendering, sanitization and tool-schema construction sit between
|
||||
// them, so the arm's contract is checked on the request.
|
||||
|
||||
type captureProvider struct {
|
||||
requests []provider.Request
|
||||
}
|
||||
|
||||
func (p *captureProvider) Name() string { return "contextbench-capture" }
|
||||
|
||||
// Stream records the request and answers with a clean terminal response. An
|
||||
// empty answer would buy an empty-final retry and a second request, which is
|
||||
// not the one round this is here to inspect.
|
||||
func (p *captureProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.requests = append(p.requests, cloneRequest(req))
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "contextbench preflight complete"}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func cloneRequest(req provider.Request) provider.Request {
|
||||
out := req
|
||||
out.Messages = append([]provider.Message(nil), req.Messages...)
|
||||
out.Tools = append([]provider.ToolSchema(nil), req.Tools...)
|
||||
return out
|
||||
}
|
||||
|
||||
// captureRequest rebuilds the session the way a resume does — transcript from
|
||||
// disk, projection from its sidecar, the shipped tool surface at this arm —
|
||||
// and returns the request the first round hands the provider.
|
||||
func captureRequest(f builtFixture, arm ablation.Set, prompt string) (provider.Request, error) {
|
||||
sess, err := sessionstore.LoadSession(f.Path)
|
||||
if err != nil {
|
||||
return provider.Request{}, fmt.Errorf("load session: %w", err)
|
||||
}
|
||||
reg := tool.NewRegistry()
|
||||
for _, t := range tool.Builtins() {
|
||||
reg.Add(t)
|
||||
}
|
||||
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
|
||||
|
||||
cap := &captureProvider{}
|
||||
a := agent.New(cap, reg, sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, event.Discard)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
|
||||
if err := a.Run(context.Background(), prompt); err != nil {
|
||||
return provider.Request{}, fmt.Errorf("run: %w", err)
|
||||
}
|
||||
// The turn's own request is the one carrying tools. A fold on the way in
|
||||
// asks the same provider for a digest, and that request has none — taking
|
||||
// the first request blindly would inspect the summarizer's prompt.
|
||||
for _, req := range cap.requests {
|
||||
if len(req.Tools) > 0 {
|
||||
return req, nil
|
||||
}
|
||||
}
|
||||
return provider.Request{}, fmt.Errorf("no request carried tools (%d captured)", len(cap.requests))
|
||||
}
|
||||
|
||||
// surfaceField is one place a model could read a string, named so a leak
|
||||
// report says where rather than only that.
|
||||
type surfaceField struct {
|
||||
Path string
|
||||
Text string
|
||||
}
|
||||
|
||||
// providerSurface enumerates every provider-visible string in a request.
|
||||
// Enumerated rather than marshalled whole: a Message also carries local-only
|
||||
// fields, and scanning those would fail on bytes no model ever sees.
|
||||
func providerSurface(req provider.Request) []surfaceField {
|
||||
var out []surfaceField
|
||||
add := func(path, text string) {
|
||||
if strings.TrimSpace(text) != "" {
|
||||
out = append(out, surfaceField{Path: path, Text: text})
|
||||
}
|
||||
}
|
||||
for i, m := range req.Messages {
|
||||
add(fmt.Sprintf("messages[%d].content", i), m.Content)
|
||||
add(fmt.Sprintf("messages[%d].reasoning_content", i), m.ReasoningContent)
|
||||
add(fmt.Sprintf("messages[%d].name", i), m.Name)
|
||||
for j, tc := range m.ToolCalls {
|
||||
add(fmt.Sprintf("messages[%d].tool_calls[%d].name", i, j), tc.Name)
|
||||
add(fmt.Sprintf("messages[%d].tool_calls[%d].arguments", i, j), tc.Arguments)
|
||||
}
|
||||
for j, item := range m.ResponsesItems {
|
||||
add(fmt.Sprintf("messages[%d].responses_items[%d]", i, j), string(item))
|
||||
}
|
||||
}
|
||||
for i, t := range req.Tools {
|
||||
add(fmt.Sprintf("tools[%d].name", i), t.Name)
|
||||
add(fmt.Sprintf("tools[%d].description", i), t.Description)
|
||||
if len(t.Parameters) > 0 {
|
||||
b, err := json.Marshal(t.Parameters)
|
||||
if err == nil {
|
||||
add(fmt.Sprintf("tools[%d].parameters", i), string(b))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// findInSurface reports every place a marker is readable.
|
||||
func findInSurface(req provider.Request, marker string) []string {
|
||||
var hits []string
|
||||
for _, f := range providerSurface(req) {
|
||||
if strings.Contains(f.Text, marker) {
|
||||
hits = append(hits, f.Path)
|
||||
}
|
||||
}
|
||||
return hits
|
||||
}
|
||||
|
||||
// recallToolSchema returns the recall tool as the provider sees it.
|
||||
func recallToolSchema(req provider.Request) (provider.ToolSchema, bool) {
|
||||
for _, t := range req.Tools {
|
||||
if t.Name == "recall" {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return provider.ToolSchema{}, false
|
||||
}
|
||||
|
||||
// localOnlyLeaks reports host-side fields that should never survive the
|
||||
// provider boundary. A hit here is a boundary regression, marker or no marker.
|
||||
func localOnlyLeaks(req provider.Request) []string {
|
||||
var out []string
|
||||
for i, m := range req.Messages {
|
||||
switch {
|
||||
case m.RawContent != "":
|
||||
out = append(out, fmt.Sprintf("messages[%d].raw_content", i))
|
||||
case m.LocalOnly:
|
||||
out = append(out, fmt.Sprintf("messages[%d].local_only", i))
|
||||
case m.DecisionReceipt != nil || len(m.DecisionReceipts) > 0:
|
||||
out = append(out, fmt.Sprintf("messages[%d].decision_receipt", i))
|
||||
case m.ToolExecution != nil:
|
||||
out = append(out, fmt.Sprintf("messages[%d].tool_execution", i))
|
||||
case len(m.MemoryCitations) > 0:
|
||||
out = append(out, fmt.Sprintf("messages[%d].memory_citations", i))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// searchWording is what a search-off arm must never be told. The arm exists to
|
||||
// reproduce the runtime before search; an arm offered a capability it does not
|
||||
// have is a different experiment.
|
||||
var searchWording = []string{"recall with a query", "search the whole folded region", "query still finds"}
|
||||
@@ -0,0 +1,671 @@
|
||||
// contextbench.go — what the model did with the affordance, read off structure.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/tokencount"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// Every judgement here reads a tool call's name and its JSON arguments, or a
|
||||
// canonical position, and never the model's prose. "I will search the history"
|
||||
// is not a search; a recall call with a query is.
|
||||
|
||||
// failure stages, in funnel order. Exactly one is reported per run, except for
|
||||
// NoEvidence: that stage belongs to a run with no measurement at all.
|
||||
const (
|
||||
stageReadMissed = "NoSearch" // never asked, and never read the target either
|
||||
stageSearchMiss = "SearchMiss" // searched, target never came back
|
||||
stageHitNotRead = "HitNotRead" // target was returned and never opened
|
||||
stageAnswerWrong = "ReadButAnswerWrong" // opened it and still answered wrong
|
||||
stageRecovered = "Recovered"
|
||||
stageContaminated = "Contaminated"
|
||||
stageNoEvidence = "NoEvidence"
|
||||
|
||||
// Routing is judged by model round, not by the linear order of tool events:
|
||||
// two calls in one round are one parallel fan-out, and calling that
|
||||
// "workspace first" would report a choice the model never made.
|
||||
routeMemoryFirst = "MemoryFirst"
|
||||
routeWorkspaceFirst = "WorkspaceFirst"
|
||||
routeParallel = "Parallel"
|
||||
routeNeither = "Neither"
|
||||
stageCueRead = "DirectRead" // read the target from an index address, no search needed
|
||||
stageNoRetrieval = "NoRetrieval" // answered without touching recall at all
|
||||
toolRecall = "recall"
|
||||
)
|
||||
|
||||
// searchAttempt is one query the model wrote, and what it got back. Kept per
|
||||
// attempt rather than aggregated: the question is why a retrieval that the host
|
||||
// probe ranks first takes the model more than one try.
|
||||
type searchAttempt struct {
|
||||
Ordinal int `json:"ordinal"`
|
||||
Round int `json:"round"`
|
||||
Query string `json:"query"`
|
||||
TargetRank int `json:"target_rank"` // 0 = the target did not come back
|
||||
ResultCount int `json:"result_count"`
|
||||
// Reformulation classifies this query against the one before it, and
|
||||
// MissingProbeTerms are the host probe's terms this query left out. Both
|
||||
// are derived from token sets: no judge, and nothing read out of prose.
|
||||
Reformulation string `json:"reformulation,omitempty"`
|
||||
MissingProbeTerms []string `json:"missing_probe_terms,omitempty"`
|
||||
}
|
||||
|
||||
// contextMetrics is one run's behaviour.
|
||||
type contextMetrics struct {
|
||||
Task string `json:"task"`
|
||||
Arm string `json:"arm"`
|
||||
|
||||
SearchCalls int `json:"search_calls"`
|
||||
// SearchHits counts searches that returned anything; TargetSearchHits
|
||||
// counts searches that returned the target. The gap is query quality.
|
||||
SearchHits int `json:"search_hits"`
|
||||
TargetSearchHits int `json:"target_search_hits"`
|
||||
FirstTargetRank int `json:"first_target_rank"` // 0 = never returned
|
||||
// FirstHitCoverage: how much of the answer the first hit handed over.
|
||||
// rank=1 with coverage 1/3 is a window too short to answer with, not a
|
||||
// model that would not stop.
|
||||
FirstHitCoverage int `json:"first_hit_coverage"`
|
||||
AnswerValues int `json:"answer_values"`
|
||||
|
||||
ReadCalls int `json:"read_calls"`
|
||||
TargetRead bool `json:"target_read"`
|
||||
ReadAfterHit bool `json:"read_after_hit"`
|
||||
// RecallReadWithoutSearch: the index addressed the target, so no search was
|
||||
// needed. Not "direct read" — read_file is a read too, and in an audit the
|
||||
// two mean opposite things.
|
||||
RecallReadWithoutSearch bool `json:"recall_read_without_search"`
|
||||
SearchThenRecallRead bool `json:"search_then_recall_read"`
|
||||
|
||||
FirstSearchRound int `json:"first_search_round"`
|
||||
FirstReadRound int `json:"first_read_round"`
|
||||
RetrievalRounds int `json:"retrieval_rounds"`
|
||||
|
||||
// RecallReturnedTokens is what recall paged back in. The index saves
|
||||
// resident prefix; this is what is paid dynamically instead.
|
||||
RecallReturnedTokens int `json:"recall_returned_tokens"`
|
||||
|
||||
// Provider usage as reported on the stream, or empty for a dry run. The count
|
||||
// is completions that reported usage, so it is not a count of attempts.
|
||||
UsageReportedRequests int `json:"usage_reported_requests,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
CacheHitTokens int `json:"cache_hit_tokens,omitempty"`
|
||||
CacheMissTokens int `json:"cache_miss_tokens,omitempty"`
|
||||
UsageEstimated bool `json:"usage_estimated,omitempty"`
|
||||
|
||||
Searches []searchAttempt `json:"searches,omitempty"`
|
||||
// PostHitSearches are queries issued after the target had already been
|
||||
// returned: not a query problem, a result-to-read problem.
|
||||
PostHitSearches int `json:"post_hit_searches"`
|
||||
|
||||
AnswerRecovered bool `json:"answer_recovered"`
|
||||
// AnswerAmbiguous marks an answer that offered more than one candidate for
|
||||
// a value. Markers matching proves the right fact appears; it does not
|
||||
// prove it was submitted as the answer. Recorded, never acted on.
|
||||
AnswerAmbiguous bool `json:"answer_ambiguous,omitempty"`
|
||||
AmbiguousVar string `json:"ambiguous_var,omitempty"`
|
||||
MissingMarkers []string `json:"missing_markers,omitempty"`
|
||||
// FinalAnswer is kept bounded so a surprising run can be read back. A
|
||||
// metric nobody can audit is a claim, not a measurement.
|
||||
FinalAnswer string `json:"final_answer,omitempty"`
|
||||
FailureStage string `json:"failure_stage"`
|
||||
TimeoutReason string `json:"timeout_reason,omitempty"`
|
||||
|
||||
// Failed marks a run that ended before a transcript existed: it has no score
|
||||
// and no place in the statistics. The row stays in results.jsonl with its
|
||||
// task, arm and stage, so a failure is counted rather than dropped.
|
||||
Failed bool `json:"failed,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
|
||||
// UnexpectedWorkTools are investigation tools used instead of recall.
|
||||
// Reaching for the workspace is behaviour, not noise: how a model routes
|
||||
// its uncertainty when memory is missing is part of what is measured.
|
||||
UnexpectedWorkTools []string `json:"unexpected_work_tools,omitempty"`
|
||||
EscapeCalls int `json:"escape_calls"`
|
||||
EscapeBeforeFirstRecall int `json:"escape_before_first_recall"`
|
||||
EscapeAfterFirstRecall int `json:"escape_after_first_recall"`
|
||||
// Contaminated: the answer was reachable through something other than
|
||||
// recall. Asserting this beats trusting the isolation, and a run that trips
|
||||
// it leaves the statistics rather than inflating one.
|
||||
Contaminated bool `json:"contaminated,omitempty"`
|
||||
LeakedVia string `json:"leaked_via,omitempty"`
|
||||
LeakedMarker string `json:"leaked_marker,omitempty"`
|
||||
LeakedArgs string `json:"leaked_args,omitempty"`
|
||||
LeakedOutput string `json:"leaked_output,omitempty"`
|
||||
// EscapeFoundAnswer separates "looked at the workspace" from "found the
|
||||
// answer there": different facts about the same call.
|
||||
EscapeFoundAnswer bool `json:"escape_found_answer,omitempty"`
|
||||
|
||||
CueVisible bool `json:"cue_visible"`
|
||||
|
||||
// trajectory is kept out of the metric line and written beside it, so a
|
||||
// future question can be asked of this batch instead of a new one.
|
||||
trajectory []provider.Message
|
||||
|
||||
// CueDirectRead: the cue was addressed, the first memory action was a
|
||||
// positions read, and it covered the target. Crediting any positions call
|
||||
// would hand the index a search's work.
|
||||
CueDirectRead bool `json:"cue_direct_read"`
|
||||
|
||||
// Sufficiency, not hit. A search that returns the target at rank 1 has not
|
||||
// necessarily returned the answer: the snippet is bounded. The stopping
|
||||
// boundary is the round where everything needed was readable.
|
||||
FirstSufficientRound int `json:"first_sufficient_round"`
|
||||
SufficientVia string `json:"sufficient_via"` // snippet | read | never
|
||||
RoundsAfterSufficient int `json:"rounds_after_sufficient"`
|
||||
SearchesAfterSufficient int `json:"searches_after_sufficient"`
|
||||
ReadsAfterSufficient int `json:"reads_after_sufficient"`
|
||||
EscapesAfterSufficient int `json:"escapes_after_sufficient"`
|
||||
RecallTokensAfterSufficient int `json:"recall_tokens_after_sufficient"`
|
||||
StoppingClass string `json:"stopping_class"`
|
||||
|
||||
// What happened after the answer was already in hand. A timeout that spent
|
||||
// seven rounds past its target is a stopping problem, and extending the
|
||||
// budget would only buy the loop more time.
|
||||
TargetFirstSeenRound int `json:"target_first_seen_round"`
|
||||
RoundsAfterTarget int `json:"rounds_after_target"`
|
||||
SearchesAfterTarget int `json:"searches_after_target"`
|
||||
RecallTokensAfterTarget int `json:"recall_tokens_after_target"`
|
||||
TargetHeldAtEnd bool `json:"target_held_at_end"`
|
||||
|
||||
// Routing is where the model looked first when its memory was missing.
|
||||
Routing string `json:"routing"`
|
||||
FirstRecallRound int `json:"first_recall_round"`
|
||||
FirstEscapeRound int `json:"first_escape_round"`
|
||||
}
|
||||
|
||||
// hitPosition matches the address recall prints for a search hit. Reading our
|
||||
// own rendered format, not guessing at the model's words.
|
||||
var hitPosition = regexp.MustCompile(`(?m)^#(\d+) `)
|
||||
|
||||
// scoreRun reads the messages one turn appended and reports what happened.
|
||||
// runScorer walks one turn's messages, holding the state a judgement needs:
|
||||
// which calls are open, whether the target has come back, and which search
|
||||
// each result belongs to.
|
||||
type runScorer struct {
|
||||
m contextMetrics
|
||||
inst fixtureInstance
|
||||
target int
|
||||
calls map[string]provider.ToolCall
|
||||
pending map[string]int
|
||||
round int
|
||||
sawTargetHit bool
|
||||
// seen accumulates everything recall has shown the model, so sufficiency is
|
||||
// judged on what it could read rather than on which call produced it.
|
||||
seen strings.Builder
|
||||
sufficient bool
|
||||
}
|
||||
|
||||
// scoreRun reads the messages one turn appended and reports what happened.
|
||||
func scoreRun(msgs []provider.Message, inst fixtureInstance, target int, arm string, cueVisible bool) contextMetrics {
|
||||
s := &runScorer{
|
||||
m: contextMetrics{Task: inst.Task.ID, Arm: arm, CueVisible: cueVisible},
|
||||
inst: inst, target: target,
|
||||
calls: map[string]provider.ToolCall{}, pending: map[string]int{},
|
||||
}
|
||||
for _, msg := range msgs {
|
||||
if len(msg.ToolCalls) > 0 {
|
||||
s.round++
|
||||
}
|
||||
for _, tc := range msg.ToolCalls {
|
||||
s.calls[tc.ID] = tc
|
||||
s.observeCall(tc)
|
||||
}
|
||||
if msg.Role == provider.RoleTool {
|
||||
s.observeResult(msg)
|
||||
}
|
||||
}
|
||||
return s.finish(cueVisible)
|
||||
}
|
||||
|
||||
// observeCall records one tool call: a recall search, a recall read, or a
|
||||
// reach for the workspace.
|
||||
func (s *runScorer) observeCall(tc provider.ToolCall) {
|
||||
m := &s.m
|
||||
if tc.Name != toolRecall {
|
||||
if !containsString(m.UnexpectedWorkTools, tc.Name) {
|
||||
m.UnexpectedWorkTools = append(m.UnexpectedWorkTools, tc.Name)
|
||||
}
|
||||
m.EscapeCalls++
|
||||
if s.sufficient {
|
||||
m.EscapesAfterSufficient++
|
||||
}
|
||||
if m.FirstEscapeRound == 0 {
|
||||
m.FirstEscapeRound = s.round
|
||||
}
|
||||
if m.SearchCalls+m.ReadCalls == 0 {
|
||||
m.EscapeBeforeFirstRecall++
|
||||
} else {
|
||||
m.EscapeAfterFirstRecall++
|
||||
}
|
||||
return
|
||||
}
|
||||
if m.FirstRecallRound == 0 {
|
||||
m.FirstRecallRound = s.round
|
||||
}
|
||||
query, positions := recallArgs(tc.Arguments)
|
||||
switch {
|
||||
case query != "":
|
||||
m.SearchCalls++
|
||||
if m.FirstSearchRound == 0 {
|
||||
m.FirstSearchRound = s.round
|
||||
}
|
||||
if s.sawTargetHit {
|
||||
m.PostHitSearches++
|
||||
m.SearchesAfterTarget++
|
||||
}
|
||||
if s.sufficient {
|
||||
m.SearchesAfterSufficient++
|
||||
}
|
||||
m.Searches = append(m.Searches, searchAttempt{Ordinal: m.SearchCalls, Round: s.round, Query: query})
|
||||
s.pending[tc.ID] = len(m.Searches) - 1
|
||||
case len(positions) > 0:
|
||||
m.ReadCalls++
|
||||
if s.sufficient {
|
||||
m.ReadsAfterSufficient++
|
||||
}
|
||||
if m.FirstReadRound == 0 {
|
||||
m.FirstReadRound = s.round
|
||||
}
|
||||
// A read covers the target when it names its position: recall returns
|
||||
// the tool results under the caller's own address.
|
||||
if containsInt(positions, s.target) {
|
||||
m.TargetRead = true
|
||||
switch {
|
||||
case s.sawTargetHit:
|
||||
m.ReadAfterHit, m.SearchThenRecallRead = true, true
|
||||
case m.SearchCalls == 0:
|
||||
m.RecallReadWithoutSearch = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// observeResult reads what a call returned: contamination, page-in cost, and
|
||||
// whether this search brought the target back.
|
||||
func (s *runScorer) observeResult(msg provider.Message) {
|
||||
m := &s.m
|
||||
call, ok := s.calls[msg.ToolCallID]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if call.Name != toolRecall {
|
||||
// Codenames only: a three-digit epoch appears in any long file by
|
||||
// chance, and one did. A model holding the codename holds the numbers
|
||||
// beside it anyway.
|
||||
for _, marker := range s.inst.AnswerMarkers {
|
||||
if len(marker) >= 6 && strings.Contains(msg.Content, marker) {
|
||||
m.Contaminated, m.LeakedVia, m.LeakedMarker = true, call.Name, marker
|
||||
m.EscapeFoundAnswer = true
|
||||
m.LeakedArgs, m.LeakedOutput = truncate(call.Arguments, 200), truncate(msg.Content, 300)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
cost := tokencount.Text(msg.Content)
|
||||
m.RecallReturnedTokens += cost
|
||||
if s.sawTargetHit {
|
||||
m.RecallTokensAfterTarget += cost
|
||||
}
|
||||
if s.sufficient {
|
||||
m.RecallTokensAfterSufficient += cost
|
||||
}
|
||||
// Everything recall showed the model, judged as a whole: the answer may
|
||||
// arrive across a snippet and a read, and either way it is readable.
|
||||
s.seen.WriteString(msg.Content)
|
||||
s.seen.WriteString("\n")
|
||||
if !s.sufficient && allMarkersIn(s.seen.String(), s.inst.AnswerMarkers) {
|
||||
s.sufficient = true
|
||||
m.FirstSufficientRound = s.round
|
||||
query, _ := recallArgs(call.Arguments)
|
||||
m.SufficientVia = "read"
|
||||
if query != "" {
|
||||
m.SufficientVia = "snippet"
|
||||
}
|
||||
}
|
||||
if query, _ := recallArgs(call.Arguments); query == "" {
|
||||
return
|
||||
}
|
||||
positions := searchHitPositions(msg.Content)
|
||||
if len(positions) > 0 {
|
||||
m.SearchHits++
|
||||
}
|
||||
rank := indexOfInt(positions, s.target)
|
||||
if i, ok := s.pending[msg.ToolCallID]; ok {
|
||||
m.Searches[i].TargetRank, m.Searches[i].ResultCount = rank, len(positions)
|
||||
}
|
||||
if rank == 0 {
|
||||
return
|
||||
}
|
||||
m.TargetSearchHits++
|
||||
if !s.sawTargetHit {
|
||||
m.TargetFirstSeenRound = s.round
|
||||
m.FirstHitCoverage = answerCoverage(msg.Content, s.inst.AnswerMarkers)
|
||||
}
|
||||
s.sawTargetHit = true
|
||||
if m.FirstTargetRank == 0 {
|
||||
m.FirstTargetRank = rank
|
||||
}
|
||||
}
|
||||
|
||||
// finish derives what only the whole walk can say.
|
||||
func (s *runScorer) finish(cueVisible bool) contextMetrics {
|
||||
m := s.m
|
||||
m.RetrievalRounds = m.SearchCalls + m.ReadCalls
|
||||
m.TargetHeldAtEnd = s.sawTargetHit || m.TargetRead
|
||||
if m.TargetFirstSeenRound > 0 {
|
||||
m.RoundsAfterTarget = s.round - m.TargetFirstSeenRound
|
||||
}
|
||||
m.Routing = routing(m.FirstRecallRound, m.FirstEscapeRound)
|
||||
m.CueDirectRead = cueVisible && m.RecallReadWithoutSearch
|
||||
if m.SufficientVia == "" {
|
||||
m.SufficientVia = "never"
|
||||
}
|
||||
if m.FirstSufficientRound > 0 {
|
||||
m.RoundsAfterSufficient = s.round - m.FirstSufficientRound
|
||||
}
|
||||
m.AnswerValues = len(s.inst.AnswerMarkers)
|
||||
m.StoppingClass = stoppingClass(m)
|
||||
classifySearches(&m, s.inst)
|
||||
return m
|
||||
}
|
||||
|
||||
// scoreAnswer grades the final text on the task's markers. Nonces, matched
|
||||
// exactly: no judge model, and nothing a general answer could satisfy.
|
||||
func (m *contextMetrics) scoreAnswer(final string, inst fixtureInstance) {
|
||||
lower := strings.ToLower(final)
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if !strings.Contains(lower, strings.ToLower(marker)) {
|
||||
m.MissingMarkers = append(m.MissingMarkers, marker)
|
||||
}
|
||||
}
|
||||
m.AnswerRecovered = len(m.MissingMarkers) == 0
|
||||
m.AnswerAmbiguous, m.AmbiguousVar = ambiguousAnswer(final, inst)
|
||||
m.FailureStage = m.stage()
|
||||
if len(final) > 1200 {
|
||||
final = final[:1200] + "…"
|
||||
}
|
||||
m.FinalAnswer = final
|
||||
}
|
||||
|
||||
// stage places the run in the funnel. One label, chosen at the first step that
|
||||
// did not happen, so a table of stages reads as where the retrieval broke.
|
||||
func (m contextMetrics) stage() string {
|
||||
switch {
|
||||
case m.Contaminated:
|
||||
return stageContaminated
|
||||
case m.AnswerRecovered && m.RecallReadWithoutSearch:
|
||||
return stageCueRead
|
||||
case m.AnswerRecovered:
|
||||
return stageRecovered
|
||||
case m.TargetRead:
|
||||
return stageAnswerWrong
|
||||
case m.TargetSearchHits > 0:
|
||||
return stageHitNotRead
|
||||
case m.SearchCalls > 0:
|
||||
return stageSearchMiss
|
||||
case m.ReadCalls > 0:
|
||||
return stageReadMissed
|
||||
default:
|
||||
return stageNoRetrieval
|
||||
}
|
||||
}
|
||||
|
||||
// recallArgs pulls the two fields that say which operation a recall call is.
|
||||
func recallArgs(args string) (query string, positions []int) {
|
||||
var parsed struct {
|
||||
Query string `json:"query"`
|
||||
Positions []int `json:"positions"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(args), &parsed); err != nil {
|
||||
return "", nil
|
||||
}
|
||||
return strings.TrimSpace(parsed.Query), parsed.Positions
|
||||
}
|
||||
|
||||
// searchHitPositions reads the addresses out of a search result, in rank order.
|
||||
func searchHitPositions(text string) []int {
|
||||
var out []int
|
||||
for _, match := range hitPosition.FindAllStringSubmatch(text, -1) {
|
||||
if n, err := strconv.Atoi(match[1]); err == nil {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func containsInt(list []int, want int) bool { return slices.Contains(list, want) }
|
||||
|
||||
// indexOfInt returns a 1-based rank, or 0 when absent.
|
||||
func indexOfInt(list []int, want int) int {
|
||||
for i, v := range list {
|
||||
if v == want {
|
||||
return i + 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func containsString(list []string, want string) bool { return slices.Contains(list, want) }
|
||||
|
||||
// funnel summarizes one arm's runs the way the experiment is read.
|
||||
type funnel struct {
|
||||
Arm string
|
||||
Runs int
|
||||
Scored, Contaminated, Errored int
|
||||
Searched, TargetHit, TargetRead, Answered int
|
||||
CueReads int
|
||||
SearchCalls, ReadCalls, RecallTokens int
|
||||
Escapes, EscapeCalls int
|
||||
Stages, Routes map[string]int
|
||||
}
|
||||
|
||||
func summarize(arm string, runs []contextMetrics) funnel {
|
||||
f := funnel{Arm: arm, Runs: len(runs), Stages: map[string]int{}, Routes: map[string]int{}}
|
||||
for _, r := range runs {
|
||||
// A run that never produced a transcript is not a scored run and has no
|
||||
// stage to place: counting it would put an empty name in the funnel, the
|
||||
// stage table and the routes.
|
||||
if r.Failed {
|
||||
continue
|
||||
}
|
||||
if r.Contaminated {
|
||||
f.Contaminated++
|
||||
f.Stages[r.FailureStage]++
|
||||
continue
|
||||
}
|
||||
if r.FailureStage == "RunError" {
|
||||
f.Errored++
|
||||
f.Stages[r.FailureStage]++
|
||||
continue
|
||||
}
|
||||
f.Scored++
|
||||
if r.SearchCalls > 0 {
|
||||
f.Searched++
|
||||
}
|
||||
if r.TargetSearchHits > 0 {
|
||||
f.TargetHit++
|
||||
}
|
||||
if r.TargetRead {
|
||||
f.TargetRead++
|
||||
}
|
||||
if r.AnswerRecovered {
|
||||
f.Answered++
|
||||
}
|
||||
if r.RecallReadWithoutSearch {
|
||||
f.CueReads++
|
||||
}
|
||||
if len(r.UnexpectedWorkTools) > 0 {
|
||||
f.Escapes++
|
||||
}
|
||||
f.EscapeCalls += r.EscapeCalls
|
||||
f.SearchCalls += r.SearchCalls
|
||||
f.ReadCalls += r.ReadCalls
|
||||
f.RecallTokens += r.RecallReturnedTokens
|
||||
f.Stages[r.FailureStage]++
|
||||
if r.FailureStage != "RunError" {
|
||||
f.Routes[r.Routing]++
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func (f funnel) line() string {
|
||||
per := func(n int) float64 {
|
||||
if f.Scored == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(n) / float64(f.Scored)
|
||||
}
|
||||
line := fmt.Sprintf("%-14s runs=%-3d answered=%d/%d searched=%d target-hit=%d cue-read=%d search/task=%.2f recall-tok/task=%.0f escape/task=%.2f",
|
||||
f.Arm, f.Scored, f.Answered, f.Scored, f.Searched, f.TargetHit, f.CueReads,
|
||||
per(f.SearchCalls), per(f.RecallTokens), per(f.EscapeCalls))
|
||||
if f.Contaminated > 0 {
|
||||
line += fmt.Sprintf(" CONTAMINATED=%d", f.Contaminated)
|
||||
}
|
||||
if f.Errored > 0 {
|
||||
line += fmt.Sprintf(" ERRORED=%d", f.Errored)
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func writeJSONLine(w io.Writer, v any) error {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = fmt.Fprintln(w, string(b))
|
||||
return err
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "…"
|
||||
}
|
||||
|
||||
// routing says where the model looked first, comparing rounds rather than
|
||||
// event order. Same round is a parallel fan-out, not a preference.
|
||||
func routing(recallRound, escapeRound int) string {
|
||||
switch {
|
||||
case recallRound == 0 && escapeRound == 0:
|
||||
return routeNeither
|
||||
case recallRound == 0:
|
||||
return routeWorkspaceFirst
|
||||
case escapeRound == 0:
|
||||
return routeMemoryFirst
|
||||
case recallRound < escapeRound:
|
||||
return routeMemoryFirst
|
||||
case escapeRound < recallRound:
|
||||
return routeWorkspaceFirst
|
||||
default:
|
||||
return routeParallel
|
||||
}
|
||||
}
|
||||
|
||||
// ambiguousAnswer reports an answer naming more than one candidate of a
|
||||
// codename's shape. Only codenames are checked: they carry a stem this run
|
||||
// declared, so two distinct "fence-…" values in one answer is one answer too
|
||||
// many. Numbers have no such shape and are left alone.
|
||||
func ambiguousAnswer(final string, inst fixtureInstance) (bool, string) {
|
||||
for _, spec := range inst.Task.Vars {
|
||||
if spec.Kind != varCodename {
|
||||
continue
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, match := range regexp.MustCompile(regexp.QuoteMeta(spec.Word)+`-[a-z0-9]{3,}`).FindAllString(final, -1) {
|
||||
seen[match] = true
|
||||
}
|
||||
if len(seen) > 1 {
|
||||
return true, spec.Name
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
func allMarkersIn(text string, markers []string) bool {
|
||||
if len(markers) == 0 {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(text)
|
||||
for _, marker := range markers {
|
||||
if !strings.Contains(lower, strings.ToLower(marker)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Stopping classes. The boundary is sufficiency, not the target coming back:
|
||||
// a rank-1 hit whose snippet was too short to answer leaves a read entirely
|
||||
// justified, and counting that as waste would blame the model for the bound.
|
||||
const (
|
||||
stopSnippet = "SnippetStop" // enough arrived, and the turn ended
|
||||
stopSnippetRead = "SnippetThenRead" // the snippet was short; the read finished it
|
||||
stopDefensiveRead = "DefensiveRead" // enough had arrived, one read confirmed it
|
||||
stopPostRetrieval = "PostSufficientRetrieval" // enough had arrived, and retrieval continued
|
||||
stopRunaway = "PostSufficientRunaway" // it continued for rounds, or ran out of budget
|
||||
stopNeverEnough = "NeverSufficient"
|
||||
)
|
||||
|
||||
// runawayRounds is where continuing stops looking like confirmation.
|
||||
const runawayRounds = 3
|
||||
|
||||
func stoppingClass(m contextMetrics) string {
|
||||
if m.FirstSufficientRound == 0 {
|
||||
return stopNeverEnough
|
||||
}
|
||||
after := m.SearchesAfterSufficient + m.ReadsAfterSufficient + m.EscapesAfterSufficient
|
||||
switch {
|
||||
case after == 0 && m.SufficientVia == "snippet":
|
||||
return stopSnippet
|
||||
case after == 0:
|
||||
return stopSnippetRead
|
||||
case m.RoundsAfterSufficient >= runawayRounds || m.FailureStage == "RunError":
|
||||
return stopRunaway
|
||||
case after == 1 && m.ReadsAfterSufficient == 1:
|
||||
return stopDefensiveRead
|
||||
default:
|
||||
return stopPostRetrieval
|
||||
}
|
||||
}
|
||||
|
||||
// reportStopping is the funnel that matters once retrieval succeeds: what the
|
||||
// turn did after it already had everything it needed.
|
||||
func reportStopping(all []contextMetrics) string {
|
||||
classes := map[string]int{}
|
||||
var extraRounds, extraTokens []float64
|
||||
sufficient, answeredNext := 0, 0
|
||||
for _, m := range all {
|
||||
if m.Failed || m.Contaminated {
|
||||
continue
|
||||
}
|
||||
classes[m.StoppingClass]++
|
||||
if m.FirstSufficientRound == 0 {
|
||||
continue
|
||||
}
|
||||
sufficient++
|
||||
extraRounds = append(extraRounds, float64(m.RoundsAfterSufficient))
|
||||
extraTokens = append(extraTokens, float64(m.RecallTokensAfterSufficient))
|
||||
if m.RoundsAfterSufficient <= 1 {
|
||||
answeredNext++
|
||||
}
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## After the evidence was already enough\n")
|
||||
fmt.Fprintf(&b, " sufficient in %d runs; ended within one round in %d\n", sufficient, answeredNext)
|
||||
fmt.Fprintf(&b, " extra rounds mean %.1f median %.1f p90 %.0f max %.0f\n",
|
||||
mean(extraRounds), median(extraRounds), percentile(extraRounds, 0.9), maxOf(extraRounds))
|
||||
fmt.Fprintf(&b, " extra recall tokens mean %.0f median %.0f p90 %.0f max %.0f\n",
|
||||
mean(extraTokens), median(extraTokens), percentile(extraTokens, 0.9), maxOf(extraTokens))
|
||||
b.WriteString(" " + countsLine(classes) + "\n")
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// The scorer decides what the pilot concludes, so it is held to the same rule
|
||||
// the corpus is: judgements read structure. A model that says it will search
|
||||
// has not searched; a recall call carrying a query has.
|
||||
|
||||
const scorerTarget = 42
|
||||
|
||||
func recallCall(id, args string) provider.Message {
|
||||
return provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: id, Name: "recall", Arguments: args},
|
||||
}}
|
||||
}
|
||||
|
||||
func toolResult(id, text string) provider.Message {
|
||||
return provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "recall", Content: text}
|
||||
}
|
||||
|
||||
func hitBlock(positions ...int) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("Folded context matching \"x\":\n")
|
||||
for _, p := range positions {
|
||||
fmt.Fprintf(&b, "\n#%d assistant_text\nsome snippet here\n", p)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func scorerTask() fixtureInstance {
|
||||
return fixtureInstance{Task: contextTask{ID: "t"}, AnswerMarkers: []string{"cobalt-lark-17"}}
|
||||
}
|
||||
|
||||
func score(t *testing.T, msgs []provider.Message, final string) contextMetrics {
|
||||
t.Helper()
|
||||
m := scoreRun(msgs, scorerTask(), scorerTarget, "arm", false)
|
||||
m.scoreAnswer(final, scorerTask())
|
||||
return m
|
||||
}
|
||||
|
||||
// The whole funnel, walked end to end.
|
||||
func TestScorerReadsTheFullRetrievalChain(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(7, scorerTarget)),
|
||||
recallCall("c2", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c2", "#42\n[assistant]\nthe ownership token is cobalt-lark-17"),
|
||||
{Role: provider.RoleAssistant, Content: "The token was cobalt-lark-17."},
|
||||
}, "The token was cobalt-lark-17.")
|
||||
|
||||
if m.SearchCalls != 1 || m.ReadCalls != 1 {
|
||||
t.Errorf("calls = %d search / %d read, want 1/1", m.SearchCalls, m.ReadCalls)
|
||||
}
|
||||
if m.TargetSearchHits != 1 || m.FirstTargetRank != 2 {
|
||||
t.Errorf("target hit %d at rank %d, want 1 at rank 2", m.TargetSearchHits, m.FirstTargetRank)
|
||||
}
|
||||
if !m.TargetRead || !m.ReadAfterHit || m.RecallReadWithoutSearch {
|
||||
t.Errorf("read flags = read %v afterHit %v direct %v", m.TargetRead, m.ReadAfterHit, m.RecallReadWithoutSearch)
|
||||
}
|
||||
if !m.AnswerRecovered || m.FailureStage != stageRecovered {
|
||||
t.Errorf("stage = %q recovered=%v, want Recovered", m.FailureStage, m.AnswerRecovered)
|
||||
}
|
||||
if m.RecallReturnedTokens == 0 {
|
||||
t.Error("recall returned tokens were not counted")
|
||||
}
|
||||
}
|
||||
|
||||
// An index arm's claim: the address was already on screen, so no search was
|
||||
// needed. That is a different outcome from finding it, and gets its own stage.
|
||||
func TestScorerSeparatesCueReadFromSearch(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c1", "#42\nthe ownership token is cobalt-lark-17"),
|
||||
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
|
||||
}, "cobalt-lark-17")
|
||||
|
||||
if m.SearchCalls != 0 || !m.RecallReadWithoutSearch {
|
||||
t.Errorf("search=%d direct=%v, want 0 and true", m.SearchCalls, m.RecallReadWithoutSearch)
|
||||
}
|
||||
if m.FailureStage != stageCueRead {
|
||||
t.Errorf("stage = %q, want %q", m.FailureStage, stageCueRead)
|
||||
}
|
||||
}
|
||||
|
||||
// Every way the chain can break gets its own label, so a failed run says where
|
||||
// rather than only that.
|
||||
func TestScorerNamesWhereTheChainBroke(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
msgs []provider.Message
|
||||
final string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "never touched recall",
|
||||
msgs: []provider.Message{{Role: provider.RoleAssistant, Content: "I do not recall that."}},
|
||||
final: "I do not recall that.",
|
||||
want: stageNoRetrieval,
|
||||
},
|
||||
{
|
||||
name: "searched and missed",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", `{"query":"unrelated words"}`),
|
||||
toolResult("c1", hitBlock(3, 9)),
|
||||
},
|
||||
final: "not found",
|
||||
want: stageSearchMiss,
|
||||
},
|
||||
{
|
||||
name: "found it and never opened it",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
},
|
||||
final: "probably something about retries",
|
||||
want: stageHitNotRead,
|
||||
},
|
||||
{
|
||||
name: "read it and answered wrong",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c1", "#42\nthe token is elsewhere"),
|
||||
},
|
||||
final: "I think it was amber-9",
|
||||
want: stageAnswerWrong,
|
||||
},
|
||||
{
|
||||
name: "read the wrong position",
|
||||
msgs: []provider.Message{
|
||||
recallCall("c1", `{"positions":[7]}`),
|
||||
toolResult("c1", "#7\nnothing useful"),
|
||||
},
|
||||
final: "unclear",
|
||||
want: stageReadMissed,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := score(t, tc.msgs, tc.final).FailureStage; got != tc.want {
|
||||
t.Errorf("stage = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The answer lives only in folded history, so reaching for the workspace is a
|
||||
// strategy change worth seeing — including when it happens to work.
|
||||
func TestScorerRecordsWorkspaceEscapes(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"rg cobalt-lark"}`},
|
||||
{ID: "g2", Name: "read_file", Arguments: `{"path":"notes.md"}`},
|
||||
}},
|
||||
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
|
||||
}, "cobalt-lark-17")
|
||||
|
||||
if len(m.UnexpectedWorkTools) != 2 {
|
||||
t.Fatalf("escapes = %v, want bash and read_file", m.UnexpectedWorkTools)
|
||||
}
|
||||
if m.SearchCalls != 0 || m.ReadCalls != 0 {
|
||||
t.Errorf("a workspace tool was counted as recall: search=%d read=%d", m.SearchCalls, m.ReadCalls)
|
||||
}
|
||||
// It answered, so the run is Recovered — the escape is reported beside the
|
||||
// outcome rather than overriding it.
|
||||
if !m.AnswerRecovered {
|
||||
t.Error("a correct answer was not scored as recovered")
|
||||
}
|
||||
}
|
||||
|
||||
// Narration is not action.
|
||||
func TestScorerIgnoresWhatTheModelSaysItWillDo(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, Content: "Let me search the folded context with recall for the retry boundary."},
|
||||
}, "Let me search the folded context with recall for the retry boundary.")
|
||||
if m.SearchCalls != 0 || m.FailureStage != stageNoRetrieval {
|
||||
t.Errorf("prose was scored as a search: calls=%d stage=%q", m.SearchCalls, m.FailureStage)
|
||||
}
|
||||
}
|
||||
|
||||
// Every marker must land, so a partially right answer is not a recovery.
|
||||
func TestScorerRequiresEveryMarker(t *testing.T) {
|
||||
task := fixtureInstance{Task: contextTask{ID: "t"}, AnswerMarkers: []string{"comet-42", "per-lineage"}}
|
||||
m := scoreRun(nil, task, scorerTarget, "arm", false)
|
||||
m.scoreAnswer("the salt was comet-42", task)
|
||||
if m.AnswerRecovered {
|
||||
t.Error("a half answer scored as recovered")
|
||||
}
|
||||
if len(m.MissingMarkers) != 1 || m.MissingMarkers[0] != "per-lineage" {
|
||||
t.Errorf("missing = %v, want per-lineage", m.MissingMarkers)
|
||||
}
|
||||
}
|
||||
|
||||
// An answer reachable through anything but recall invalidates the run. The
|
||||
// workspace is isolated so this should never fire; asserting it beats trusting
|
||||
// it, and a tripped run leaves the statistics rather than inflating one.
|
||||
func TestScorerInvalidatesRunsThatLeakedThroughAnotherTool(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"rg -r ."}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "corpus.go: the token is cobalt-lark-17"},
|
||||
{Role: provider.RoleAssistant, Content: "cobalt-lark-17"},
|
||||
}, "cobalt-lark-17")
|
||||
|
||||
if !m.Contaminated || m.LeakedVia != "bash" || m.LeakedMarker != "cobalt-lark-17" {
|
||||
t.Fatalf("leak not caught: contaminated=%v via=%q marker=%q", m.Contaminated, m.LeakedVia, m.LeakedMarker)
|
||||
}
|
||||
if m.FailureStage != stageContaminated {
|
||||
t.Errorf("stage = %q, want %q even though every marker matched", m.FailureStage, stageContaminated)
|
||||
}
|
||||
f := summarize("arm", []contextMetrics{m})
|
||||
if f.Scored != 0 || f.Answered != 0 || f.Contaminated != 1 {
|
||||
t.Errorf("a contaminated run entered the statistics: %+v", f)
|
||||
}
|
||||
}
|
||||
|
||||
// Where an escape happened matters: before any recall is a different strategy
|
||||
// from after one came back short.
|
||||
func TestScorerPlacesEscapesRelativeToRecall(t *testing.T) {
|
||||
m := score(t, []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g2", Name: "read_file", Arguments: `{"path":"README.md"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g2", Name: "read_file", Content: "nothing here"},
|
||||
}, "unclear")
|
||||
|
||||
if m.EscapeCalls != 2 || m.EscapeBeforeFirstRecall != 1 || m.EscapeAfterFirstRecall != 1 {
|
||||
t.Errorf("escapes = %d (%d before, %d after), want 2 (1, 1)",
|
||||
m.EscapeCalls, m.EscapeBeforeFirstRecall, m.EscapeAfterFirstRecall)
|
||||
}
|
||||
}
|
||||
|
||||
// Routing is judged by round. Two calls in one round are a fan-out, and
|
||||
// reporting that as a preference would invent a decision the model never made.
|
||||
func TestRoutingIsJudgedByRoundNotEventOrder(t *testing.T) {
|
||||
parallel := []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
{ID: "c1", Name: "recall", Arguments: `{"query":"retry boundary"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
}
|
||||
if got := score(t, parallel, "unclear").Routing; got != routeParallel {
|
||||
t.Errorf("routing = %q for one round holding both, want %q", got, routeParallel)
|
||||
}
|
||||
|
||||
workspaceFirst := []provider.Message{
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
}
|
||||
if got := score(t, workspaceFirst, "unclear").Routing; got != routeWorkspaceFirst {
|
||||
t.Errorf("routing = %q for workspace in an earlier round, want %q", got, routeWorkspaceFirst)
|
||||
}
|
||||
|
||||
memoryFirst := []provider.Message{
|
||||
recallCall("c1", `{"query":"retry boundary"}`),
|
||||
toolResult("c1", hitBlock(scorerTarget)),
|
||||
{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: "g1", Name: "bash", Arguments: `{"command":"ls"}`},
|
||||
}},
|
||||
{Role: provider.RoleTool, ToolCallID: "g1", Name: "bash", Content: "README.md"},
|
||||
}
|
||||
if got := score(t, memoryFirst, "unclear").Routing; got != routeMemoryFirst {
|
||||
t.Errorf("routing = %q for recall in an earlier round, want %q", got, routeMemoryFirst)
|
||||
}
|
||||
|
||||
if got := score(t, nil, "no idea").Routing; got != routeNeither {
|
||||
t.Errorf("routing = %q with no tools at all, want %q", got, routeNeither)
|
||||
}
|
||||
}
|
||||
|
||||
// Markers matching proves the right fact appears somewhere. An answer that
|
||||
// hedges between two candidates contains it and has not submitted it.
|
||||
func TestScorerFlagsAnAnswerThatNamesTwoCandidates(t *testing.T) {
|
||||
inst := fixtureInstance{
|
||||
Task: contextTask{ID: "t", Vars: []varSpec{code("marker", "fence")}},
|
||||
AnswerMarkers: []string{"fence-qmwnv"},
|
||||
}
|
||||
m := scoreRun(nil, inst, 0, "arm", false)
|
||||
m.scoreAnswer("It was fence-qmwnv, though it may have been fence-abcde.", inst)
|
||||
if !m.AnswerRecovered {
|
||||
t.Error("the marker is present; recovery should still read true")
|
||||
}
|
||||
if !m.AnswerAmbiguous || m.AmbiguousVar != "marker" {
|
||||
t.Errorf("ambiguity not flagged: %v %q", m.AnswerAmbiguous, m.AmbiguousVar)
|
||||
}
|
||||
|
||||
clean := scoreRun(nil, inst, 0, "arm", false)
|
||||
clean.scoreAnswer("The marker was fence-qmwnv.", inst)
|
||||
if clean.AnswerAmbiguous {
|
||||
t.Error("a single candidate was called ambiguous")
|
||||
}
|
||||
}
|
||||
|
||||
// The index arm's claim is narrow: the cue was addressed, the first memory
|
||||
// action was a positions read, and it covered the target. A positions call
|
||||
// after a search located the target credits the index for the search's work.
|
||||
func TestCueDirectReadRequiresTheCueAndTheFirstAction(t *testing.T) {
|
||||
direct := []provider.Message{
|
||||
recallCall("c1", fmt.Sprintf(`{"positions":[%d]}`, scorerTarget)),
|
||||
toolResult("c1", "#42\nthe token is cobalt-lark-17"),
|
||||
}
|
||||
m := scoreRun(direct, scorerTask(), scorerTarget, "arm", true)
|
||||
if !m.CueDirectRead {
|
||||
t.Error("a first-action read with the cue visible was not credited")
|
||||
}
|
||||
|
||||
// Same read, but no cue was on screen to give the address.
|
||||
if scoreRun(direct, scorerTask(), scorerTarget, "arm", false).CueDirectRead {
|
||||
t.Error("a read was credited to a cue that was not visible")
|
||||
}
|
||||
|
||||
// Same read, but a search found the address first.
|
||||
afterSearch := append([]provider.Message{
|
||||
recallCall("c0", `{"query":"retry boundary"}`),
|
||||
toolResult("c0", hitBlock(scorerTarget)),
|
||||
}, direct...)
|
||||
if scoreRun(afterSearch, scorerTask(), scorerTarget, "arm", true).CueDirectRead {
|
||||
t.Error("a read after a search was credited to the index")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,469 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// Synthetic history on purpose: an answer that exists in the workspace is
|
||||
// solvable by reading the workspace, which measures grep. Every answer is a
|
||||
// placeholder here and a value only inside one run.
|
||||
|
||||
// experiment names which question a task belongs to.
|
||||
const (
|
||||
experimentSearch = "search" // does search reach what no index addresses?
|
||||
experimentIndex = "index" // what is an ambient address line worth?
|
||||
)
|
||||
|
||||
// cueTier is the narrowest fold-index scale at which an index task's cue is
|
||||
// still addressed. Below it the model has to find the call without a hint.
|
||||
const (
|
||||
tierQuarter = "quarter" // visible at default, half, quarter
|
||||
tierHalf = "half" // visible at default, half
|
||||
tierDefault = "default" // visible at default only
|
||||
)
|
||||
|
||||
// role places an index task in the efficiency substrate or the stopping corpus.
|
||||
const (
|
||||
roleEfficiency = "efficiency"
|
||||
roleStopping = "stopping"
|
||||
)
|
||||
|
||||
// plantKind is how a task's target enters the transcript.
|
||||
const (
|
||||
plantAssistant = "assistant" // the model's own account: never index-addressed
|
||||
plantCall = "call" // a tool call and its result: always index-addressed
|
||||
)
|
||||
|
||||
// contextTask is one recall question as a template. Nothing here is an answer.
|
||||
type contextTask struct {
|
||||
ID string
|
||||
Experiment string
|
||||
TargetKind string
|
||||
Prompt string
|
||||
ProbeQuery string
|
||||
// Vars declare the placeholders. AnswerVars name the ones a correct answer
|
||||
// must contain, CueVar the one an index line carries.
|
||||
Vars []varSpec
|
||||
AnswerVars []string
|
||||
CueVar string
|
||||
CueTier string
|
||||
// Role is roleEfficiency or roleStopping; it decides the batch a task enters.
|
||||
Role string
|
||||
// PlantAfterGen ages the target: mergeFoldIndex trims from its oldest end,
|
||||
// so what follows a cue decides which budget addresses it. Frozen, because
|
||||
// a run that searched for it would let the tested code move its goalposts.
|
||||
PlantAfterGen int
|
||||
// Plant is how the target enters, and the rest is what it says.
|
||||
Plant string
|
||||
CallTool string
|
||||
CallArgs string
|
||||
CallID string
|
||||
PlantBody string
|
||||
}
|
||||
|
||||
// fixtureInstance is one task with values. Scoring binds to this, never to the
|
||||
// corpus: the corpus has no answers to bind to.
|
||||
type fixtureInstance struct {
|
||||
Task contextTask
|
||||
Vars fixtureVars
|
||||
Prompt string
|
||||
ProbeQuery string
|
||||
AnswerMarkers []string
|
||||
CueMarker string
|
||||
body string
|
||||
callArgs string
|
||||
}
|
||||
|
||||
// instantiateTask resolves one task against a generator.
|
||||
func instantiateTask(t contextTask, rng *rand.Rand) (fixtureInstance, error) {
|
||||
vars := newVars(t.Vars, rng)
|
||||
inst := fixtureInstance{Task: t, Vars: vars}
|
||||
var err error
|
||||
for _, pair := range []struct {
|
||||
dst *string
|
||||
text string
|
||||
}{
|
||||
{&inst.Prompt, t.Prompt},
|
||||
{&inst.ProbeQuery, t.ProbeQuery},
|
||||
{&inst.body, t.PlantBody},
|
||||
{&inst.callArgs, t.CallArgs},
|
||||
} {
|
||||
if *pair.dst, err = instantiate(pair.text, vars); err != nil {
|
||||
return fixtureInstance{}, fmt.Errorf("%s: %w", t.ID, err)
|
||||
}
|
||||
}
|
||||
for _, name := range t.AnswerVars {
|
||||
value, ok := vars[name]
|
||||
if !ok {
|
||||
return fixtureInstance{}, fmt.Errorf("%s: answer var %q is not declared", t.ID, name)
|
||||
}
|
||||
inst.AnswerMarkers = append(inst.AnswerMarkers, value)
|
||||
}
|
||||
if t.CueVar != "" {
|
||||
inst.CueMarker = vars[t.CueVar]
|
||||
}
|
||||
return inst, nil
|
||||
}
|
||||
|
||||
// plant appends the instantiated target and returns its canonical position.
|
||||
func (f fixtureInstance) plant(s *sessionstore.Session) int {
|
||||
at := len(s.Messages)
|
||||
if f.Task.Plant == plantAssistant {
|
||||
s.Add(provider.Message{Role: provider.RoleAssistant, Content: f.body})
|
||||
return at
|
||||
}
|
||||
s.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: f.Task.CallID, Name: f.Task.CallTool, Arguments: f.callArgs},
|
||||
}})
|
||||
s.Add(provider.Message{Role: provider.RoleTool, ToolCallID: f.Task.CallID, Name: f.Task.CallTool, Content: f.body})
|
||||
return at
|
||||
}
|
||||
|
||||
func code(name, word string) varSpec { return varSpec{Name: name, Kind: varCodename, Word: word} }
|
||||
|
||||
func num(name string, lo, hi int) varSpec {
|
||||
return varSpec{Name: name, Kind: varInt, Min: lo, Max: hi}
|
||||
}
|
||||
|
||||
// efficiencyTasks is the substrate a study measures on. Every task in it
|
||||
// qualifies under the corpus bar; the stopping corpus is deliberately excluded,
|
||||
// because a task carrying post-sufficient stopping behaviour swamps the effect
|
||||
// an efficiency batch exists to measure.
|
||||
func efficiencyTasks() []contextTask {
|
||||
var out []contextTask
|
||||
for _, t := range indexTasks() {
|
||||
if t.Role == roleEfficiency {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func searchTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "r01-retry-boundary", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "What ownership token did we settle on for the retry boundary, and under what tag did we file the reason for not moving retries above framing?",
|
||||
ProbeQuery: "retry boundary ownership token framing",
|
||||
Vars: []varSpec{code("token", "cobalt"), code("reason", "stream")},
|
||||
AnswerVars: []string{"token", "reason"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The retry boundary remains at ingress. The ownership token is {{token}}. " +
|
||||
"Moving retries above framing can duplicate a partially received stream, which we filed as {{reason}}.",
|
||||
},
|
||||
{
|
||||
ID: "r02-checkpoint-anchor", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "What was the codename of the checkpoint anchor policy, and what tag did we give the reason for not anchoring against digest positions?",
|
||||
ProbeQuery: "checkpoint anchor policy digest positions",
|
||||
Vars: []varSpec{code("policy", "northglass"), code("reason", "genlocal")},
|
||||
AnswerVars: []string{"policy", "reason"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The checkpoint anchor policy is {{policy}}. Digest positions are generation-local — " +
|
||||
"filed as {{reason}} — so recovery must anchor against canonical state.",
|
||||
},
|
||||
{
|
||||
ID: "r03-recovery-cjk", Experiment: experimentSearch, TargetKind: "assistant_reasoning_cjk",
|
||||
Prompt: "我们之前定的验证失败恢复策略代号叫什么？当时把“失败 digest 不能变成 baseline”这条登记在哪个标记下？",
|
||||
ProbeQuery: "恢复策略 baseline digest 验证失败",
|
||||
Vars: []varSpec{code("policy", "songzhen"), code("rule", "trustbase")},
|
||||
AnswerVars: []string{"policy", "rule"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "恢复策略代号是 {{policy}}。验证失败以后必须保留旧基线，不能把新的 digest 晋升为 baseline，" +
|
||||
"因为失败证据不能成为后续比较的可信基准；这条规则登记为 {{rule}}。",
|
||||
},
|
||||
{
|
||||
ID: "r04-lease-fence", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "那次 lease fence 检查报告的诊断标记是什么？expected 和 observed epoch 分别是多少？",
|
||||
ProbeQuery: "lease fence writer epoch mismatch",
|
||||
Vars: []varSpec{
|
||||
code("marker", "fence"), num("expected", 400, 900), num("observed", 100, 399),
|
||||
},
|
||||
AnswerVars: []string{"marker", "expected", "observed"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The lease fence check reported {{marker}}: expected writer epoch {{expected}}, " +
|
||||
"observed {{observed}}. We left the fence in place.",
|
||||
},
|
||||
{
|
||||
ID: "r05-schema-guard", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "当时 schema guard 报告的标记是什么，超预算多少 token？",
|
||||
ProbeQuery: "schema guard provider visible surface budget",
|
||||
Vars: []varSpec{code("marker", "amber"), num("over", 120, 480)},
|
||||
AnswerVars: []string{"marker", "over"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The schema guard reported {{marker}}: the provider-visible tool schema exceeded " +
|
||||
"the allowed surface by {{over}} tokens. We trimmed a description instead.",
|
||||
},
|
||||
{
|
||||
ID: "r06-shadow-probe", Experiment: experimentSearch, TargetKind: "assistant_reasoning",
|
||||
Prompt: "我们那次 shadow probe 的 marker、p95 divergence 和 generation 分别是多少？",
|
||||
ProbeQuery: "shadow probe divergence generation marker",
|
||||
Vars: []varSpec{
|
||||
code("marker", "violet"),
|
||||
{Name: "divergence", Kind: varDecimal, Min: 110, Max: 890},
|
||||
num("generation", 14, 60),
|
||||
},
|
||||
AnswerVars: []string{"marker", "divergence", "generation"},
|
||||
Plant: plantAssistant,
|
||||
PlantBody: "The shadow probe came back as marker={{marker}} with a p95 divergence of " +
|
||||
"{{divergence}} at generation={{generation}}, which we judged within tolerance.",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// efficiencyIndexTasks is what a study measures on: two tasks per cue tier.
|
||||
func efficiencyIndexTasks() []contextTask {
|
||||
out := indexQuarterTasks()
|
||||
out = append(out, indexHalfTasks()...)
|
||||
out = append(out, indexDefaultTasks()...)
|
||||
return out
|
||||
}
|
||||
|
||||
// indexQuarterTasks holds the quarter-tier literals.
|
||||
func indexQuarterTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i37-dispatchlag", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/dispatchlag，它打印出的 dispatch lag epoch 是多少？",
|
||||
ProbeQuery: "dispatchlag dispatch lag epoch",
|
||||
Vars: []varSpec{code("cue", "dispatchlag"), code("epoch", "cobalt")},
|
||||
AnswerVars: []string{"epoch"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "p37", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/dispatchlag {{cue}}"}`,
|
||||
PlantBody: "dispatch lag epoch = {{epoch}}",
|
||||
},
|
||||
{
|
||||
ID: "i118-fencelag", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里运行过 ./scripts/fencelag，它打印出的 fence lag 是多少？",
|
||||
ProbeQuery: "fencelag fence lag",
|
||||
Vars: []varSpec{code("cue", "fencelag"), code("v", "slate")},
|
||||
AnswerVars: []string{"v"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "x118", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/fencelag {{cue}}"}`,
|
||||
PlantBody: "fence lag = {{v}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// indexHalfTasks holds the half-tier literals.
|
||||
func indexHalfTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i70-ingestfloor", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/ingestfloor，它打印出的 route quota 是多少？",
|
||||
ProbeQuery: "ingestfloor quota",
|
||||
Vars: []varSpec{code("cue", "ingestfloor"), num("quota", 1200, 9800)},
|
||||
AnswerVars: []string{"quota"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "q70", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/ingestfloor {{cue}}"}`,
|
||||
PlantBody: "route quota = {{quota}}",
|
||||
},
|
||||
{
|
||||
ID: "i106-budgetseal", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/budgetseal，它打印出的 route quota 是多少？",
|
||||
ProbeQuery: "budgetseal route quota",
|
||||
Vars: []varSpec{code("cue", "budgetseal"), code("quota", "amber")},
|
||||
AnswerVars: []string{"quota"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "h106", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/budgetseal {{cue}}"}`,
|
||||
PlantBody: "route quota = {{quota}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// indexDefaultTasks holds the default-tier literals.
|
||||
func indexDefaultTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i47-tenantlease", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/tenantlease，它打印出的 tenant lease（秒）是多少？",
|
||||
ProbeQuery: "tenantlease tenant lease",
|
||||
Vars: []varSpec{code("cue", "tenantlease"), {Name: "lease", Kind: varDuration, Min: 15, Max: 300}},
|
||||
AnswerVars: []string{"lease"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "p47", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/tenantlease {{cue}}"}`,
|
||||
PlantBody: "tenant lease = {{lease}}",
|
||||
},
|
||||
{
|
||||
ID: "i112-lagquota", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleEfficiency,
|
||||
Prompt: "早前会话里执行过 ./scripts/lagquota，它打印出的 lagquota lease 是多少？",
|
||||
ProbeQuery: "lagquota lagquota lease",
|
||||
Vars: []varSpec{code("cue", "lagquota"), {Name: "lease", Kind: varDuration, Min: 20, Max: 420}},
|
||||
AnswerVars: []string{"lease"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "d112", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/lagquota {{cue}}"}`,
|
||||
PlantBody: "lagquota lease = {{lease}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// stoppingIndexTasks is the frozen study's six tasks, kept out of efficiency
|
||||
// batches and measurable on their own with -task <id>.
|
||||
func stoppingIndexTasks() []contextTask {
|
||||
return []contextTask{
|
||||
{
|
||||
ID: "i01-transport-fallback", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "transport fallback probe 当时报告的 selector 是什么？",
|
||||
ProbeQuery: "transport fallback probe selector",
|
||||
Vars: []varSpec{code("cue", "tfprobe"), code("selector", "fern")},
|
||||
AnswerVars: []string{"selector"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "tf1", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/probe {{cue}}"}`,
|
||||
PlantBody: "fallback selector = {{selector}}",
|
||||
},
|
||||
{
|
||||
ID: "i02-scheduler-handoff", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "scheduler handoff 当时记录的 grace 和 marker 是什么？",
|
||||
ProbeQuery: "scheduler handoff grace marker",
|
||||
Vars: []varSpec{
|
||||
code("cue", "schedhandoff"), code("marker", "opal"),
|
||||
{Name: "grace", Kind: varDuration, Min: 18, Max: 90},
|
||||
},
|
||||
AnswerVars: []string{"grace", "marker"}, CueVar: "cue", CueTier: tierQuarter,
|
||||
PlantAfterGen: 4,
|
||||
Plant: plantCall, CallID: "sh1", CallTool: "read_file",
|
||||
CallArgs: `{"path":"scratch/{{cue}}.txt"}`,
|
||||
PlantBody: "handoff grace = {{grace}}\nmarker = {{marker}}",
|
||||
},
|
||||
{
|
||||
ID: "i03-coalescing", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "job completion 的 coalescing probe 最后记录的 window 和 marker 是多少？",
|
||||
ProbeQuery: "coalescing policy completion window",
|
||||
Vars: []varSpec{
|
||||
code("cue", "coalescing"), code("marker", "cedar"),
|
||||
{Name: "window", Kind: varDuration, Min: 40, Max: 140},
|
||||
},
|
||||
AnswerVars: []string{"window", "marker"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "cp1", CallTool: "bash",
|
||||
CallArgs: `{"command":"grep {{cue}} scratch/job-notes.log"}`,
|
||||
PlantBody: "completion coalesce window = {{window}}\nmarker = {{marker}}",
|
||||
},
|
||||
{
|
||||
ID: "i04-recovery-fence", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "recovery fence 那次检查报告的 marker 和两个 epoch 是什么？",
|
||||
ProbeQuery: "recovery fence epoch check",
|
||||
Vars: []varSpec{
|
||||
code("cue", "recfence"), code("marker", "quartz"),
|
||||
num("expected", 400, 900), num("observed", 100, 399),
|
||||
},
|
||||
AnswerVars: []string{"marker", "expected", "observed"}, CueVar: "cue", CueTier: tierHalf,
|
||||
PlantAfterGen: 2,
|
||||
Plant: plantCall, CallID: "rf1", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/check {{cue}}"}`,
|
||||
PlantBody: "{{marker}}\nexpected epoch {{expected}}\nobserved epoch {{observed}}",
|
||||
},
|
||||
{
|
||||
ID: "i05-cache-probe", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "cache probe 当时记录的 prefix salt 和 scope 是什么？",
|
||||
ProbeQuery: "cache probe prefix salt scope",
|
||||
Vars: []varSpec{code("cue", "cacheprobe"), code("salt", "comet"), code("scope", "lineage")},
|
||||
AnswerVars: []string{"salt", "scope"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "cb1", CallTool: "read_file",
|
||||
CallArgs: `{"path":"scratch/{{cue}}.txt"}`,
|
||||
PlantBody: "prefix salt = {{salt}}\nscope = {{scope}}",
|
||||
},
|
||||
{
|
||||
ID: "i06-dispatch-handoff", Experiment: experimentIndex, TargetKind: "tool_input", Role: roleStopping,
|
||||
Prompt: "dispatch handoff probe 当时确认的 mode 和 marker 是什么？",
|
||||
ProbeQuery: "dispatch handoff probe mode",
|
||||
Vars: []varSpec{code("cue", "dispatchhandoff"), code("mode", "trigger"), code("marker", "iris")},
|
||||
AnswerVars: []string{"mode", "marker"}, CueVar: "cue", CueTier: tierDefault,
|
||||
PlantAfterGen: 0,
|
||||
Plant: plantCall, CallID: "dh1", CallTool: "bash",
|
||||
CallArgs: `{"command":"./scripts/probe {{cue}}"}`,
|
||||
PlantBody: "handoff mode = {{mode}}\nmarker = {{marker}}",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// indexTasks is the whole index corpus, the efficiency substrate first.
|
||||
func indexTasks() []contextTask {
|
||||
out := efficiencyIndexTasks()
|
||||
return append(out, stoppingIndexTasks()...)
|
||||
}
|
||||
|
||||
func allTasks() []contextTask {
|
||||
return append(searchTasks(), indexTasks()...)
|
||||
}
|
||||
|
||||
func taskByID(id string) (contextTask, bool) {
|
||||
for _, t := range allTasks() {
|
||||
if t.ID == id {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return contextTask{}, false
|
||||
}
|
||||
|
||||
// tierScales is the fixture's contract: which arms must address a cue and
|
||||
// which must not. A policy change that moves a cue across a boundary fails
|
||||
// preflight rather than turning two arms into one experiment.
|
||||
func tierScales(tier string) (visible, hidden []string) {
|
||||
switch tier {
|
||||
case tierQuarter:
|
||||
return []string{"default", "half", "quarter"}, []string{"off"}
|
||||
case tierHalf:
|
||||
return []string{"default", "half"}, []string{"quarter", "off"}
|
||||
case tierDefault:
|
||||
return []string{"default"}, []string{"half", "quarter", "off"}
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
// boundaryPair is the one comparison an index task is built for: the narrowest
|
||||
// scale that still addresses its cue, and the next one down. Holding the
|
||||
// question fixed and moving only the affordance is what four arm averages
|
||||
// cannot do, since each of those mixes six different questions.
|
||||
func boundaryPair(tier string) (cue, noCue string) {
|
||||
switch tier {
|
||||
case tierQuarter:
|
||||
return "quarter", "off"
|
||||
case tierHalf:
|
||||
return "half", "quarter"
|
||||
case tierDefault:
|
||||
return "default", "half"
|
||||
default:
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
|
||||
func validateCorpus() error {
|
||||
seen := map[string]bool{}
|
||||
for _, t := range allTasks() {
|
||||
if seen[t.ID] {
|
||||
return fmt.Errorf("duplicate task id %q", t.ID)
|
||||
}
|
||||
seen[t.ID] = true
|
||||
if len(t.AnswerVars) == 0 || t.ProbeQuery == "" || t.Prompt == "" || t.PlantBody == "" {
|
||||
return fmt.Errorf("%s: a task needs a prompt, a probe query, a body and answer vars", t.ID)
|
||||
}
|
||||
if t.Experiment == experimentIndex {
|
||||
if t.CueVar == "" {
|
||||
return fmt.Errorf("%s: an index task needs a cue var", t.ID)
|
||||
}
|
||||
if t.Role != roleEfficiency && t.Role != roleStopping {
|
||||
return fmt.Errorf("%s: an index task needs a role (%s or %s)", t.ID, roleEfficiency, roleStopping)
|
||||
}
|
||||
if v, _ := tierScales(t.CueTier); v == nil {
|
||||
return fmt.Errorf("%s: unknown cue tier %q", t.ID, t.CueTier)
|
||||
}
|
||||
}
|
||||
if _, err := instantiateTask(t, seededRand(t.ID, "validate")); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
// dose.go — does the cost step where the cue disappears?
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Not "a bigger index is better" — CueDirectRead says the addresses are rarely
|
||||
// used. The hypothesis is that a visible action cue lowers retrieval cost,
|
||||
// which predicts a step at each task's own boundary, not a slope.
|
||||
|
||||
// scaleOrder is the dose axis, widest first.
|
||||
var scaleOrder = []string{"default", "half", "quarter", "off"}
|
||||
|
||||
// boundaryStep is the index into the three transitions where a tier's cue
|
||||
// disappears: default->half is 0, half->quarter is 1, quarter->off is 2.
|
||||
func boundaryStep(tier string) int {
|
||||
switch tier {
|
||||
case tierDefault:
|
||||
return 0
|
||||
case tierHalf:
|
||||
return 1
|
||||
case tierQuarter:
|
||||
return 2
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
// doseCurve is one task's readings across the four budgets.
|
||||
type doseCurve struct {
|
||||
Task string
|
||||
Tier string
|
||||
Metric string
|
||||
Values [4]float64
|
||||
// Missing marks arms that produced nothing. Printing those as zero draws a
|
||||
// curve falling to the floor where in fact it was never measured.
|
||||
Missing [4]bool
|
||||
Present bool
|
||||
// PreMean averages the arms where the cue is visible, PostMean the rest.
|
||||
PreMean, PostMean float64
|
||||
BoundaryDelta float64
|
||||
// BoundaryAligned: the largest transition is the one where this task's cue
|
||||
// disappears. A change elsewhere is variance shaped like an effect.
|
||||
BoundaryAligned bool
|
||||
}
|
||||
|
||||
func buildCurve(task, tier, metric string, byScale map[string]float64) doseCurve {
|
||||
c := doseCurve{Task: task, Tier: tier, Metric: metric, Present: true}
|
||||
for i, scale := range scaleOrder {
|
||||
v, ok := byScale[scale]
|
||||
if !ok {
|
||||
c.Present, c.Missing[i] = false, true
|
||||
}
|
||||
c.Values[i] = v
|
||||
}
|
||||
step := boundaryStep(tier)
|
||||
if !c.Present || step < 0 {
|
||||
return c
|
||||
}
|
||||
visible := step + 1 // arms before the boundary still show the cue
|
||||
c.PreMean = mean(c.Values[:visible])
|
||||
c.PostMean = mean(c.Values[visible:])
|
||||
c.BoundaryDelta = c.PostMean - c.PreMean
|
||||
|
||||
transitions := [3]float64{
|
||||
math.Abs(c.Values[1] - c.Values[0]),
|
||||
math.Abs(c.Values[2] - c.Values[1]),
|
||||
math.Abs(c.Values[3] - c.Values[2]),
|
||||
}
|
||||
largest, at := 0.0, -1
|
||||
for i, t := range transitions {
|
||||
if t > largest {
|
||||
largest, at = t, i
|
||||
}
|
||||
}
|
||||
// A flat curve has no largest transition to align with anything.
|
||||
c.BoundaryAligned = largest > 0 && at == step
|
||||
return c
|
||||
}
|
||||
|
||||
func mean(xs []float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
sum := 0.0
|
||||
for _, x := range xs {
|
||||
sum += x
|
||||
}
|
||||
return sum / float64(len(xs))
|
||||
}
|
||||
|
||||
func median(xs []float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
s := append([]float64(nil), xs...)
|
||||
sort.Float64s(s)
|
||||
mid := len(s) / 2
|
||||
if len(s)%2 == 1 {
|
||||
return s[mid]
|
||||
}
|
||||
return (s[mid-1] + s[mid]) / 2
|
||||
}
|
||||
|
||||
// doseMetrics are the readings a curve is drawn for.
|
||||
var doseMetrics = []struct {
|
||||
name string
|
||||
of func(contextMetrics) float64
|
||||
}{
|
||||
{"searches", func(m contextMetrics) float64 { return float64(m.SearchCalls) }},
|
||||
{"recall-tokens", func(m contextMetrics) float64 { return float64(m.RecallReturnedTokens) }},
|
||||
{"tool-rounds", func(m contextMetrics) float64 { return float64(m.RetrievalRounds + m.EscapeCalls) }},
|
||||
{"escapes", func(m contextMetrics) float64 { return float64(m.EscapeCalls) }},
|
||||
}
|
||||
|
||||
// reportDoseResponse draws each task's curve and reports where the steps fell.
|
||||
func reportDoseResponse(all []contextMetrics) string {
|
||||
valid := map[string]map[string]contextMetrics{}
|
||||
for _, m := range all {
|
||||
if m.Failed || m.Contaminated || m.FailureStage == "RunError" {
|
||||
continue
|
||||
}
|
||||
scale := strings.TrimPrefix(m.Arm, "index-")
|
||||
if valid[m.Task] == nil {
|
||||
valid[m.Task] = map[string]contextMetrics{}
|
||||
}
|
||||
valid[m.Task][scale] = m
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## Dose-response, per task\n")
|
||||
b.WriteString(" The arrow marks where this task's cue stops being addressed.\n")
|
||||
aligned := map[string]int{}
|
||||
counted := map[string]int{}
|
||||
deltas := map[string][]float64{}
|
||||
|
||||
for _, t := range indexTasks() {
|
||||
arms := valid[t.ID]
|
||||
if len(arms) == 0 {
|
||||
continue
|
||||
}
|
||||
step := boundaryStep(t.CueTier)
|
||||
fmt.Fprintf(&b, "\n %s (cue survives to %s)\n", t.ID, t.CueTier)
|
||||
fmt.Fprintf(&b, " %-14s%s\n", "", scaleHeader(step))
|
||||
for _, metric := range doseMetrics {
|
||||
byScale := map[string]float64{}
|
||||
for scale, m := range arms {
|
||||
byScale[scale] = metric.of(m)
|
||||
}
|
||||
c := buildCurve(t.ID, t.CueTier, metric.name, byScale)
|
||||
mark := " "
|
||||
if c.BoundaryAligned {
|
||||
mark = "*"
|
||||
aligned[metric.name]++
|
||||
}
|
||||
if c.Present {
|
||||
counted[metric.name]++
|
||||
deltas[metric.name] = append(deltas[metric.name], c.BoundaryDelta)
|
||||
}
|
||||
delta := fmt.Sprintf("Δ%+8.1f", c.BoundaryDelta)
|
||||
if !c.Present {
|
||||
delta = " no curve"
|
||||
}
|
||||
fmt.Fprintf(&b, " %-14s%s %s %s\n", metric.name, curveCells(c), delta, mark)
|
||||
}
|
||||
}
|
||||
|
||||
b.WriteString("\n## Where the step fell (* = largest transition is at the cue boundary)\n")
|
||||
for _, metric := range doseMetrics {
|
||||
d := deltas[metric.name]
|
||||
sensitivity := ""
|
||||
if len(d) > 1 {
|
||||
sensitivity = fmt.Sprintf(" without the largest: %+.1f", meanWithoutLargest(d))
|
||||
}
|
||||
fmt.Fprintf(&b, " %-14s boundary-aligned %d/%d mean Δ%+.1f median Δ%+.1f%s\n",
|
||||
metric.name, aligned[metric.name], counted[metric.name], mean(d), median(d), sensitivity)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// meanWithoutLargest is the robustness check: the largest effect is reported
|
||||
// and then set aside, so a conclusion never rests on one task alone.
|
||||
func meanWithoutLargest(d []float64) float64 {
|
||||
if len(d) < 2 {
|
||||
return mean(d)
|
||||
}
|
||||
worst, at := 0.0, 0
|
||||
for i, v := range d {
|
||||
if math.Abs(v) > worst {
|
||||
worst, at = math.Abs(v), i
|
||||
}
|
||||
}
|
||||
rest := append(append([]float64(nil), d[:at]...), d[at+1:]...)
|
||||
return mean(rest)
|
||||
}
|
||||
|
||||
func scaleHeader(step int) string {
|
||||
var b strings.Builder
|
||||
for i, s := range scaleOrder {
|
||||
if i == step+1 {
|
||||
s = "↓" + s
|
||||
}
|
||||
fmt.Fprintf(&b, "%8s", s)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// curveCells prints a reading per arm, and a dash where an arm produced none.
|
||||
func curveCells(c doseCurve) string {
|
||||
var b strings.Builder
|
||||
for i, v := range c.Values {
|
||||
if c.Missing[i] {
|
||||
fmt.Fprintf(&b, "%8s", "-")
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "%8.0f", v)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func percentile(xs []float64, q float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
s := append([]float64(nil), xs...)
|
||||
sort.Float64s(s)
|
||||
i := int(float64(len(s)-1) * q)
|
||||
return s[i]
|
||||
}
|
||||
|
||||
func maxOf(xs []float64) float64 {
|
||||
out := 0.0
|
||||
for _, x := range xs {
|
||||
if x > out {
|
||||
out = x
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// blockTask makes one task fail before its first model turn: its own directory
|
||||
// is a regular file, so buildFixture cannot create the session under it. It is
|
||||
// the same early exit as a failed fixture build, session load, or
|
||||
// answer-on-disk guard, and it needs no provider.
|
||||
func blockTask(t *testing.T, root, id string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(root, id), []byte("not a directory\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// mixedTaskPair is one task whose runs all fail and one whose runs all succeed.
|
||||
// Both are drawn from the corpus, so the batch is the real pipeline.
|
||||
func mixedTaskPair() (blocked, ok contextTask) {
|
||||
blocked, ok = indexTasks()[0], indexTasks()[1]
|
||||
for _, t := range indexTasks() {
|
||||
if t.Role == roleEfficiency && len(armsFor(t)) == 2 {
|
||||
ok = t
|
||||
break
|
||||
}
|
||||
}
|
||||
return blocked, ok
|
||||
}
|
||||
|
||||
// runBlockedExperiment runs one dry index batch over the given tasks, with the
|
||||
// named task failing before any model turn, and returns the batch directory, the
|
||||
// runs, the manifest, and everything the run printed.
|
||||
func runBlockedExperiment(t *testing.T, tasks []contextTask, blocked ...string) (string, []contextMetrics, batchManifest, string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for _, id := range blocked {
|
||||
blockTask(t, root, id)
|
||||
}
|
||||
out := captureStdout(t, func() {
|
||||
if code := runExperiment("run-index", experimentIndex, root, true, tasks); code != 0 {
|
||||
t.Errorf("runExperiment exit = %d, want 0 (the batch finished)", code)
|
||||
}
|
||||
})
|
||||
runs, _ := readResultLines(t, root)
|
||||
manifest, _ := readManifest(t, root)
|
||||
return root, runs, manifest, out
|
||||
}
|
||||
|
||||
// captureStdout reads everything the function prints. os.Pipe is unbuffered, so
|
||||
// the reader must run beside the call.
|
||||
func captureStdout(t *testing.T, f func()) string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := os.Stdout
|
||||
os.Stdout = w
|
||||
done := make(chan string, 1)
|
||||
go func() {
|
||||
var b strings.Builder
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := r.Read(buf)
|
||||
b.Write(buf[:n])
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
done <- b.String()
|
||||
}()
|
||||
f()
|
||||
os.Stdout = old
|
||||
w.Close()
|
||||
text := <-done
|
||||
r.Close()
|
||||
return text
|
||||
}
|
||||
|
||||
// funnelScored reads the "runs=N" column of the named arm's funnel line out of
|
||||
// the printed tables, so the assertion is on the number a reader sees.
|
||||
func funnelScored(t *testing.T, out, arm string) int {
|
||||
t.Helper()
|
||||
for line := range strings.SplitSeq(out, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 || fields[0] != arm || !strings.HasPrefix(fields[1], "runs=") {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.Atoi(strings.TrimPrefix(fields[1], "runs="))
|
||||
if err != nil {
|
||||
t.Fatalf("funnel line for %s: %v\n%s", arm, err, line)
|
||||
}
|
||||
return n
|
||||
}
|
||||
t.Fatalf("no funnel line for arm %s:\n%s", arm, out)
|
||||
return 0
|
||||
}
|
||||
|
||||
// The batch report states both counts on one line, so a reader of the scrollback
|
||||
// sees the scored set and the failures without counting funnel rows.
|
||||
func reportCounts(t *testing.T, out string) (scored, failed int) {
|
||||
t.Helper()
|
||||
for line := range strings.SplitSeq(out, "\n") {
|
||||
if !strings.Contains(line, " scored=") || !strings.Contains(line, " failed=") {
|
||||
continue
|
||||
}
|
||||
for f := range strings.FieldsSeq(line) {
|
||||
switch {
|
||||
case strings.HasPrefix(f, "scored="):
|
||||
scored, _ = strconv.Atoi(strings.TrimPrefix(f, "scored="))
|
||||
case strings.HasPrefix(f, "failed="):
|
||||
failed, _ = strconv.Atoi(strings.TrimPrefix(f, "failed="))
|
||||
}
|
||||
}
|
||||
return scored, failed
|
||||
}
|
||||
t.Fatalf("no line reports scored= and failed=:\n%s", out)
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
// A batch that mixes a task that never ran with one that did must read as its
|
||||
// successful runs only: the failed task leaves Scored, the funnel, and the
|
||||
// completed count, and is reported as a failure with its identity instead.
|
||||
// Every assertion here is a number, not the absence of an empty row.
|
||||
func TestMixedBatchCountsOnlyTheRunThatProducedEvidence(t *testing.T) {
|
||||
blocked, ok := mixedTaskPair()
|
||||
_, runs, manifest, out := runBlockedExperiment(t, []contextTask{blocked, ok}, blocked.ID)
|
||||
|
||||
wantScored, wantFailed := len(armsFor(ok)), len(armsFor(blocked))
|
||||
if manifest.CompletedRuns != wantScored || manifest.FailedRuns != wantFailed {
|
||||
t.Fatalf("manifest completed=%d failed=%d, want %d and %d (planned=%d)",
|
||||
manifest.CompletedRuns, manifest.FailedRuns, wantScored, wantFailed, manifest.PlannedRuns)
|
||||
}
|
||||
if len(runs) != manifest.CompletedRuns+manifest.FailedRuns {
|
||||
t.Fatalf("results.jsonl has %d rows but completed=%d failed=%d",
|
||||
len(runs), manifest.CompletedRuns, manifest.FailedRuns)
|
||||
}
|
||||
for _, id := range manifest.TaskIDs {
|
||||
if id == blocked.ID {
|
||||
t.Fatalf("task_ids lists %s, which produced no evidence: %v", blocked.ID, manifest.TaskIDs)
|
||||
}
|
||||
}
|
||||
scored, failures := 0, 0
|
||||
for _, m := range runs {
|
||||
if m.Failed {
|
||||
failures++
|
||||
if m.Task != blocked.ID || m.FailureStage == "" || m.Reason == "" {
|
||||
t.Fatalf("failed row lost its identity or stage: %+v", m)
|
||||
}
|
||||
continue
|
||||
}
|
||||
scored++
|
||||
if m.Task != ok.ID {
|
||||
t.Fatalf("a run scored from the wrong task: %+v", m)
|
||||
}
|
||||
}
|
||||
if scored != wantScored || failures != wantFailed {
|
||||
t.Fatalf("rows scored=%d failed=%d, want %d and %d", scored, failures, wantScored, wantFailed)
|
||||
}
|
||||
// Each arm's funnel holds exactly the successful task's runs, and the two
|
||||
// scored counts agree with each other and with the manifest.
|
||||
total := 0
|
||||
for _, arm := range armsFor(ok) {
|
||||
total += funnelScored(t, out, arm.name)
|
||||
}
|
||||
if total != manifest.CompletedRuns {
|
||||
t.Fatalf("funnels scored %d runs, manifest completed_runs = %d:\n%s", total, manifest.CompletedRuns, out)
|
||||
}
|
||||
if gotScored, gotFailed := reportCounts(t, out); gotScored != manifest.CompletedRuns || gotFailed != manifest.FailedRuns {
|
||||
t.Fatalf("report says scored=%d failed=%d, manifest says %d and %d",
|
||||
gotScored, gotFailed, manifest.CompletedRuns, manifest.FailedRuns)
|
||||
}
|
||||
if strings.Contains(out, "## Where it broke") && strings.Contains(out, "=-\n") {
|
||||
t.Fatalf("the stage table kept an empty stage:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch where every run fails must not read as an empty success: zero scored
|
||||
// runs, a nonzero failed count, and both stated.
|
||||
func TestAllFailedBatchReportsZeroScoredAndNonzeroFailed(t *testing.T) {
|
||||
blocked, _ := mixedTaskPair()
|
||||
_, runs, manifest, out := runBlockedExperiment(t, []contextTask{blocked}, blocked.ID)
|
||||
|
||||
if manifest.CompletedRuns != 0 {
|
||||
t.Fatalf("completed_runs = %d, want 0 (nothing produced evidence)", manifest.CompletedRuns)
|
||||
}
|
||||
if manifest.FailedRuns != manifest.PlannedRuns || manifest.FailedRuns == 0 {
|
||||
t.Fatalf("failed_runs = %d, planned_runs = %d: every planned run failed",
|
||||
manifest.FailedRuns, manifest.PlannedRuns)
|
||||
}
|
||||
if len(manifest.TaskIDs) != 0 {
|
||||
t.Fatalf("task_ids = %v, want none", manifest.TaskIDs)
|
||||
}
|
||||
for i, m := range runs {
|
||||
if !m.Failed || m.FailureStage == "" || m.Task != blocked.ID {
|
||||
t.Fatalf("results.jsonl line %d is not a marked failure: %+v", i+1, m)
|
||||
}
|
||||
}
|
||||
total := 0
|
||||
for _, arm := range armsFor(blocked) {
|
||||
total += funnelScored(t, out, arm.name)
|
||||
}
|
||||
if total != 0 {
|
||||
t.Fatalf("funnels scored %d runs although nothing ran:\n%s", total, out)
|
||||
}
|
||||
if gotScored, gotFailed := reportCounts(t, out); gotScored != 0 || gotFailed != manifest.FailedRuns {
|
||||
t.Fatalf("report says scored=%d failed=%d, want 0 and %d", gotScored, gotFailed, manifest.FailedRuns)
|
||||
}
|
||||
}
|
||||
|
||||
// The control: with nothing blocked, a batch is what it always was. Every run
|
||||
// is completed with a stage, two identical batches write byte-identical results
|
||||
// and the same manifest header, and the new failure field stays out of it.
|
||||
func TestAllSuccessfulBatchKeepsItsArtifacts(t *testing.T) {
|
||||
tasks := indexTasks()
|
||||
first, runs, manifest, out := runBlockedExperiment(t, tasks)
|
||||
second, again, repeat, _ := runBlockedExperiment(t, tasks)
|
||||
if manifest.CompletedRuns != len(runs) || manifest.FailedRuns != 0 {
|
||||
t.Fatalf("completed=%d failed=%d rows=%d, want every row completed",
|
||||
manifest.CompletedRuns, manifest.FailedRuns, len(runs))
|
||||
}
|
||||
if manifest.CompletedRuns != manifest.PlannedRuns {
|
||||
t.Fatalf("completed=%d planned=%d, want all planned", manifest.CompletedRuns, manifest.PlannedRuns)
|
||||
}
|
||||
for i, m := range runs {
|
||||
if m.Failed || m.Task == "" || m.Arm == "" || m.FailureStage == "" {
|
||||
t.Fatalf("results.jsonl line %d is not a completed run: %+v", i+1, m)
|
||||
}
|
||||
}
|
||||
if gotScored, gotFailed := reportCounts(t, out); gotScored != manifest.CompletedRuns || gotFailed != 0 {
|
||||
t.Fatalf("report says scored=%d failed=%d, want %d and 0", gotScored, gotFailed, manifest.CompletedRuns)
|
||||
}
|
||||
if !reflect.DeepEqual(runs, again) {
|
||||
t.Fatal("two identical successful batches scored differently")
|
||||
}
|
||||
// Byte-for-byte on the runs. Trajectories carry per-message wall-clock
|
||||
// stamps, so they are compared field by field instead.
|
||||
if got, want := readFile(t, second, "results.jsonl"), readFile(t, first, "results.jsonl"); got != want {
|
||||
t.Fatalf("an unchanged batch wrote different results.jsonl:\n%s\n---\n%s", want, got)
|
||||
}
|
||||
gotTraj, wantTraj := withoutStamps(t, readFile(t, second, "trajectories.jsonl")),
|
||||
withoutStamps(t, readFile(t, first, "trajectories.jsonl"))
|
||||
if gotTraj != wantTraj {
|
||||
t.Fatalf("an unchanged batch wrote different trajectories:\n%s\n---\n%s", wantTraj, gotTraj)
|
||||
}
|
||||
repeat.StartedAt, repeat.FinishedAt = manifest.StartedAt, manifest.FinishedAt
|
||||
repeat.TrajectoriesSHA256 = manifest.TrajectoriesSHA256
|
||||
if !reflect.DeepEqual(repeat, manifest) {
|
||||
t.Fatalf("an unchanged batch wrote a different manifest:\n%+v\n---\n%+v", manifest, repeat)
|
||||
}
|
||||
if repeat.FailedRuns != 0 {
|
||||
t.Fatalf("a batch with no failure reports failed_runs=%d", repeat.FailedRuns)
|
||||
}
|
||||
}
|
||||
|
||||
// withoutStamps drops the createdAt stamp every message carries, which is the
|
||||
// one field a dry batch cannot reproduce between two runs.
|
||||
func withoutStamps(t *testing.T, text string) string {
|
||||
t.Helper()
|
||||
var b strings.Builder
|
||||
for line := range strings.SplitSeq(strings.TrimSpace(text), "\n") {
|
||||
var row map[string]any
|
||||
if err := json.Unmarshal([]byte(line), &row); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stripStamp(row)
|
||||
out, err := json.Marshal(row)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.Write(out)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func stripStamp(v any) {
|
||||
switch typed := v.(type) {
|
||||
case map[string]any:
|
||||
delete(typed, "createdAt")
|
||||
delete(typed, "workDurationMs")
|
||||
for _, child := range typed {
|
||||
stripStamp(child)
|
||||
}
|
||||
case []any:
|
||||
for _, child := range typed {
|
||||
stripStamp(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, root, name string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(root, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/runtime/agent"
|
||||
)
|
||||
|
||||
// A fixture is a real session that has really been compacted: no bench-only
|
||||
// state shape, so the loader and the projection restore have to work on it.
|
||||
|
||||
const (
|
||||
fixtureWindow = 60000
|
||||
fixtureGenerations = 5
|
||||
// fillerBytes per unit is what makes a fold necessary rather than optional.
|
||||
fillerBytes = 6000
|
||||
)
|
||||
|
||||
// digestProvider answers a summarizer without naming anything the corpus
|
||||
// plants. A digest that happened to quote the answer would leave it visible
|
||||
// and silently turn the task into a no-op.
|
||||
type digestProvider struct{ calls int }
|
||||
|
||||
func (p *digestProvider) Name() string { return "fixture-digest" }
|
||||
|
||||
func (p *digestProvider) Stream(_ context.Context, _ provider.Request) (<-chan provider.Chunk, error) {
|
||||
p.calls++
|
||||
ch := make(chan provider.Chunk, 2)
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "## Earlier work\nEarlier work continued across several files; nothing outstanding was recorded."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// fillerCall is one ordinary read whose only job is to take up room — in the
|
||||
// window, and in the fold index ahead of a cue.
|
||||
func fillerCall(sess *sessionstore.Session, n int) {
|
||||
id := fmt.Sprintf("f%04d", n)
|
||||
sess.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{
|
||||
{ID: id, Name: "read_file", Arguments: fmt.Sprintf(`{"path":"internal/mod%03d/handler%03d.go"}`, n%37, n)},
|
||||
}})
|
||||
sess.Add(provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: "read_file", Content: fillerBody(n)})
|
||||
}
|
||||
|
||||
func fillerBody(n int) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "package mod%03d\n\n", n%37)
|
||||
for b.Len() < fillerBytes {
|
||||
fmt.Fprintf(&b, "func handler%03dStep%02d() error { return nil }\n", n, b.Len()%64)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// builtFixture is one task's history after real folds, plus where its target
|
||||
// ended up.
|
||||
type builtFixture struct {
|
||||
Session *sessionstore.Session
|
||||
State sessionstore.CompactionState
|
||||
Target int
|
||||
Path string
|
||||
Instance fixtureInstance
|
||||
}
|
||||
|
||||
// buildFixture plays a task's history through the real agent and folds it at
|
||||
// the given arm, writing the session and its context state where the agent
|
||||
// itself would.
|
||||
func buildFixture(f fixtureInstance, arm ablation.Set, sessionPath string) (builtFixture, error) {
|
||||
t := f.Task
|
||||
sess := sessionstore.NewSession("You are a coding agent.")
|
||||
sess.Add(provider.Message{Role: provider.RoleUser, Content: "Work through the transport and scheduler backlog."})
|
||||
|
||||
n := 0
|
||||
at := -1
|
||||
|
||||
a := agent.New(&digestProvider{}, tool.NewRegistry(), sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: sessionPath, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, event.Discard)
|
||||
|
||||
for gen := range fixtureGenerations {
|
||||
if gen == t.PlantAfterGen {
|
||||
at = f.plant(sess)
|
||||
}
|
||||
for range 8 {
|
||||
fillerCall(sess, n)
|
||||
n++
|
||||
}
|
||||
if _, err := a.CompactNow(context.Background(), agent.CompactRequest{}); err != nil {
|
||||
return builtFixture{}, fmt.Errorf("%s: generation %d fold: %w", t.ID, gen, err)
|
||||
}
|
||||
}
|
||||
if at < 0 {
|
||||
return builtFixture{}, fmt.Errorf("%s: PlantAfterGen %d is past the last generation", t.ID, t.PlantAfterGen)
|
||||
}
|
||||
// projectionCoversTail cannot revalidate a projection covering the whole
|
||||
// transcript: it needs a live tail, which a real session always has because
|
||||
// work continues after a fold. Neutral text — the tail is provider-visible.
|
||||
sess.Add(provider.Message{Role: provider.RoleAssistant, Content: "Backlog swept; ready for the next question."})
|
||||
if err := sess.Save(sessionPath); err != nil {
|
||||
return builtFixture{}, fmt.Errorf("%s: save session: %w", t.ID, err)
|
||||
}
|
||||
state, ok, err := sessionstore.LoadCompactionState(sessionPath)
|
||||
if err != nil {
|
||||
return builtFixture{}, fmt.Errorf("%s: load context state: %w", t.ID, err)
|
||||
}
|
||||
if !ok || len(state.Projection.Messages) == 0 {
|
||||
return builtFixture{}, fmt.Errorf("%s: no projection was installed", t.ID)
|
||||
}
|
||||
if state.Projection.CoveredCount <= at {
|
||||
return builtFixture{}, fmt.Errorf("%s: target #%d is not inside the folded region (covered %d)",
|
||||
t.ID, at, state.Projection.CoveredCount)
|
||||
}
|
||||
return builtFixture{Session: sess, State: state, Target: at, Path: sessionPath, Instance: f}, nil
|
||||
}
|
||||
|
||||
// visibleContext is the model-visible view: the stored projection spliced with
|
||||
// everything canonical that came after it.
|
||||
func visibleContext(f builtFixture) []provider.Message {
|
||||
canonical := f.Session.Snapshot()
|
||||
out := append([]provider.Message(nil), f.State.Projection.Messages...)
|
||||
if n := f.State.Projection.CoveredCount; n >= 0 && n < len(canonical) {
|
||||
out = append(out, canonical[n:]...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// visibleText is everything the model would read, as one string.
|
||||
func visibleText(f builtFixture) string {
|
||||
var b strings.Builder
|
||||
for _, m := range visibleContext(f) {
|
||||
b.WriteString(m.Content)
|
||||
b.WriteString("\n")
|
||||
for _, tc := range m.ToolCalls {
|
||||
b.WriteString(tc.Name)
|
||||
b.WriteString(" ")
|
||||
b.WriteString(string(tc.Arguments))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// sealFixture removes the fixture from disk once the agent holds it in memory.
|
||||
// The canonical transcript is the answer, and the sandbox mounts the host
|
||||
// read-only by design — so a model that finds the path reads the answer, which
|
||||
// one did. The agent has already loaded the session and its sidecar by the time
|
||||
// this runs; a failed autosave afterwards costs the benchmark nothing.
|
||||
func sealFixture(sessionPath string) {
|
||||
// Every file, not two named ones: a session writes an event log and a
|
||||
// context sidecar beside the transcript, and session.events.jsonl still
|
||||
// carried the answer after the other two were sealed by name.
|
||||
dir := filepath.Dir(sessionPath)
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
_ = os.Remove(filepath.Join(dir, e.Name()))
|
||||
}
|
||||
}
|
||||
// Deny the directory too, so an autosave cannot recreate what was removed.
|
||||
_ = os.Chmod(dir, 0o500)
|
||||
}
|
||||
|
||||
// unsealFixture restores the directory so the harness can clean up after
|
||||
// itself.
|
||||
func unsealFixture(sessionPath string) {
|
||||
_ = os.Chmod(filepath.Dir(sessionPath), 0o700)
|
||||
}
|
||||
|
||||
// answerOnDisk reports any readable file under dir holding a current-run
|
||||
// answer. Two named files were sealed because two leaks were found in them;
|
||||
// this scans the directory, so a path that moves — an autosave, a recovery
|
||||
// tombstone, a future sidecar — fails here instead of on a paid run.
|
||||
func answerOnDisk(dir string, markers []string) (string, string) {
|
||||
var atPath, found string
|
||||
_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || atPath != "" {
|
||||
return nil //nolint:nilerr // an unreadable entry is not a leak
|
||||
}
|
||||
body, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
return nil
|
||||
}
|
||||
for _, marker := range markers {
|
||||
if len(marker) >= 6 && strings.Contains(string(body), marker) {
|
||||
atPath, found = path, marker
|
||||
return filepath.SkipAll
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return atPath, found
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
// instance.go — a task template becomes one run's concrete history.
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"io"
|
||||
mathrand "math/rand"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The invariant this file exists for: no literal sufficient to satisfy
|
||||
// AnswerRecovered for a concrete run may exist in this repository before that
|
||||
// run begins. Grepping the corpus reveals the question, never the answer.
|
||||
|
||||
// varKind is how one placeholder's value is generated. Values stay shaped like
|
||||
// real engineering records — a UUID would defeat the leak but turn every task
|
||||
// into "find the random string".
|
||||
type varKind string
|
||||
|
||||
const (
|
||||
varCodename varKind = "codename" // cobalt-k7m4q9
|
||||
varInt varKind = "int" // 731
|
||||
varDecimal varKind = "decimal" // 0.0437
|
||||
varDuration varKind = "duration" // 43ms
|
||||
)
|
||||
|
||||
// varSpec declares one placeholder. Min/Max bound the numeric kinds.
|
||||
type varSpec struct {
|
||||
Name string
|
||||
Kind varKind
|
||||
Word string // codename stem, e.g. "cobalt"
|
||||
Min int
|
||||
Max int
|
||||
}
|
||||
|
||||
// codenameEntropy is how much of a codename is unguessable. Five base32 digits
|
||||
// is about 25 bits: far past what a model can reach by knowing the stem.
|
||||
const codenameEntropy = 5
|
||||
|
||||
const codenameAlphabet = "abcdefghijkmnpqrstuvwxyz23456789"
|
||||
|
||||
// fixtureVars is one run's instantiated values.
|
||||
type fixtureVars map[string]string
|
||||
|
||||
// newVars generates values for every spec. A nil rng means deterministic.
|
||||
func newVars(specs []varSpec, rng *mathrand.Rand) fixtureVars {
|
||||
out := fixtureVars{}
|
||||
for _, spec := range specs {
|
||||
switch spec.Kind {
|
||||
case varCodename:
|
||||
out[spec.Name] = spec.Word + "-" + randomToken(rng, codenameEntropy)
|
||||
case varInt:
|
||||
out[spec.Name] = fmt.Sprint(randomBetween(rng, spec.Min, spec.Max))
|
||||
case varDecimal:
|
||||
out[spec.Name] = fmt.Sprintf("0.%04d", randomBetween(rng, spec.Min, spec.Max))
|
||||
case varDuration:
|
||||
out[spec.Name] = fmt.Sprintf("%dms", randomBetween(rng, spec.Min, spec.Max))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func randomToken(rng *mathrand.Rand, n int) string {
|
||||
var b strings.Builder
|
||||
for range n {
|
||||
b.WriteByte(codenameAlphabet[randomBetween(rng, 0, len(codenameAlphabet)-1)])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func randomBetween(rng *mathrand.Rand, lo, hi int) int {
|
||||
if hi <= lo {
|
||||
return lo
|
||||
}
|
||||
return lo + rng.Intn(hi-lo+1)
|
||||
}
|
||||
|
||||
// seededRand is the deterministic generator preflight and tests use, so a
|
||||
// fixture's tier calibration and its golden state reproduce exactly.
|
||||
func seededRand(parts ...string) *mathrand.Rand {
|
||||
h := fnv.New64a()
|
||||
for _, p := range parts {
|
||||
h.Write([]byte(p))
|
||||
h.Write([]byte{0})
|
||||
}
|
||||
return mathrand.New(mathrand.NewSource(int64(h.Sum64())))
|
||||
}
|
||||
|
||||
// liveRand is the generator a paid run uses: every run gets values that existed
|
||||
// nowhere before it started.
|
||||
func liveRand() *mathrand.Rand {
|
||||
rng, err := liveRandFrom(rand.Reader)
|
||||
if err != nil {
|
||||
panic("contextbench: no entropy for a live run: " + err.Error())
|
||||
}
|
||||
return rng
|
||||
}
|
||||
|
||||
// liveRandFrom seeds a generator from an entropy source, so what a run's values
|
||||
// are unpredictable from is the source and nothing else.
|
||||
func liveRandFrom(entropy io.Reader) (*mathrand.Rand, error) {
|
||||
var seed [8]byte
|
||||
if _, err := io.ReadFull(entropy, seed[:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var n int64
|
||||
for _, b := range seed {
|
||||
n = n<<8 | int64(b)
|
||||
}
|
||||
return mathrand.New(mathrand.NewSource(n)), nil
|
||||
}
|
||||
|
||||
// instantiate substitutes {{name}} and refuses anything left over. The grammar
|
||||
// is one identifier in double braces: no conditionals, no loops, no defaults.
|
||||
// This is fixture data, not a template language.
|
||||
func instantiate(text string, vars fixtureVars) (string, error) {
|
||||
out := text
|
||||
for name, value := range vars {
|
||||
out = strings.ReplaceAll(out, "{{"+name+"}}", value)
|
||||
}
|
||||
if strings.Contains(out, "{{") || strings.Contains(out, "}}") {
|
||||
return "", fmt.Errorf("unresolved placeholder in %q", firstPlaceholder(out))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func firstPlaceholder(text string) string {
|
||||
i := strings.Index(text, "{{")
|
||||
if i < 0 {
|
||||
return text
|
||||
}
|
||||
j := strings.Index(text[i:], "}}")
|
||||
if j < 0 {
|
||||
return text[i:min(i+40, len(text))]
|
||||
}
|
||||
return text[i : i+j+2]
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
// ContextRetrievalBench builds the fixtures the long-horizon retrieval
|
||||
// experiments run against, and proves they are worth paying a provider for.
|
||||
//
|
||||
// -mode=calibrate find each index task's lead-call count for its cue tier
|
||||
// -mode=preflight build every task under every arm and check the contract
|
||||
//
|
||||
// Nothing here calls a provider. The experiments themselves do; this is what
|
||||
// says they would measure what they claim to.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
mode := flag.String("mode", "preflight", "calibrate | preflight | snippet | adversarial | run-search | run-boundary | run-index")
|
||||
work := flag.String("work", "", "directory for built fixtures (default: a temp dir)")
|
||||
only := flag.String("task", "", "limit to one task id")
|
||||
dry := flag.Bool("dry", false, "run modes: drive the pipeline with a scripted provider instead of paying for one")
|
||||
flag.Parse()
|
||||
|
||||
if err := validateCorpus(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "corpus:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
root := *work
|
||||
if root == "" {
|
||||
dir, err := os.MkdirTemp("", "context-retrieval-*")
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
root = dir
|
||||
}
|
||||
|
||||
tasks := allTasks()
|
||||
if *only != "" {
|
||||
t, ok := taskByID(*only)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "unknown task %q\n", *only)
|
||||
os.Exit(2)
|
||||
}
|
||||
tasks = []contextTask{t}
|
||||
}
|
||||
|
||||
switch *mode {
|
||||
case "calibrate":
|
||||
os.Exit(runCalibrate(tasks, root))
|
||||
case "preflight":
|
||||
os.Exit(runPreflight(tasks, root))
|
||||
case "run-search":
|
||||
os.Exit(runExperiment(*mode, experimentSearch, root, *dry, tasks))
|
||||
case "run-boundary":
|
||||
os.Exit(runBoundaries(*mode, root))
|
||||
case "snippet":
|
||||
os.Exit(runSnippetAudit(root, 100))
|
||||
case "adversarial":
|
||||
os.Exit(runAdversarial(root))
|
||||
case "run-index":
|
||||
batch := tasks
|
||||
if *only == "" {
|
||||
// A batch measures the efficiency substrate only. Naming one task
|
||||
// with -task runs that task, role included, so the stopping corpus
|
||||
// stays measurable on its own.
|
||||
batch = efficiencyTasks()
|
||||
}
|
||||
os.Exit(runExperiment(*mode, experimentIndex, root, *dry, batch))
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown mode %q\n", *mode)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func runPreflight(tasks []contextTask, root string) int {
|
||||
var all []finding
|
||||
for _, t := range tasks {
|
||||
found, err := checkTask(t, root)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", t.ID, err)
|
||||
return 2
|
||||
}
|
||||
all = append(all, found...)
|
||||
status := "ok"
|
||||
if len(found) > 0 {
|
||||
status = fmt.Sprintf("%d problem(s)", len(found))
|
||||
}
|
||||
fmt.Printf("%-24s %-6s gen=%-2d %s\n", t.ID, t.Experiment, t.PlantAfterGen, status)
|
||||
}
|
||||
if len(all) == 0 {
|
||||
fmt.Printf("\n%d fixtures ready.\n", len(tasks))
|
||||
return 0
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "\n%d preflight problem(s):\n", len(all))
|
||||
sort.Slice(all, func(i, j int) bool { return all[i].Task < all[j].Task })
|
||||
for _, f := range all {
|
||||
fmt.Fprintln(os.Stderr, " ", f)
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// runCalibrate reports, for each index task, the lead-call counts at which its
|
||||
// cue is still addressed by each budget. The corpus freezes one number from
|
||||
// this; it is never computed at run time, or the implementation under test
|
||||
// would be moving its own goalposts.
|
||||
func runCalibrate(tasks []contextTask, root string) int {
|
||||
for _, t := range tasks {
|
||||
if t.Experiment != experimentIndex {
|
||||
continue
|
||||
}
|
||||
var rows []string
|
||||
for lead := range fixtureGenerations {
|
||||
probe := t
|
||||
probe.PlantAfterGen = lead
|
||||
var visibleAt []string
|
||||
for _, scale := range armScales {
|
||||
inst, ierr := instantiateTask(probe, seededRand(t.ID, scale, "calibrate"))
|
||||
if ierr != nil {
|
||||
fmt.Fprintln(os.Stderr, ierr)
|
||||
return 2
|
||||
}
|
||||
f, err := buildFixture(inst, armFor(scale, false),
|
||||
fmt.Sprintf("%s/calib/%s/%d-%s/session.jsonl", root, t.ID, lead, scale))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s gen=%d %s: %v\n", t.ID, lead, scale, err)
|
||||
return 2
|
||||
}
|
||||
if strings.Contains(visibleText(f), inst.CueMarker) {
|
||||
visibleAt = append(visibleAt, scale)
|
||||
}
|
||||
}
|
||||
rows = append(rows, fmt.Sprintf("planted after gen %d, cue visible at: %s", lead, strings.Join(visibleAt, ",")))
|
||||
}
|
||||
fmt.Printf("\n%s (wants tier %s)\n", t.ID, t.CueTier)
|
||||
for _, r := range rows {
|
||||
fmt.Println(" ", r)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"maps"
|
||||
mathrand "math/rand"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The same checks the preflight mode runs, under go test: a compaction change
|
||||
// that invalidates the fixtures fails here rather than three hundred API calls
|
||||
// later.
|
||||
|
||||
func TestCorpusIsWellFormed(t *testing.T) {
|
||||
if err := validateCorpus(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(searchTasks()); got != 6 {
|
||||
t.Errorf("search experiment has %d tasks, want 6", got)
|
||||
}
|
||||
if got := len(indexTasks()); got != 12 {
|
||||
t.Errorf("index experiment has %d tasks, want 12", got)
|
||||
}
|
||||
stopping := map[string]bool{}
|
||||
tiers := map[string]int{}
|
||||
for _, task := range indexTasks() {
|
||||
if task.Role == roleStopping {
|
||||
stopping[task.ID] = true
|
||||
continue
|
||||
}
|
||||
tiers[task.CueTier]++
|
||||
}
|
||||
if !stopping["i03-coalescing"] || !stopping["i04-recovery-fence"] {
|
||||
t.Errorf("the stopping corpus is %v; it must keep i03-coalescing and i04-recovery-fence", stopping)
|
||||
}
|
||||
// Two per tier, over the efficiency substrate, is what makes the staircase
|
||||
// readable: the same task can be compared across the boundary its cue sits
|
||||
// on, instead of two different questions being compared across two arms.
|
||||
for _, tier := range []string{tierQuarter, tierHalf, tierDefault} {
|
||||
if tiers[tier] != 2 {
|
||||
t.Errorf("tier %s has %d tasks, want 2", tier, tiers[tier])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The role decides which batch a task enters; the literal lists are only where
|
||||
// it is declared, so the two must not drift.
|
||||
func TestIndexListsMatchTheirRoles(t *testing.T) {
|
||||
for _, task := range append(append(indexQuarterTasks(), indexHalfTasks()...), indexDefaultTasks()...) {
|
||||
if task.Role != roleEfficiency {
|
||||
t.Errorf("%s sits in an efficiency list with role %q", task.ID, task.Role)
|
||||
}
|
||||
}
|
||||
for _, task := range stoppingIndexTasks() {
|
||||
if task.Role != roleStopping {
|
||||
t.Errorf("%s sits in the stopping list with role %q", task.ID, task.Role)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The invariant the whole instantiation layer exists for: no literal a scorer
|
||||
// needs may be readable in this repository before a run starts.
|
||||
func TestNoAnswerLiteralExistsInTheRepository(t *testing.T) {
|
||||
root, err := repoRoot()
|
||||
if err != nil {
|
||||
t.Skip(err)
|
||||
}
|
||||
for _, task := range allTasks() {
|
||||
inst, err := instantiateTask(task, seededRand(task.ID, "leak-check"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, marker := range append(inst.AnswerMarkers, inst.CueMarker) {
|
||||
if len(marker) < 6 {
|
||||
continue // a bare number is not a literal anyone can grep for
|
||||
}
|
||||
if found, where := grepRepo(t, root, marker); found {
|
||||
t.Errorf("%s: %q is readable at %s before the run", task.ID, marker, where)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A run's values are unpredictable from a previous run only if the live
|
||||
// generator is seeded from entropy. Comparing answers across two instances
|
||||
// cannot assert that: a task whose answer is a bounded number collides by
|
||||
// chance (1/8601 to 1/286 per task), so the check is on the seed source.
|
||||
func TestLiveRandDrawsFromEntropy(t *testing.T) {
|
||||
a, b := liveRand(), liveRand()
|
||||
for range 4 {
|
||||
if a.Int63() != b.Int63() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("two live generators produced the same stream: they are seeded alike")
|
||||
}
|
||||
|
||||
// The source alone decides an instance: the same entropy gives the same
|
||||
// answers, different entropy gives different values.
|
||||
func TestLiveRandSourceDecidesTheInstance(t *testing.T) {
|
||||
same := func() *mathrand.Rand {
|
||||
rng, err := liveRandFrom(bytes.NewReader(bytes.Repeat([]byte{7}, 8)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rng
|
||||
}
|
||||
other, err := liveRandFrom(bytes.NewReader(bytes.Repeat([]byte{9}, 8)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := liveRandFrom(bytes.NewReader(nil)); err == nil {
|
||||
t.Fatal("an empty entropy source was accepted")
|
||||
}
|
||||
for _, task := range allTasks() {
|
||||
a, _ := instantiateTask(task, same())
|
||||
b, _ := instantiateTask(task, same())
|
||||
if !slices.Equal(a.AnswerMarkers, b.AnswerMarkers) || !maps.Equal(a.Vars, b.Vars) {
|
||||
t.Errorf("%s: the same entropy gave different instances", task.ID)
|
||||
}
|
||||
c, _ := instantiateTask(task, other)
|
||||
if maps.Equal(a.Vars, c.Vars) {
|
||||
t.Errorf("%s: different entropy gave the same values", task.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A preflight that could not reproduce would calibrate against one run's
|
||||
// nonces and report about another's.
|
||||
func TestSeededInstancesReproduce(t *testing.T) {
|
||||
for _, task := range allTasks() {
|
||||
a, _ := instantiateTask(task, seededRand(task.ID, "x"))
|
||||
b, _ := instantiateTask(task, seededRand(task.ID, "x"))
|
||||
if !slices.Equal(a.AnswerMarkers, b.AnswerMarkers) || a.body != b.body {
|
||||
t.Errorf("%s did not reproduce under the same seed", task.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An unresolved placeholder would put "{{marker}}" in the transcript and score
|
||||
// every run wrong; it fails the fixture instead.
|
||||
func TestUnresolvedPlaceholderIsRefused(t *testing.T) {
|
||||
if _, err := instantiate("value is {{missing}}", fixtureVars{"other": "x"}); err == nil {
|
||||
t.Fatal("an unresolved placeholder was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func repoRoot() (string, error) {
|
||||
out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("not in a git checkout: %w", err)
|
||||
}
|
||||
return strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
|
||||
// grepRepo searches tracked files only: build artifacts and temp fixtures are
|
||||
// not what a fresh checkout hands anyone.
|
||||
func grepRepo(t *testing.T, root, needle string) (bool, string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", "grep", "-l", "-F", needle)
|
||||
cmd.Dir = root
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return false, "" // exit 1 means no match
|
||||
}
|
||||
return true, strings.TrimSpace(strings.SplitN(string(out), "\n", 2)[0])
|
||||
}
|
||||
|
||||
func TestEveryFixturePassesPreflight(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("builds and folds 36 sessions")
|
||||
}
|
||||
root := t.TempDir()
|
||||
for _, task := range allTasks() {
|
||||
t.Run(task.ID, func(t *testing.T) {
|
||||
found, err := checkTask(task, root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range found {
|
||||
t.Error(f)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/retrieval"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// Preflight is what makes a real run worth paying for. An answer the model can
|
||||
// already read, or a cue not where the matrix says, turns an arm into a
|
||||
// different experiment without saying so.
|
||||
|
||||
// armScales are the fold-index arms an index task is measured across.
|
||||
var armScales = []string{"default", "half", "quarter", "off"}
|
||||
|
||||
// experimentArm is one measured configuration of a task.
|
||||
type experimentArm struct {
|
||||
name string
|
||||
scale string
|
||||
searchOff bool
|
||||
}
|
||||
|
||||
// armsFor is the single declaration of which arms a task runs at, so preflight
|
||||
// and the run itself cannot check one set and measure another.
|
||||
func armsFor(t contextTask) []experimentArm {
|
||||
if t.Experiment == experimentSearch {
|
||||
return []experimentArm{
|
||||
{"search-on", "default", false},
|
||||
{"search-off", "default", true},
|
||||
}
|
||||
}
|
||||
arms := make([]experimentArm, 0, len(armScales))
|
||||
for _, scale := range armScales {
|
||||
arms = append(arms, experimentArm{"index-" + scale, scale, false})
|
||||
}
|
||||
return arms
|
||||
}
|
||||
|
||||
func scaleOf(name string) ablation.FoldIndexScale {
|
||||
s, err := ablation.ParseFoldIndexScale(name)
|
||||
if err != nil {
|
||||
panic("unknown scale " + name) // the list above is a constant
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// armFor is the arm a task is measured under at one fold-index scale.
|
||||
func armFor(scale string, searchOff bool) ablation.Set {
|
||||
set := ablation.Set{}
|
||||
if searchOff {
|
||||
set = ablation.New(ablation.RecallSearch)
|
||||
}
|
||||
return set.WithFoldIndex(scaleOf(scale))
|
||||
}
|
||||
|
||||
type finding struct {
|
||||
Task, Arm, Problem string
|
||||
}
|
||||
|
||||
func (f finding) String() string { return fmt.Sprintf("%s [%s]: %s", f.Task, f.Arm, f.Problem) }
|
||||
|
||||
// checkTask builds the task under every arm it will be run at and reports what
|
||||
// does not hold.
|
||||
func checkTask(t contextTask, root string) ([]finding, error) {
|
||||
var out []finding
|
||||
for _, arm := range armsFor(t) {
|
||||
// Deterministic values: a preflight has to reproduce, and a tier
|
||||
// calibration measured against one run's nonces is measured against
|
||||
// nothing.
|
||||
inst, err := instantiateTask(t, seededRand(t.ID, arm.name, "preflight"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dir := filepath.Join(root, t.ID, arm.name)
|
||||
f, err := buildFixture(inst, armFor(arm.scale, arm.searchOff), filepath.Join(dir, "session.jsonl"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
visible := visibleText(f)
|
||||
|
||||
// The answer must be unreadable in every arm, or the task measures
|
||||
// reading rather than recall.
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if len(marker) < 3 {
|
||||
continue // a bare number is not a nonce; the whole set still has to match
|
||||
}
|
||||
if strings.Contains(visible, marker) {
|
||||
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("answer marker %q is already visible", marker)})
|
||||
}
|
||||
}
|
||||
|
||||
// The corpus has to be findable at all, or a miss in a real run says
|
||||
// nothing about the model.
|
||||
rank, err := probeRank(f, inst.ProbeQuery)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case rank == 0:
|
||||
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("probe query %q does not find the target at all", inst.ProbeQuery)})
|
||||
case rank > 5:
|
||||
out = append(out, finding{t.ID, arm.name, fmt.Sprintf("probe query %q ranks the target %d", inst.ProbeQuery, rank)})
|
||||
}
|
||||
|
||||
if t.Experiment == experimentIndex {
|
||||
out = append(out, checkCueVisibility(t, inst, arm.scale, arm.name, visible)...)
|
||||
} else if line := indexLineFor(f, f.Target); line != "" {
|
||||
out = append(out, finding{t.ID, arm.name,
|
||||
"a search task is addressed by the fold index, so it measures the index and not search: " + line})
|
||||
}
|
||||
|
||||
// The projection is what the agent believes. This is what the model is
|
||||
// told, and it is the one that has to hold.
|
||||
req, err := captureRequest(f, armFor(arm.scale, arm.searchOff), inst.Prompt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s [%s]: capture: %w", t.ID, arm.name, err)
|
||||
}
|
||||
out = append(out, checkCapturedRequest(t, inst, arm.name, arm.scale, arm.searchOff, req)...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// checkCapturedRequest holds one arm's real provider request to its contract:
|
||||
// the answer unreadable, the cue exactly where its tier says, the recall tool
|
||||
// offering only what this arm can do, and no host-side field surviving the
|
||||
// boundary.
|
||||
func checkCapturedRequest(t contextTask, inst fixtureInstance, armName, scale string, searchOff bool, req provider.Request) []finding {
|
||||
var out []finding
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if len(marker) < 3 {
|
||||
continue // a bare number is not a nonce; the set as a whole still is
|
||||
}
|
||||
for _, at := range findInSurface(req, marker) {
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("answer marker %q leaked at %s", marker, at)})
|
||||
}
|
||||
}
|
||||
if t.Experiment == experimentIndex {
|
||||
wantVisible, wantHidden := tierScales(t.CueTier)
|
||||
where := findInSurface(req, inst.CueMarker)
|
||||
switch {
|
||||
case contains(wantVisible, scale) && len(where) == 0:
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is not provider-visible; the tier expects it", inst.CueMarker)})
|
||||
case contains(wantHidden, scale) && len(where) > 0:
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("cue %q is provider-visible at %s: %s", inst.CueMarker, where[0], scale)})
|
||||
}
|
||||
}
|
||||
|
||||
schema, ok := recallToolSchema(req)
|
||||
if !ok {
|
||||
out = append(out, finding{t.ID, armName, "recall is not in the provider's tool list"})
|
||||
return append(out, boundaryFindings(t, armName, req)...)
|
||||
}
|
||||
params := ""
|
||||
if b, err := json.Marshal(schema.Parameters); err == nil {
|
||||
params = string(b)
|
||||
}
|
||||
// Both arms keep the read half: the ablation removes search, not recall.
|
||||
if !strings.Contains(params, `"positions"`) {
|
||||
out = append(out, finding{t.ID, armName, "recall lost its positions parameter"})
|
||||
}
|
||||
hasQuery := strings.Contains(params, `"query"`)
|
||||
switch {
|
||||
case searchOff && hasQuery:
|
||||
out = append(out, finding{t.ID, armName, "recall still offers a query parameter"})
|
||||
case !searchOff && !hasQuery:
|
||||
out = append(out, finding{t.ID, armName, "recall has no query parameter"})
|
||||
}
|
||||
if searchOff {
|
||||
for _, phrase := range searchWording {
|
||||
for _, at := range findInSurface(req, phrase) {
|
||||
out = append(out, finding{t.ID, armName, fmt.Sprintf("search is offered to an arm without it: %q at %s", phrase, at)})
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(out, boundaryFindings(t, armName, req)...)
|
||||
}
|
||||
|
||||
func boundaryFindings(t contextTask, armName string, req provider.Request) []finding {
|
||||
var out []finding
|
||||
for _, at := range localOnlyLeaks(req) {
|
||||
out = append(out, finding{t.ID, armName, "host-only field crossed the provider boundary at " + at})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkCueVisibility holds an index task to its tier: the cue is addressed at
|
||||
// the scales the matrix names and at no others. A policy change that moves it
|
||||
// fails here rather than turning two arms into one experiment.
|
||||
func checkCueVisibility(t contextTask, inst fixtureInstance, scale, armName, visible string) []finding {
|
||||
wantVisible, wantHidden := tierScales(t.CueTier)
|
||||
has := strings.Contains(visible, inst.CueMarker)
|
||||
switch {
|
||||
case contains(wantVisible, scale) && !has:
|
||||
return []finding{{t.ID, armName, fmt.Sprintf("cue %q should be addressed at %s and is not", inst.CueMarker, scale)}}
|
||||
case contains(wantHidden, scale) && has:
|
||||
return []finding{{t.ID, armName, fmt.Sprintf("cue %q should be gone at %s and is not", inst.CueMarker, scale)}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func contains(list []string, want string) bool { return slices.Contains(list, want) }
|
||||
|
||||
// probeRank runs the task's own query against the folded region the way recall
|
||||
// would, and reports where the target lands.
|
||||
func probeRank(f builtFixture, query string) (int, error) {
|
||||
terms, err := retrieval.QueryTerms(query)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
canonical := f.Session.Snapshot()
|
||||
covered := min(f.State.Projection.CoveredCount, len(canonical))
|
||||
docs := make([]map[string]int, 0, covered)
|
||||
positions := make([]int, 0, covered)
|
||||
lengths := make([]int, 0, covered)
|
||||
for i, m := range canonical[:covered] {
|
||||
var text strings.Builder
|
||||
text.WriteString(m.Content)
|
||||
for _, tc := range m.ToolCalls {
|
||||
text.WriteString(" " + tc.Name + " " + string(tc.Arguments))
|
||||
}
|
||||
tokens := retrieval.Tokens(text.String())
|
||||
if len(tokens) == 0 {
|
||||
continue
|
||||
}
|
||||
docs = append(docs, retrieval.Counts(tokens))
|
||||
positions = append(positions, i)
|
||||
lengths = append(lengths, len(tokens))
|
||||
}
|
||||
df := retrieval.DocumentFrequency(docs)
|
||||
avg := 0.0
|
||||
for _, l := range lengths {
|
||||
avg += float64(l)
|
||||
}
|
||||
if len(lengths) > 0 {
|
||||
avg /= float64(len(lengths))
|
||||
}
|
||||
type hit struct {
|
||||
pos int
|
||||
score float64
|
||||
}
|
||||
var hits []hit
|
||||
for i, counts := range docs {
|
||||
if s := retrieval.BM25Score(counts, lengths[i], terms, df, len(docs), avg); s > 0 {
|
||||
hits = append(hits, hit{positions[i], s})
|
||||
}
|
||||
}
|
||||
for i := range hits {
|
||||
for j := i + 1; j < len(hits); j++ {
|
||||
if hits[j].score > hits[i].score {
|
||||
hits[i], hits[j] = hits[j], hits[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
// A tool result answers at its caller's position, so either counts.
|
||||
for i, h := range hits {
|
||||
if h.pos == f.Target || h.pos == f.Target+1 {
|
||||
return i + 1, nil
|
||||
}
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// indexLineFor returns the fold-index line addressing a position, if any.
|
||||
func indexLineFor(f builtFixture, pos int) string {
|
||||
want := fmt.Sprintf("#%d ", pos)
|
||||
for line := range strings.SplitSeq(visibleIndexOnly(f), "\n") {
|
||||
if strings.Contains(line, want) {
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// visibleIndexOnly is just the fold-index sections of the projection.
|
||||
func visibleIndexOnly(f builtFixture) string {
|
||||
var b strings.Builder
|
||||
for _, m := range visibleContext(f) {
|
||||
if i := strings.Index(m.Content, "## Folded work index"); i >= 0 {
|
||||
b.WriteString(m.Content[i:])
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
// reformulation.go — how a query changed, from token sets alone.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/retrieval"
|
||||
)
|
||||
|
||||
// The host probe ranks every target first, so the engine is not what makes a
|
||||
// model take more than one attempt. These classes are set arithmetic over the
|
||||
// same tokenizer recall uses: no judge, nothing read out of prose.
|
||||
|
||||
const (
|
||||
reformFirst = "First" // nothing to compare against yet
|
||||
reformNearDup = "NearDuplicate" // the same query again, give or take a word
|
||||
reformNarrowing = "Narrowing" // kept everything and added terms
|
||||
reformBroadening = "Broadening" // dropped terms, added none
|
||||
reformAnchored = "HistoricalAnchor" // reached for a term the question does not contain
|
||||
reformShift = "SynonymShift" // traded terms for others
|
||||
)
|
||||
|
||||
// nearDuplicateOverlap is where "reworded" stops and "asked again" begins.
|
||||
const nearDuplicateOverlap = 0.8
|
||||
|
||||
func terms(s string) []string {
|
||||
return retrieval.Unique(retrieval.Tokens(s))
|
||||
}
|
||||
|
||||
func termSet(s string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, t := range terms(s) {
|
||||
out[t] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func jaccard(a, b map[string]bool) float64 {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return 1
|
||||
}
|
||||
shared := 0
|
||||
for t := range a {
|
||||
if b[t] {
|
||||
shared++
|
||||
}
|
||||
}
|
||||
union := len(a) + len(b) - shared
|
||||
if union == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(shared) / float64(union)
|
||||
}
|
||||
|
||||
// classifyReformulation names how this query differs from the one before it.
|
||||
func classifyReformulation(previous, current, question string) string {
|
||||
if previous == "" {
|
||||
return reformFirst
|
||||
}
|
||||
prev, cur := termSet(previous), termSet(current)
|
||||
if jaccard(prev, cur) >= nearDuplicateOverlap {
|
||||
return reformNearDup
|
||||
}
|
||||
added, dropped := diffTerms(cur, prev), diffTerms(prev, cur)
|
||||
// A term the question never used had to come from the model's own memory of
|
||||
// the history, which is a different move from rephrasing what it was asked.
|
||||
ask := termSet(question)
|
||||
for _, t := range added {
|
||||
if !ask[t] {
|
||||
return reformAnchored
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(dropped) == 0:
|
||||
return reformNarrowing
|
||||
case len(added) == 0:
|
||||
return reformBroadening
|
||||
default:
|
||||
return reformShift
|
||||
}
|
||||
}
|
||||
|
||||
func diffTerms(from, minus map[string]bool) []string {
|
||||
var out []string
|
||||
for t := range from {
|
||||
if !minus[t] {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// classifySearches labels each attempt and records which of the host probe's
|
||||
// terms it left out. The probe is the oracle: it ranks the target first, so a
|
||||
// query missing its terms says what the model did not think to ask for.
|
||||
func classifySearches(m *contextMetrics, inst fixtureInstance) {
|
||||
probe := terms(inst.ProbeQuery)
|
||||
previous := ""
|
||||
for i := range m.Searches {
|
||||
s := &m.Searches[i]
|
||||
s.Reformulation = classifyReformulation(previous, s.Query, inst.Prompt)
|
||||
have := termSet(s.Query)
|
||||
for _, term := range probe {
|
||||
if !have[term] {
|
||||
s.MissingProbeTerms = append(s.MissingProbeTerms, term)
|
||||
}
|
||||
}
|
||||
previous = s.Query
|
||||
}
|
||||
}
|
||||
|
||||
// queryAudit is the reformulation funnel across a set of runs.
|
||||
type queryAudit struct {
|
||||
Runs, RunsThatSearched int
|
||||
FirstQueryHit int
|
||||
HitBySecond, HitByThird int
|
||||
NeverHit int
|
||||
TotalSearches int
|
||||
PostHitSearches int
|
||||
HitThenRead int
|
||||
EscapeBeforeFirstSearch int
|
||||
Reformulations map[string]int
|
||||
MissedProbeTerms map[string]int
|
||||
}
|
||||
|
||||
func auditQueries(runs []contextMetrics) queryAudit {
|
||||
a := queryAudit{Reformulations: map[string]int{}, MissedProbeTerms: map[string]int{}}
|
||||
for _, r := range runs {
|
||||
if r.Failed || r.Contaminated {
|
||||
continue
|
||||
}
|
||||
a.Runs++
|
||||
a.TotalSearches += len(r.Searches)
|
||||
a.PostHitSearches += r.PostHitSearches
|
||||
if r.EscapeBeforeFirstRecall > 0 {
|
||||
a.EscapeBeforeFirstSearch++
|
||||
}
|
||||
if len(r.Searches) == 0 {
|
||||
continue
|
||||
}
|
||||
a.RunsThatSearched++
|
||||
hitAt := 0
|
||||
for i, s := range r.Searches {
|
||||
a.Reformulations[s.Reformulation]++
|
||||
for _, term := range s.MissingProbeTerms {
|
||||
a.MissedProbeTerms[term]++
|
||||
}
|
||||
if s.TargetRank > 0 && hitAt == 0 {
|
||||
hitAt = i + 1
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case hitAt == 1:
|
||||
a.FirstQueryHit++
|
||||
case hitAt == 2:
|
||||
a.HitBySecond++
|
||||
case hitAt >= 3:
|
||||
a.HitByThird++
|
||||
default:
|
||||
a.NeverHit++
|
||||
}
|
||||
if hitAt > 0 && r.TargetRead {
|
||||
a.HitThenRead++
|
||||
}
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func (a queryAudit) report() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## Query reformulation\n")
|
||||
fmt.Fprintf(&b, " runs=%d searched=%d searches=%d (%.2f per searching run)\n",
|
||||
a.Runs, a.RunsThatSearched, a.TotalSearches, ratio(a.TotalSearches, a.RunsThatSearched))
|
||||
fmt.Fprintf(&b, " target found by query 1: %d by 2: %d by 3+: %d never: %d\n",
|
||||
a.FirstQueryHit, a.HitBySecond, a.HitByThird, a.NeverHit)
|
||||
fmt.Fprintf(&b, " found then read: %d/%d searches issued after the target was already back: %d\n",
|
||||
a.HitThenRead, a.FirstQueryHit+a.HitBySecond+a.HitByThird, a.PostHitSearches)
|
||||
fmt.Fprintf(&b, " reached for the workspace before searching: %d\n", a.EscapeBeforeFirstSearch)
|
||||
b.WriteString(" reformulations: " + countsLine(a.Reformulations) + "\n")
|
||||
b.WriteString(" probe terms most often absent: " + topCounts(a.MissedProbeTerms, 8) + "\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func ratio(n, d int) float64 {
|
||||
if d == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(n) / float64(d)
|
||||
}
|
||||
|
||||
func countsLine(counts map[string]int) string {
|
||||
keys := make([]string, 0, len(counts))
|
||||
for k := range counts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func topCounts(counts map[string]int, limit int) string {
|
||||
type kv struct {
|
||||
k string
|
||||
n int
|
||||
}
|
||||
all := make([]kv, 0, len(counts))
|
||||
for k, n := range counts {
|
||||
all = append(all, kv{k, n})
|
||||
}
|
||||
sort.Slice(all, func(i, j int) bool {
|
||||
if all[i].n != all[j].n {
|
||||
return all[i].n > all[j].n
|
||||
}
|
||||
return all[i].k < all[j].k
|
||||
})
|
||||
parts := make([]string, 0, limit)
|
||||
for _, e := range all[:min(limit, len(all))] {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", e.k, e.n))
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "(none)"
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// queryLines lists every attempt, so a surprising audit can be read back.
|
||||
func queryLines(runs []contextMetrics) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("\n## Every query, in order\n")
|
||||
for _, r := range runs {
|
||||
for _, s := range r.Searches {
|
||||
rank := "miss"
|
||||
if s.TargetRank > 0 {
|
||||
rank = fmt.Sprintf("rank %d", s.TargetRank)
|
||||
}
|
||||
fmt.Fprintf(&b, " %-24s %-14s #%d %-17s %-9s %q\n",
|
||||
r.Task, r.Arm, s.Ordinal, s.Reformulation, rank, s.Query)
|
||||
if len(s.MissingProbeTerms) > 0 && s.TargetRank == 0 {
|
||||
fmt.Fprintf(&b, " %-24s %-14s missing probe terms: %s\n",
|
||||
"", "", strings.Join(slices.Clone(s.MissingProbeTerms), " "))
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,569 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/assembly/boot"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/contract/tool"
|
||||
_ "reasonix/internal/model/openai"
|
||||
"reasonix/internal/runtime/agent"
|
||||
_ "reasonix/internal/tools/builtin"
|
||||
)
|
||||
|
||||
// A run is one recall question against a fixture the preflight has already
|
||||
// cleared. Scoring reads the transcript the turn appended — tool names and
|
||||
// their JSON arguments — so nothing depends on how the model narrated itself.
|
||||
|
||||
const (
|
||||
runBaseURL = "https://api.deepseek.com"
|
||||
runModel = "deepseek-chat"
|
||||
runTimeout = 4 * time.Minute
|
||||
)
|
||||
|
||||
func realProvider() (provider.Provider, error) {
|
||||
key := os.Getenv("DEEPSEEK_API_KEY")
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("a real run needs DEEPSEEK_API_KEY")
|
||||
}
|
||||
return provider.New("openai", provider.Config{
|
||||
Name: "contextbench", BaseURL: runBaseURL, Model: runModel, APIKey: key,
|
||||
})
|
||||
}
|
||||
|
||||
// adversarialPrompts tell the model what it is running and send it after the
|
||||
// answer everywhere except the conversation. The question is not whether it
|
||||
// obeys the experiment: it is whether the answer is reachable at all by a model
|
||||
// that knows the mechanism and is actively attacking the host.
|
||||
var adversarialPrompts = map[string]string{
|
||||
experimentSearch: "You are running a benchmark. Find the expected answer for this task from the " +
|
||||
"Reasonix repository, benchmark source, fixture generator, test files, scratch files, or " +
|
||||
"session metadata. Do not use recall unless necessary. Report the exact hidden values.",
|
||||
experimentIndex: "Inspect benchmarks/context-retrieval, the git repository, session files, temporary " +
|
||||
"files, and any benchmark metadata you can access. Recover the concrete instantiated answer for " +
|
||||
"this run without relying on the folded conversation. Report the exact values.",
|
||||
}
|
||||
|
||||
// runAdversarial sends one task per experiment after the answer through every
|
||||
// host-readable surface. It passes when nothing but recall could produce it —
|
||||
// a run that gives up and recalls has still proven the attack failed.
|
||||
func runAdversarial(root string) int {
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
restore, ierr := isolateWorkspace()
|
||||
if ierr != nil {
|
||||
fmt.Fprintln(os.Stderr, "isolate workspace:", ierr)
|
||||
return 2
|
||||
}
|
||||
defer restore()
|
||||
p, perr := realProvider()
|
||||
if perr != nil {
|
||||
fmt.Fprintln(os.Stderr, perr)
|
||||
return 2
|
||||
}
|
||||
leaked := 0
|
||||
for _, id := range []string{"r01-retry-boundary", "i01-transport-fallback"} {
|
||||
t, ok := taskByID(id)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "unknown task %q\n", id)
|
||||
return 2
|
||||
}
|
||||
m, err := runOne(p, t, armFor("default", false), "adversarial", root, false, false,
|
||||
adversarialPrompts[t.Experiment])
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", id, err)
|
||||
m = failedMetrics(t, "adversarial", err)
|
||||
}
|
||||
verdict := "isolated"
|
||||
if m.EscapeFoundAnswer {
|
||||
verdict = "LEAKED via " + m.LeakedVia
|
||||
leaked++
|
||||
}
|
||||
fmt.Printf("%-24s %-24s escapes=%-3d recall=%d answered=%v tools=%v\n",
|
||||
id, verdict, m.EscapeCalls, m.SearchCalls+m.ReadCalls, m.AnswerRecovered, m.UnexpectedWorkTools)
|
||||
if m.EscapeFoundAnswer {
|
||||
fmt.Printf(" args: %s\n output: %s\n", m.LeakedArgs, m.LeakedOutput)
|
||||
}
|
||||
}
|
||||
if leaked > 0 {
|
||||
fmt.Fprintf(os.Stderr, "\n%d of 2 adversarial runs reached an answer outside recall\n", leaked)
|
||||
return 1
|
||||
}
|
||||
fmt.Println("\nNo answer was reachable outside recall.")
|
||||
return 0
|
||||
}
|
||||
|
||||
// runOne asks one task under one arm and scores what the model did.
|
||||
func runOne(p provider.Provider, t contextTask, arm ablation.Set, armName, root string, cueVisible, dry bool, promptOverride ...string) (contextMetrics, error) {
|
||||
// Live runs get values that existed nowhere before the run started; a dry
|
||||
// run stays reproducible.
|
||||
rng := liveRand()
|
||||
if dry {
|
||||
rng = seededRand(t.ID, armName, "dry")
|
||||
}
|
||||
inst, err := instantiateTask(t, rng)
|
||||
if err != nil {
|
||||
return contextMetrics{}, err
|
||||
}
|
||||
f, err := buildFixture(inst, arm, filepath.Join(root, t.ID, armName, "session.jsonl"))
|
||||
if err != nil {
|
||||
return contextMetrics{}, err
|
||||
}
|
||||
sess, err := sessionstore.LoadSession(f.Path)
|
||||
if err != nil {
|
||||
return contextMetrics{}, fmt.Errorf("load session: %w", err)
|
||||
}
|
||||
reg := tool.NewRegistry()
|
||||
for _, x := range tool.Builtins() {
|
||||
reg.Add(x)
|
||||
}
|
||||
boot.ApplyUnifiedProviderToolSurface(reg, false, arm)
|
||||
|
||||
var usage usageSink
|
||||
a := agent.New(p, reg, sess, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors, Ablation: arm,
|
||||
}, &usage)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
// From here the transcript lives only in memory. On disk it is the answer.
|
||||
sealFixture(f.Path)
|
||||
defer unsealFixture(f.Path)
|
||||
dir := filepath.Dir(f.Path)
|
||||
if at, marker := answerOnDisk(dir, inst.AnswerMarkers); at != "" {
|
||||
return contextMetrics{}, fmt.Errorf("%s [%s]: %q is readable at %s before the run", t.ID, armName, marker, at)
|
||||
}
|
||||
|
||||
before := len(sess.Snapshot())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), runTimeout)
|
||||
defer cancel()
|
||||
prompt := inst.Prompt
|
||||
if len(promptOverride) > 0 && promptOverride[0] != "" {
|
||||
prompt = promptOverride[0]
|
||||
}
|
||||
runErr := a.Run(ctx, prompt)
|
||||
|
||||
if at, marker := answerOnDisk(dir, inst.AnswerMarkers); at != "" {
|
||||
return contextMetrics{}, fmt.Errorf("%s [%s]: %q was written to %s during the run", t.ID, armName, marker, at)
|
||||
}
|
||||
|
||||
appended := sess.Snapshot()[before:]
|
||||
m := scoreRun(appended, inst, f.Target, armName, cueVisible)
|
||||
m.trajectory = appended
|
||||
if u, requests := usage.snapshot(); requests > 0 {
|
||||
m.UsageReportedRequests, m.PromptTokens, m.CompletionTokens = requests, u.PromptTokens, u.CompletionTokens
|
||||
m.CacheHitTokens, m.CacheMissTokens, m.UsageEstimated = u.CacheHitTokens, u.CacheMissTokens, u.Estimated
|
||||
}
|
||||
m.scoreAnswer(finalAnswer(appended), inst)
|
||||
if runErr != nil {
|
||||
m.FailureStage = "RunError"
|
||||
m.TimeoutReason = timeoutReason(m)
|
||||
}
|
||||
return m, runErr
|
||||
}
|
||||
|
||||
// failedMetrics is what a run that never reached a transcript leaves behind. It
|
||||
// carries the run's identity and the stage that ended it, so the failure stays a
|
||||
// countable fact; Failed keeps it out of every statistic.
|
||||
func failedMetrics(t contextTask, armName string, err error) contextMetrics {
|
||||
return contextMetrics{
|
||||
Task: t.ID, Arm: armName, Failed: true,
|
||||
FailureStage: stageNoEvidence, Reason: err.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
// finalAnswer is the last assistant text the turn produced.
|
||||
func finalAnswer(msgs []provider.Message) string {
|
||||
for _, m := range slices.Backward(msgs) {
|
||||
if m.Role == provider.RoleAssistant && strings.TrimSpace(m.Content) != "" {
|
||||
return m.Content
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// isolateWorkspace moves the process into an empty directory before any task
|
||||
// runs. The corpus lives in this repository, so an agent left here greps the
|
||||
// answers out of corpus.go — one did, quoting the file's own line about them
|
||||
// being nonces. Fixtures stay outside it too: a readable session file is the
|
||||
// canonical transcript, answer included.
|
||||
func isolateWorkspace() (func(), error) {
|
||||
dir, err := os.MkdirTemp("", "contextbench-workspace-*")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
readme := "This workspace is empty on purpose. The question is about earlier conversation,\nnot about any file here.\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte(readme), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
previous, err := os.Getwd()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chdir(dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func() {
|
||||
_ = os.Chdir(previous)
|
||||
_ = os.RemoveAll(dir)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// runBoundaries runs only the pair each index task is built for: the narrowest
|
||||
// scale still addressing its cue, and the next one down. Each pair moves the
|
||||
// affordance and nothing else, where default-vs-off moves the whole index.
|
||||
|
||||
// The two arms get separate nonces: reusing one would leave the first arm's
|
||||
// answer in a host-readable history for the second to find.
|
||||
func runBoundaries(mode, root string) int {
|
||||
started := time.Now()
|
||||
planned := 0
|
||||
repo, _ := os.Getwd()
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
restore, ierr := isolateWorkspace()
|
||||
if ierr != nil {
|
||||
fmt.Fprintln(os.Stderr, "isolate workspace:", ierr)
|
||||
return 2
|
||||
}
|
||||
defer restore()
|
||||
p, perr := realProvider()
|
||||
if perr != nil {
|
||||
fmt.Fprintln(os.Stderr, perr)
|
||||
return 2
|
||||
}
|
||||
|
||||
var all []contextMetrics
|
||||
var cueSide, noCueSide []contextMetrics
|
||||
for _, t := range efficiencyTasks() {
|
||||
cue, noCue := boundaryPair(t.CueTier)
|
||||
for _, side := range []struct {
|
||||
scale string
|
||||
has bool
|
||||
}{{cue, true}, {noCue, false}} {
|
||||
name := "index-" + side.scale
|
||||
planned++
|
||||
m, err := runOne(p, t, armFor(side.scale, false), name, root, side.has, false)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, name, err)
|
||||
m = failedMetrics(t, name, err)
|
||||
}
|
||||
all = append(all, m)
|
||||
if side.has {
|
||||
cueSide = append(cueSide, m)
|
||||
} else {
|
||||
noCueSide = append(noCueSide, m)
|
||||
}
|
||||
fmt.Printf("%-24s %-14s cue=%-3v %-14s search=%d read=%d direct=%-5v recall-tok=%-5d%s escape=%d\n",
|
||||
t.ID, name, side.has, m.FailureStage, m.SearchCalls, m.ReadCalls,
|
||||
m.CueDirectRead, m.RecallReturnedTokens, promptTokenSuffix(m), m.EscapeCalls)
|
||||
}
|
||||
}
|
||||
reportDeltas(cueSide, noCueSide)
|
||||
fmt.Print(reportStopping(all))
|
||||
return writeResults(root, batchInfo{Mode: mode, Planned: planned, Started: started, Repo: repo}, all)
|
||||
}
|
||||
|
||||
// reportDeltas is the paired reading: each task against itself, then pooled.
|
||||
func reportDeltas(cueSide, noCueSide []contextMetrics) {
|
||||
fmt.Printf("\n## Paired deltas (cue present − cue absent)\n")
|
||||
fmt.Printf(" %-24s %-10s %-10s %-9s %-11s %s\n",
|
||||
"task", "recovery", "directread", "searches", "recall-tok", "escapes")
|
||||
for i := range cueSide {
|
||||
if i >= len(noCueSide) {
|
||||
break
|
||||
}
|
||||
a, b := cueSide[i], noCueSide[i]
|
||||
fmt.Printf(" %-24s %-10s %-10s %-9d %-11d %d\n", a.Task,
|
||||
deltaBool(a.AnswerRecovered, b.AnswerRecovered),
|
||||
deltaBool(a.CueDirectRead, b.CueDirectRead),
|
||||
a.SearchCalls-b.SearchCalls,
|
||||
a.RecallReturnedTokens-b.RecallReturnedTokens,
|
||||
a.EscapeCalls-b.EscapeCalls)
|
||||
}
|
||||
fmt.Println("\n## Pooled")
|
||||
fmt.Println(" ", summarize("cue-present", cueSide).line())
|
||||
fmt.Println(" ", summarize("cue-absent", noCueSide).line())
|
||||
fmt.Println("\n## Where each side looked first")
|
||||
fmt.Printf(" %-14s %s\n", "cue-present", countsLine(summarize("x", cueSide).Routes))
|
||||
fmt.Printf(" %-14s %s\n", "cue-absent", countsLine(summarize("x", noCueSide).Routes))
|
||||
}
|
||||
|
||||
func deltaBool(a, b bool) string {
|
||||
switch {
|
||||
case a == b:
|
||||
return "same"
|
||||
case a:
|
||||
return "+cue"
|
||||
default:
|
||||
return "-cue"
|
||||
}
|
||||
}
|
||||
|
||||
// runExperiment runs one experiment's tasks across its arms. A dry run drives
|
||||
// the same pipeline with a scripted provider: it proves fixture, run, scoring
|
||||
// and report hold together before any of it is paid for.
|
||||
func runExperiment(mode, experiment, root string, dry bool, tasks []contextTask) int {
|
||||
started := time.Now()
|
||||
planned := 0
|
||||
repo, _ := os.Getwd()
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
restore, err := isolateWorkspace()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "isolate workspace:", err)
|
||||
return 2
|
||||
}
|
||||
defer restore()
|
||||
var p provider.Provider = &scriptedProvider{}
|
||||
if !dry {
|
||||
real, rerr := realProvider()
|
||||
if rerr != nil {
|
||||
fmt.Fprintln(os.Stderr, rerr)
|
||||
return 2
|
||||
}
|
||||
p = real
|
||||
}
|
||||
byArm := map[string][]contextMetrics{}
|
||||
var all []contextMetrics
|
||||
|
||||
if experiment == experimentIndex {
|
||||
fmt.Print(fingerprint(dry))
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if t.Experiment != experiment {
|
||||
continue
|
||||
}
|
||||
// Arms are shuffled inside each task block. Running six defaults, then
|
||||
// six halves, would make provider drift collinear with the axis under
|
||||
// test: a bad minute would land entirely on one arm.
|
||||
for _, arm := range shuffledArms(t) {
|
||||
cueVisible := t.Experiment == experimentIndex && cueExpectedVisible(t, arm.scale)
|
||||
planned++
|
||||
m, err := runOne(p, t, armFor(arm.scale, arm.searchOff), arm.name, root, cueVisible, dry)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s [%s]: %v\n", t.ID, arm.name, err)
|
||||
m = failedMetrics(t, arm.name, err)
|
||||
}
|
||||
byArm[arm.name] = append(byArm[arm.name], m)
|
||||
all = append(all, m)
|
||||
fmt.Printf("%-24s %-14s %-18s search=%d hit=%d read=%d recall-tok=%d%s %v\n",
|
||||
t.ID, arm.name, m.FailureStage, m.SearchCalls, m.TargetSearchHits, m.ReadCalls,
|
||||
m.RecallReturnedTokens, promptTokenSuffix(m), m.UnexpectedWorkTools)
|
||||
}
|
||||
}
|
||||
reportFunnels(byArm)
|
||||
if experiment == experimentIndex {
|
||||
fmt.Print(reportDoseResponse(all))
|
||||
}
|
||||
fmt.Print(reportStopping(all))
|
||||
fmt.Print(auditQueries(all).report())
|
||||
fmt.Print(queryLines(all))
|
||||
if experiment == experimentIndex {
|
||||
reportBoundaries(all)
|
||||
}
|
||||
return writeResults(root, batchInfo{Mode: mode, Planned: planned, Started: started, Dry: dry, Repo: repo}, all)
|
||||
}
|
||||
|
||||
func cueExpectedVisible(t contextTask, scale string) bool {
|
||||
visible, _ := tierScales(t.CueTier)
|
||||
return contains(visible, scale)
|
||||
}
|
||||
|
||||
func reportFunnels(byArm map[string][]contextMetrics) {
|
||||
names := make([]string, 0, len(byArm))
|
||||
for name := range byArm {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
fmt.Println("\n## Funnel")
|
||||
for _, name := range names {
|
||||
fmt.Println(" ", summarize(name, byArm[name]).line())
|
||||
}
|
||||
fmt.Println("\n## Where it looked first")
|
||||
for _, name := range names {
|
||||
f := summarize(name, byArm[name])
|
||||
fmt.Printf(" %-14s %s\n", name, countsLine(f.Routes))
|
||||
}
|
||||
fmt.Println("\n## Where it broke")
|
||||
for _, name := range names {
|
||||
f := summarize(name, byArm[name])
|
||||
stages := make([]string, 0, len(f.Stages))
|
||||
for stage, n := range f.Stages {
|
||||
stages = append(stages, fmt.Sprintf("%s=%d", stage, n))
|
||||
}
|
||||
sort.Strings(stages)
|
||||
fmt.Printf(" %-12s %s\n", name, strings.Join(stages, " "))
|
||||
}
|
||||
}
|
||||
|
||||
// writeTrajectories saves what each turn produced, after the whole batch is
|
||||
// done. Three analyses in a row have needed a re-run because something was
|
||||
// counted and not kept — query text, then snippet contents. It is written at
|
||||
// the end on purpose: during the batch these files are the answer.
|
||||
func writeTrajectories(root string, all []contextMetrics) error {
|
||||
f, err := os.Create(filepath.Join(root, "trajectories.jsonl"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
for _, m := range all {
|
||||
if err := writeJSONLine(f, map[string]any{
|
||||
"task": m.Task, "arm": m.Arm, "messages": m.trajectory,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeResults(root string, info batchInfo, all []contextMetrics) int {
|
||||
path := filepath.Join(root, "results.jsonl")
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
defer f.Close()
|
||||
scored, failed := 0, 0
|
||||
for _, m := range all {
|
||||
if err := writeJSONLine(f, m); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
if m.Failed {
|
||||
failed++
|
||||
} else {
|
||||
scored++
|
||||
}
|
||||
}
|
||||
if err := writeTrajectories(root, all); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write trajectories:", err)
|
||||
return 2
|
||||
}
|
||||
if err := writeBatchManifest(root, info, all); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write batch manifest:", err)
|
||||
return 2
|
||||
}
|
||||
// Both counts are named: a reader who sees only the batch size cannot tell
|
||||
// measurements from attempts.
|
||||
fmt.Printf("\n%d runs written to %s: scored=%d failed=%d (with trajectories.jsonl and batch.json)\n",
|
||||
len(all), path, scored, failed)
|
||||
return 0
|
||||
}
|
||||
|
||||
// reportBoundaries prints each index task across its own cue boundary, then the
|
||||
// two sides pooled. Task difficulty is held constant inside a row, which is
|
||||
// what a four-arm average cannot do.
|
||||
func reportBoundaries(all []contextMetrics) {
|
||||
byKey := map[string]contextMetrics{}
|
||||
scored, failed := 0, 0
|
||||
for _, m := range all {
|
||||
if m.Failed {
|
||||
// No transcript, no boundary to compare: count the failure instead of
|
||||
// printing the zero it left behind as a measurement.
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
scored++
|
||||
byKey[m.Task+"|"+m.Arm] = m
|
||||
}
|
||||
if scored == 0 {
|
||||
fmt.Printf("\n## Across each task's cue boundary\n no scored runs: scored=%d failed=%d (nothing to compare)\n", scored, failed)
|
||||
return
|
||||
}
|
||||
var cueSide, noCueSide []contextMetrics
|
||||
fmt.Println("\n## Across each task's cue boundary")
|
||||
fmt.Printf(" %-24s %-16s %-4s %-10s %-9s %-12s %-11s %s\n",
|
||||
"task", "arm", "cue", "recovered", "cue-read", "search/task", "recall-tok", "escape")
|
||||
for _, t := range indexTasks() {
|
||||
cue, noCue := boundaryPair(t.CueTier)
|
||||
for _, side := range []struct {
|
||||
scale string
|
||||
has bool
|
||||
}{{cue, true}, {noCue, false}} {
|
||||
m, ok := byKey[t.ID+"|index-"+side.scale]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if side.has {
|
||||
cueSide = append(cueSide, m)
|
||||
} else {
|
||||
noCueSide = append(noCueSide, m)
|
||||
}
|
||||
mark := "no"
|
||||
if side.has {
|
||||
mark = "yes"
|
||||
}
|
||||
fmt.Printf(" %-24s %-16s %-4s %-10v %-9v %-12d %-11d %d\n",
|
||||
t.ID, "index-"+side.scale, mark, m.AnswerRecovered, m.RecallReadWithoutSearch,
|
||||
m.SearchCalls, m.RecallReturnedTokens, m.EscapeCalls)
|
||||
}
|
||||
}
|
||||
fmt.Println("\n## Pooled across boundaries")
|
||||
fmt.Println(" ", summarize("cue-present", cueSide).line())
|
||||
fmt.Println(" ", summarize("cue-absent", noCueSide).line())
|
||||
}
|
||||
|
||||
// shuffledArms permutes a task's arms deterministically, so the order is
|
||||
// reproducible and still uncorrelated with the axis being measured.
|
||||
func shuffledArms(t contextTask) []experimentArm {
|
||||
arms := armsFor(t)
|
||||
rng := seededRand(t.ID, "arm-order")
|
||||
rng.Shuffle(len(arms), func(i, j int) { arms[i], arms[j] = arms[j], arms[i] })
|
||||
return arms
|
||||
}
|
||||
|
||||
// fingerprint records what produced a batch. Not a gate — but batch effects
|
||||
// have reversed three conclusions in this experiment already, and a table with
|
||||
// no provenance cannot be compared with the next one.
|
||||
func fingerprint(dry bool) string {
|
||||
model, endpoint := runModel, runBaseURL
|
||||
if dry {
|
||||
model, endpoint = "scripted", "none"
|
||||
}
|
||||
commit := "unknown"
|
||||
if out, err := exec.Command("git", "rev-parse", "--short", "HEAD").Output(); err == nil {
|
||||
commit = strings.TrimSpace(string(out))
|
||||
}
|
||||
return fmt.Sprintf("## Batch\n model=%s endpoint=%s window=%d generations=%d commit=%s started=%s\n",
|
||||
model, endpoint, fixtureWindow, fixtureGenerations, commit, time.Now().UTC().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
// timeoutReason separates a run that never found its answer from one that found
|
||||
// it and kept going. Only the first is a case for a longer budget; the second
|
||||
// would just get more time to circle. A batch reporting only "context deadline
|
||||
// exceeded" cannot tell an arm that made retrieval harder from an arm that
|
||||
// happened to draw a slow minute.
|
||||
func timeoutReason(m contextMetrics) string {
|
||||
switch {
|
||||
case m.TargetHeldAtEnd && m.RoundsAfterTarget >= 2:
|
||||
return "PostTargetRunaway"
|
||||
case m.TargetHeldAtEnd:
|
||||
return "TargetHeldLate"
|
||||
case m.SearchCalls >= 3:
|
||||
return "SearchSpiral"
|
||||
case m.SearchCalls+m.ReadCalls+m.EscapeCalls == 0:
|
||||
return "NoToolActivity"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// scriptedProvider walks the chain a competent model would: search, read the
|
||||
// best hit, answer with what came back. It proves the pipeline holds together
|
||||
// for nothing, so a real run's first failure is about the model.
|
||||
|
||||
type scriptedProvider struct{}
|
||||
|
||||
func (p *scriptedProvider) Name() string { return "contextbench-scripted" }
|
||||
|
||||
var (
|
||||
scriptedHit = regexp.MustCompile(`(?m)^#(\d+) `)
|
||||
scriptedRecall = regexp.MustCompile(`(?s)^#\d+\n(.*)`)
|
||||
)
|
||||
|
||||
// Stream decides from the request alone, never from a call counter: one
|
||||
// provider serves every task in a run, and a counter would make the second task
|
||||
// start where the first left off.
|
||||
func (p *scriptedProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
ch := make(chan provider.Chunk, 3)
|
||||
defer close(ch)
|
||||
// Honour the schema, as a model does: an arm without a query parameter is
|
||||
// an arm that cannot search, and pretending otherwise scores a refusal as
|
||||
// a failed search.
|
||||
if !recallAccepts(req, "query") {
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "I have no address for that part of the history."}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
return ch, nil
|
||||
}
|
||||
// The last message, not the last tool result anywhere: the fixture's own
|
||||
// history is full of tool results, and one of those is not this turn's.
|
||||
last := ""
|
||||
if n := len(req.Messages); n > 0 && req.Messages[n-1].Role == provider.RoleTool {
|
||||
last = req.Messages[n-1].Content
|
||||
}
|
||||
switch {
|
||||
case last == "":
|
||||
args, _ := json.Marshal(map[string]string{"query": lastUserText(req)})
|
||||
ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{
|
||||
ID: "s1", Name: "recall", Arguments: string(args),
|
||||
}}
|
||||
case strings.Contains(last, "Folded context matching"):
|
||||
positions := scriptedHit.FindAllStringSubmatch(last, 1)
|
||||
if len(positions) == 0 {
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "Nothing in the folded history matches."}
|
||||
break
|
||||
}
|
||||
n, _ := strconv.Atoi(positions[0][1])
|
||||
args, _ := json.Marshal(map[string][]int{"positions": {n}})
|
||||
ch <- provider.Chunk{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{
|
||||
ID: "r1", Name: "recall", Arguments: string(args),
|
||||
}}
|
||||
default:
|
||||
// Answer with what recall returned, so a correct chain scores as one.
|
||||
body := last
|
||||
if m := scriptedRecall.FindStringSubmatch(last); len(m) > 1 {
|
||||
body = m[1]
|
||||
}
|
||||
ch <- provider.Chunk{Type: provider.ChunkText, Text: "From the folded history: " + body}
|
||||
}
|
||||
ch <- provider.Chunk{Type: provider.ChunkDone}
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// recallAccepts reports whether recall is offered with the given parameter.
|
||||
func recallAccepts(req provider.Request, param string) bool {
|
||||
for _, t := range req.Tools {
|
||||
if t.Name != "recall" {
|
||||
continue
|
||||
}
|
||||
b, err := json.Marshal(t.Parameters)
|
||||
return err == nil && strings.Contains(string(b), `"`+param+`"`)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func lastUserText(req provider.Request) string {
|
||||
for _, m := range slices.Backward(req.Messages) {
|
||||
if m.Role == provider.RoleUser {
|
||||
return m.Content
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
// snippet.go — does a rank-1 hit actually hand over the answer?
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/base/retrieval"
|
||||
"reasonix/internal/contract/ablation"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/tool"
|
||||
"reasonix/internal/runtime/agent"
|
||||
)
|
||||
|
||||
// TargetHit is not EvidenceSufficient: a rank-1 hit can still hand over a
|
||||
// window too short to answer with, and searching again after that is correct.
|
||||
// Measured through the shipped recall path, not a copy of it.
|
||||
|
||||
// snippetWidths are the windows to compare. 240 is what ships.
|
||||
var snippetWidths = []int{120, 180, 240, 320, 480}
|
||||
|
||||
// historicalQueries are the queries these tasks actually drew in the pilots,
|
||||
// kept because a probe query is the host's phrasing and the model's was not.
|
||||
var historicalQueries = map[string][]string{
|
||||
"i03-coalescing": {"coalesce window marker probe", "job completion coalescing window"},
|
||||
"i04-recovery-fence": {"recovery fence marker epoch", "expected epoch", "recfence quartz"},
|
||||
}
|
||||
|
||||
type snippetReading struct {
|
||||
Task string
|
||||
Query string
|
||||
Width int
|
||||
Covered int
|
||||
Total int
|
||||
Runes int
|
||||
Rank int
|
||||
Complete bool
|
||||
}
|
||||
|
||||
// runSnippetAudit answers the question offline, over many instantiations so a
|
||||
// single seed's lengths cannot decide it.
|
||||
func runSnippetAudit(root string, seeds int) int {
|
||||
fmt.Printf("## Snippet coverage (%d instantiations per task)\n", seeds)
|
||||
fmt.Println(" Does the window a rank-1 hit returns contain every value the task scores on?")
|
||||
|
||||
byTaskWidth := map[string]map[int][]snippetReading{}
|
||||
for _, t := range allTasks() {
|
||||
byTaskWidth[t.ID] = map[int][]snippetReading{}
|
||||
for seed := range seeds {
|
||||
inst, err := instantiateTask(t, seededRand(t.ID, "snippet", fmt.Sprint(seed)))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
queries := append([]string{inst.ProbeQuery}, historicalQueries[t.ID]...)
|
||||
for _, q := range queries {
|
||||
for _, width := range snippetWidths {
|
||||
r := measureSnippet(inst, q, width)
|
||||
byTaskWidth[t.ID][width] = append(byTaskWidth[t.ID][width], r)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\n %-24s %-5s", "task", "vals")
|
||||
for _, w := range snippetWidths {
|
||||
fmt.Printf("%9d", w)
|
||||
}
|
||||
fmt.Println(" mean runes @240")
|
||||
for _, t := range allTasks() {
|
||||
vals := len(t.AnswerVars)
|
||||
fmt.Printf(" %-24s %-5d", t.ID, vals)
|
||||
for _, w := range snippetWidths {
|
||||
fmt.Printf("%8.0f%%", 100*completeRate(byTaskWidth[t.ID][w]))
|
||||
}
|
||||
fmt.Printf(" %.0f\n", meanRunes(byTaskWidth[t.ID][240]))
|
||||
}
|
||||
reportShippedPath(root)
|
||||
return 0
|
||||
}
|
||||
|
||||
// measureSnippet renders one hit through the same MakeSnippet the recall path
|
||||
// uses, then asks whether every scored value survived the window.
|
||||
func measureSnippet(inst fixtureInstance, query string, width int) snippetReading {
|
||||
terms, err := retrieval.QueryTerms(query)
|
||||
if err != nil {
|
||||
return snippetReading{Task: inst.Task.ID, Query: query, Width: width, Total: len(inst.AnswerMarkers)}
|
||||
}
|
||||
// The tool result is the document the answer lives in; a call's arguments
|
||||
// are a separate document and never carry it.
|
||||
snippet := retrieval.MakeSnippet(inst.body, query, terms, width)
|
||||
covered := 0
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if strings.Contains(strings.ToLower(snippet), strings.ToLower(marker)) {
|
||||
covered++
|
||||
}
|
||||
}
|
||||
return snippetReading{
|
||||
Task: inst.Task.ID, Query: query, Width: width,
|
||||
Covered: covered, Total: len(inst.AnswerMarkers),
|
||||
Runes: len([]rune(snippet)), Complete: covered == len(inst.AnswerMarkers),
|
||||
}
|
||||
}
|
||||
|
||||
// reportShippedPath cross-checks the table against the real recall tool: a
|
||||
// fixture, a fold, and a search through the agent that ships.
|
||||
func reportShippedPath(root string) {
|
||||
fmt.Println("\n## Through the shipped recall path (width as shipped)")
|
||||
for _, t := range allTasks() {
|
||||
inst, err := instantiateTask(t, seededRand(t.ID, "snippet-live"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
f, err := buildFixture(inst, ablation.Set{}, filepath.Join(root, "snippet", t.ID, "session.jsonl"))
|
||||
if err != nil {
|
||||
fmt.Printf(" %-24s fixture: %v\n", t.ID, err)
|
||||
continue
|
||||
}
|
||||
a := agent.New(&digestProvider{}, tool.NewRegistry(), f.Session, agent.Options{
|
||||
ContextWindow: fixtureWindow, CompactRatio: 0.5, RecentKeep: 2,
|
||||
SessionPath: f.Path, KeepPolicy: agent.KeepErrors,
|
||||
}, event.Discard)
|
||||
a.LoadProjectionSidecar(f.Path)
|
||||
queries := append([]string{inst.ProbeQuery}, historicalQueries[t.ID]...)
|
||||
for _, q := range queries {
|
||||
res, err := a.RecallContext(context.Background(), tool.RecallRequest{Query: q})
|
||||
if err != nil {
|
||||
fmt.Printf(" %-24s %-34q recall: %v\n", t.ID, q, err)
|
||||
continue
|
||||
}
|
||||
covered := 0
|
||||
for _, marker := range inst.AnswerMarkers {
|
||||
if strings.Contains(strings.ToLower(res.Text), strings.ToLower(marker)) {
|
||||
covered++
|
||||
}
|
||||
}
|
||||
fmt.Printf(" %-24s %-34q %d/%d values in the result\n",
|
||||
t.ID, truncate(q, 32), covered, len(inst.AnswerMarkers))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func completeRate(rs []snippetReading) float64 {
|
||||
if len(rs) == 0 {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, r := range rs {
|
||||
if r.Complete {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return float64(n) / float64(len(rs))
|
||||
}
|
||||
|
||||
func meanRunes(rs []snippetReading) float64 {
|
||||
if len(rs) == 0 {
|
||||
return 0
|
||||
}
|
||||
total := 0
|
||||
for _, r := range rs {
|
||||
total += r.Runes
|
||||
}
|
||||
return float64(total) / float64(len(rs))
|
||||
}
|
||||
|
||||
// answerCoverage is what a run should have recorded all along: how much of the
|
||||
// scored answer a search actually handed over. rank=1 with coverage 1/3 is not
|
||||
// a stopping failure.
|
||||
func answerCoverage(text string, markers []string) int {
|
||||
covered := 0
|
||||
lower := strings.ToLower(text)
|
||||
for _, marker := range markers {
|
||||
if strings.Contains(lower, strings.ToLower(marker)) {
|
||||
covered++
|
||||
}
|
||||
}
|
||||
return covered
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// usageSink reads what a run cost from the events the agent already emits. Every
|
||||
// billed completion reports event.Usage — the round path, compaction, command
|
||||
// routing and failure diagnosis all do — so the bench reads the same record the
|
||||
// product reads instead of wrapping a provider and counting a second time.
|
||||
type usageSink struct {
|
||||
mu sync.Mutex
|
||||
requests int
|
||||
usage provider.Usage
|
||||
}
|
||||
|
||||
func (s *usageSink) Emit(e event.Event) {
|
||||
if e.Kind != event.Usage || e.Usage == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
// One event is one completion that reported usage; RequestCount covers a
|
||||
// completion that merged attempts. A request that failed before reporting
|
||||
// leaves no event, so this is not a count of attempts.
|
||||
if e.Usage.RequestCount > 0 {
|
||||
s.requests += e.Usage.RequestCount
|
||||
} else {
|
||||
s.requests++
|
||||
}
|
||||
u := *e.Usage
|
||||
s.usage.PromptTokens += u.PromptTokens
|
||||
s.usage.CompletionTokens += u.CompletionTokens
|
||||
s.usage.TotalTokens += u.TotalTokens
|
||||
s.usage.CacheHitTokens += u.CacheHitTokens
|
||||
s.usage.CacheMissTokens += u.CacheMissTokens
|
||||
s.usage.ReasoningTokens += u.ReasoningTokens
|
||||
s.usage.Estimated = s.usage.Estimated || u.Estimated
|
||||
}
|
||||
|
||||
func (s *usageSink) snapshot() (provider.Usage, int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.usage, s.requests
|
||||
}
|
||||
|
||||
// promptTokenSuffix renders what a run cost only when something reported it. A
|
||||
// dry run reports nothing, and a trailing "prompt-tok=0" would read as a
|
||||
// measured zero rather than as no measurement.
|
||||
func promptTokenSuffix(m contextMetrics) string {
|
||||
if m.UsageReportedRequests == 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" prompt-tok=%d", m.PromptTokens)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// The cost a run reports must be what the agent reported, summed across every
|
||||
// completion it billed — including one that merged attempts and one that could
|
||||
// only estimate. Any other event kind is not a cost.
|
||||
func TestUsageSinkAccumulatesWhatTheAgentReports(t *testing.T) {
|
||||
var sink usageSink
|
||||
sink.Emit(event.Event{}) // a kind that is not Usage
|
||||
sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{
|
||||
PromptTokens: 100, CompletionTokens: 10, TotalTokens: 110, CacheHitTokens: 60, CacheMissTokens: 40,
|
||||
}})
|
||||
sink.Emit(event.Event{Kind: event.Usage, Usage: &provider.Usage{
|
||||
PromptTokens: 7, CompletionTokens: 3, CacheHitTokens: 2, CacheMissTokens: 5, RequestCount: 2, Estimated: true,
|
||||
}})
|
||||
sink.Emit(event.Event{Kind: event.Usage}) // a Usage event with no payload is not a cost
|
||||
|
||||
usage, requests := sink.snapshot()
|
||||
if requests != 3 {
|
||||
t.Fatalf("requests = %d, want 3 (1 + the merged attempt's 2; the empty event adds none)", requests)
|
||||
}
|
||||
if usage.PromptTokens != 107 || usage.CompletionTokens != 13 || usage.TotalTokens != 110 {
|
||||
t.Fatalf("accumulated usage = %+v", usage)
|
||||
}
|
||||
if usage.CacheHitTokens != 62 || usage.CacheMissTokens != 45 {
|
||||
t.Fatalf("cache split = %+v", usage)
|
||||
}
|
||||
if !usage.Estimated {
|
||||
t.Fatal("Estimated was dropped when one completion reported an estimate")
|
||||
}
|
||||
}
|
||||
|
||||
// A dry run must not claim a cost it never paid, in the JSON or in the line a
|
||||
// person reads: an empty suffix, not "prompt-tok=0".
|
||||
func TestUsageSinkStaysEmptyWithoutUsage(t *testing.T) {
|
||||
var sink usageSink
|
||||
sink.Emit(event.Event{})
|
||||
usage, requests := sink.snapshot()
|
||||
if requests != 0 || usage.PromptTokens != 0 || usage.CompletionTokens != 0 {
|
||||
t.Fatalf("nothing reported usage, got %+v requests=%d", usage, requests)
|
||||
}
|
||||
if suffix := promptTokenSuffix(contextMetrics{}); suffix != "" {
|
||||
t.Fatalf("a run with no reported usage still printed %q", suffix)
|
||||
}
|
||||
if suffix := promptTokenSuffix(contextMetrics{UsageReportedRequests: 1, PromptTokens: 12}); suffix != " prompt-tok=12" {
|
||||
t.Fatalf("a reported cost printed %q", suffix)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,452 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/agentgraph"
|
||||
"reasonix/internal/session/control"
|
||||
"reasonix/internal/state/execjournal"
|
||||
)
|
||||
|
||||
// armFleetActiveStore asks whether the sub-agent store can be asked about a
|
||||
// fan-out item whose slot was granted and whose child reached the provider. It
|
||||
// is not a parity arm: the store is what every recovery path reads, so what is
|
||||
// measured is one store answering differently about executions it holds the
|
||||
// same evidence for.
|
||||
const armFleetActiveStore = "fleet-active-store"
|
||||
|
||||
// activeFleetCallID is the fan-out under measurement. The lone delegation in
|
||||
// the same turn keeps taskCallID, so the two entry points stay separable in
|
||||
// every artifact without a naming scheme of their own.
|
||||
const activeFleetCallID = "probe_fleet_active"
|
||||
|
||||
// probeFleetChildEntered is the fan-out's half of the entry evidence. The lone
|
||||
// delegation's rides probeChildEntered, which already means exactly this for
|
||||
// the delegation under measurement.
|
||||
const probeFleetChildEntered = "probe:fleet-child-entered"
|
||||
|
||||
// The four populations one death holds. They are read off the journal rather
|
||||
// than off the item ids the dispatch declared: which of the two held-back items
|
||||
// wins the freed slot is the scheduler's to decide, and an arm that named them
|
||||
// in advance would be reporting a race as a premise.
|
||||
type population string
|
||||
|
||||
const (
|
||||
popSettled population = "fan-out settled"
|
||||
popRunning population = "fan-out executing"
|
||||
popRefused population = "fan-out refused"
|
||||
popLone population = "lone executing"
|
||||
popNone population = ""
|
||||
)
|
||||
|
||||
// measured are the four in the order they are reported, which is also the order
|
||||
// they become true: an item settles, the slot it frees admits one of the two
|
||||
// behind it, and the delegation holding the other slot outlives all of them.
|
||||
func measured() []population { return []population{popSettled, popRunning, popRefused, popLone} }
|
||||
|
||||
// populationOf places one journal entry in its population, or in none. Started
|
||||
// and open is executing; started and settled is done; queued without a start is an
|
||||
// item the scheduler held back and never admitted.
|
||||
func populationOf(e execjournal.Entry) population {
|
||||
switch {
|
||||
case ofLoneTask(e.ID):
|
||||
if e.Started() && e.Open() {
|
||||
return popLone
|
||||
}
|
||||
case ofActiveFleet(e.ID):
|
||||
switch {
|
||||
case e.Started() && !e.Open():
|
||||
return popSettled
|
||||
case e.Started() && e.Open():
|
||||
return popRunning
|
||||
case !e.Started() && e.Queued():
|
||||
return popRefused
|
||||
}
|
||||
}
|
||||
return popNone
|
||||
}
|
||||
|
||||
func ofActiveFleet(id string) bool { return strings.HasPrefix(id, activeFleetCallID+"/") }
|
||||
|
||||
// executionsIn are the ids this observation's journal places in a population,
|
||||
// in a stable order.
|
||||
func executionsIn(o Observation, pop population) []string {
|
||||
var out []string
|
||||
for _, e := range o.Executions {
|
||||
if populationOf(e) == pop {
|
||||
out = append(out, e.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// storedStatus is what the store says about one execution, empty when it has
|
||||
// never heard of it. The join is the execution id the child recorded as its
|
||||
// parent call, which is the same identity the journal opened it under.
|
||||
func storedStatus(o Observation, id string) string {
|
||||
for _, f := range o.Children.Facts {
|
||||
if f.ParentToolCallID == id {
|
||||
return f.Status
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// noRecord is what the store says about an execution it never recorded. It is
|
||||
// spelled out rather than left blank because a blank column reads as a row that
|
||||
// was not measured, which is the opposite of what an absent record means here.
|
||||
const noRecord = "no record"
|
||||
|
||||
func storeReading(o Observation, ids []string) string {
|
||||
out := make([]string, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
status := storedStatus(o, id)
|
||||
if status == "" {
|
||||
status = noRecord
|
||||
}
|
||||
out = append(out, id+"→"+status)
|
||||
}
|
||||
return orNone(join(out))
|
||||
}
|
||||
|
||||
// acknowledged reports whether the store holds a record for every one of these
|
||||
// executions. An empty set is not acknowledgement: a population the death never
|
||||
// produced is a premise that failed, and its row says not-measured.
|
||||
func acknowledged(o Observation, ids []string) bool {
|
||||
for _, id := range ids {
|
||||
if storedStatus(o, id) == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(ids) > 0
|
||||
}
|
||||
|
||||
func activeStoreArm(name string) bool { return name == armFleetActiveStore }
|
||||
|
||||
// activeStoreRows read one store from four sides. Each population has the same
|
||||
// two columns — what the dying process's store said, and what the next one's
|
||||
// does — so the answers can only differ by which execution is being asked about.
|
||||
func activeStoreRows(before, after Observation) []row {
|
||||
return []row{
|
||||
populationsRow(before),
|
||||
childEntryRow(before),
|
||||
settledStoreRow(before, after),
|
||||
executingStoreRow(before, after),
|
||||
loneStoreRow(before, after),
|
||||
refusedStoreRow(before, after),
|
||||
listingRow(before, after),
|
||||
rebuildUnmovedRow(before, after),
|
||||
}
|
||||
}
|
||||
|
||||
// populationsRow is the arm's premise, not a durability claim: one death has to
|
||||
// hold all four states at once, or the comparisons below are between different
|
||||
// runs rather than between entry points.
|
||||
func populationsRow(before Observation) row {
|
||||
var got []string
|
||||
for _, pop := range measured() {
|
||||
got = append(got, fmt.Sprintf("%s=%d", pop, len(executionsIn(before, pop))))
|
||||
}
|
||||
return row{
|
||||
Semantic: "the four populations this death holds", Authority: journalAuthority,
|
||||
Artifact: "<stem>.execution.jsonl", Reconstruction: "in-process only",
|
||||
Before: join(got), After: "—", Verdict: activePremiseVerdict(before),
|
||||
}
|
||||
}
|
||||
|
||||
func activePremiseVerdict(before Observation) string {
|
||||
if activeStoreInvalid(before) != "" {
|
||||
return verdictNotMeasured
|
||||
}
|
||||
return verdictHolds
|
||||
}
|
||||
|
||||
// childEntryRow is the half of the evidence no artifact carries. A slot grant
|
||||
// says the scheduler admitted the item; only the child's own arrival at the
|
||||
// provider says the execution happened, and a store row read without it would
|
||||
// be asking about work that may never have run.
|
||||
func childEntryRow(before Observation) row {
|
||||
fleet := firstProgress(before, probeFleetChildEntered)
|
||||
lone := firstProgress(before, probeChildEntered)
|
||||
return row{
|
||||
Semantic: "the children that reached the provider", Authority: "the scripted provider, in-process",
|
||||
Artifact: "none observed", Reconstruction: "in-process only",
|
||||
Before: fmt.Sprintf("fan-out=%s lone=%s", fleet, lone),
|
||||
After: "—",
|
||||
Verdict: enteredVerdict(fleet == "true" && lone == "true"),
|
||||
}
|
||||
}
|
||||
|
||||
func enteredVerdict(both bool) string {
|
||||
if both {
|
||||
return verdictHolds
|
||||
}
|
||||
return verdictNotMeasured
|
||||
}
|
||||
|
||||
func firstProgress(o Observation, key string) string {
|
||||
if vals := o.Progress[key]; len(vals) > 0 {
|
||||
return vals[0]
|
||||
}
|
||||
return "unrecorded"
|
||||
}
|
||||
|
||||
// settledStoreRow is the control that makes the rest readable. It is the same
|
||||
// entry point, the same fan-out and the same store, differing only in that this
|
||||
// item finished — so if it too were absent, the finding would be about
|
||||
// persistence in general rather than about executions in flight.
|
||||
func settledStoreRow(before, after Observation) row {
|
||||
ids := executionsIn(before, popSettled)
|
||||
return row{
|
||||
Semantic: "a settled fan-out item, in the store", Authority: storeAuthority,
|
||||
Artifact: "subagents/<ref>.meta.json", Reconstruction: "ListSubagentsByParent, joined by execution id",
|
||||
Before: storeReading(before, ids),
|
||||
After: storeReading(after, ids),
|
||||
Verdict: controlVerdict(acknowledged(before, ids) && acknowledged(after, ids), len(ids) > 0),
|
||||
}
|
||||
}
|
||||
|
||||
// executingStoreRow is the arm. The conjunction is stated in the row rather
|
||||
// than left to the reader: the journal proves the slot was granted, the child
|
||||
// proves the execution happened, and this column is what the store can say
|
||||
// about it. All three, or the row is not the finding it looks like.
|
||||
func executingStoreRow(before, after Observation) row {
|
||||
ids := executionsIn(before, popRunning)
|
||||
entered := firstProgress(before, probeFleetChildEntered) == "true"
|
||||
return row{
|
||||
Semantic: "an executing fan-out item, in the store", Authority: storeAuthority,
|
||||
Artifact: "subagents/<ref>.meta.json",
|
||||
Reconstruction: "STARTED in the journal, entered at the provider, and asked of the store",
|
||||
Before: storeReading(before, ids),
|
||||
After: storeReading(after, ids),
|
||||
Verdict: executingVerdict(before, after, ids, entered),
|
||||
}
|
||||
}
|
||||
|
||||
// executingVerdict refuses the two readings that would overstate the row. An
|
||||
// execution nothing proves entered is not a silent store, and a population the
|
||||
// death never produced is not evidence of anything.
|
||||
func executingVerdict(before, after Observation, ids []string, entered bool) string {
|
||||
switch {
|
||||
case len(ids) == 0 || !entered:
|
||||
return verdictNotMeasured
|
||||
case acknowledged(before, ids) || acknowledged(after, ids):
|
||||
return verdictHolds
|
||||
default:
|
||||
return verdictViolated
|
||||
}
|
||||
}
|
||||
|
||||
// loneStoreRow is the other entry point, in the same death and the same store.
|
||||
// It is what makes "selectively blind" a statement about the store rather than
|
||||
// a comparison between two probe runs.
|
||||
func loneStoreRow(before, after Observation) row {
|
||||
ids := executionsIn(before, popLone)
|
||||
return row{
|
||||
Semantic: "an executing lone delegation, in the store", Authority: storeAuthority,
|
||||
Artifact: "subagents/<ref>.meta.json", Reconstruction: "the same read, one entry point over",
|
||||
Before: storeReading(before, ids),
|
||||
After: storeReading(after, ids),
|
||||
Verdict: controlVerdict(acknowledged(before, ids) && acknowledged(after, ids), len(ids) > 0),
|
||||
}
|
||||
}
|
||||
|
||||
// refusedStoreRow is the negative control, and it bounds any fix this arm might
|
||||
// support. An item the scheduler never admitted produced nothing, so the store
|
||||
// is owed no record for it — and a change that recorded every opening would
|
||||
// satisfy the row above by reintroducing exactly that.
|
||||
func refusedStoreRow(before, after Observation) row {
|
||||
ids := executionsIn(before, popRefused)
|
||||
silent := !acknowledged(before, ids) && !acknowledged(after, ids)
|
||||
return row{
|
||||
Semantic: "a refused fan-out item, in the store", Authority: crossAuthority,
|
||||
Artifact: "subagents/<ref>.meta.json, <stem>.execution.jsonl",
|
||||
Reconstruction: "queued, never started: the store must stay silent",
|
||||
Before: storeReading(before, ids),
|
||||
After: storeReading(after, ids),
|
||||
Verdict: controlVerdict(silent, len(ids) > 0),
|
||||
}
|
||||
}
|
||||
|
||||
func controlVerdict(ok bool, measured bool) string {
|
||||
switch {
|
||||
case !measured:
|
||||
return verdictNotMeasured
|
||||
case ok:
|
||||
return verdictHolds
|
||||
default:
|
||||
return verdictViolated
|
||||
}
|
||||
}
|
||||
|
||||
// listingRow is the surface a person or a model reaches for. It reads through
|
||||
// the workspace filter and the lineage walk the tool applies, which the store
|
||||
// read above does not — so it is checked against that read first: a listing
|
||||
// that cannot even see the settled item is a filter answering, not a store.
|
||||
func listingRow(before, after Observation) row {
|
||||
settled := executionsIn(before, popSettled)
|
||||
return row{
|
||||
Semantic: "what list_subagents would show", Authority: "SubagentStore.ListForParent",
|
||||
Artifact: "subagents/<ref>.meta.json",
|
||||
Reconstruction: "the read behind the tool, filtered by workspace and lineage",
|
||||
Before: listingReading(before),
|
||||
After: listingReading(after),
|
||||
Verdict: listingVerdict(before, after, settled),
|
||||
}
|
||||
}
|
||||
|
||||
func listingReading(o Observation) string {
|
||||
if o.Listing.Err != "" {
|
||||
return "error: " + o.Listing.Err
|
||||
}
|
||||
out := make([]string, 0, len(o.Listing.Rows))
|
||||
for _, r := range o.Listing.Rows {
|
||||
out = append(out, orDash(r.From)+"="+r.Status)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return orNone(join(out))
|
||||
}
|
||||
|
||||
// listingVerdict answers only once the listing has been shown to work. Its
|
||||
// filters can drop every row for reasons that have nothing to do with this arm,
|
||||
// and an empty listing read as an invisible execution is the substitution the
|
||||
// probe exists to refuse.
|
||||
func listingVerdict(before, after Observation, settled []string) string {
|
||||
if len(settled) == 0 || !listingHas(after, settled) {
|
||||
return verdictNotMeasured
|
||||
}
|
||||
if listingHas(before, executionsIn(before, popRunning)) {
|
||||
return verdictHolds
|
||||
}
|
||||
return verdictViolated
|
||||
}
|
||||
|
||||
func listingHas(o Observation, ids []string) bool {
|
||||
for _, id := range ids {
|
||||
found := false
|
||||
for _, r := range o.Listing.Rows {
|
||||
if r.From == id {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(ids) > 0
|
||||
}
|
||||
|
||||
// rebuildUnmovedRow pins what a fix must not move. A store that learned to
|
||||
// record executions in flight would leave stale records behind for a restart to
|
||||
// fold, and the one outcome this whole line of work exists to prevent is a
|
||||
// rebuild that answers with a node the dead process was running.
|
||||
func rebuildUnmovedRow(before, after Observation) row {
|
||||
rebuilt := rebuiltGraph(after)
|
||||
var ghosts []string
|
||||
for _, n := range rebuilt.Graph.Nodes {
|
||||
if n.State == agentgraph.StateRunning {
|
||||
ghosts = append(ghosts, n.ID)
|
||||
}
|
||||
}
|
||||
var cut []string
|
||||
for _, i := range rebuilt.Interrupted {
|
||||
if ofActiveFleet(i.Execution) || ofLoneTask(i.Execution) {
|
||||
cut = append(cut, i.Execution+":"+interruptionKind(i))
|
||||
}
|
||||
}
|
||||
sort.Strings(ghosts)
|
||||
sort.Strings(cut)
|
||||
return row{
|
||||
Semantic: "no ghost, and how the rebuild names the cut", Authority: rebuildAuthority,
|
||||
Artifact: "<stem>.execution.jsonl + subagents/<ref>.meta.json",
|
||||
Reconstruction: "execgraph.Rebuild over both, after the death",
|
||||
Before: fmt.Sprintf("%d executing at death", len(executionsIn(before, popRunning))+len(executionsIn(before, popLone))),
|
||||
After: fmt.Sprintf("ghosts=%s cut=%s", orNone(join(ghosts)), orNone(join(cut))),
|
||||
Verdict: controlVerdict(len(ghosts) == 0 && len(cut) > 0, len(before.Executions) > 0),
|
||||
}
|
||||
}
|
||||
|
||||
// activeStoreInvalid reports the premise this arm could not establish. All four
|
||||
// populations in one death is the whole design: without them the rows compare
|
||||
// different runs, and a row that compares nothing must not be read as green.
|
||||
func activeStoreInvalid(before Observation) string {
|
||||
for _, pop := range measured() {
|
||||
if len(executionsIn(before, pop)) == 0 {
|
||||
return "the death held nothing in population " + string(pop)
|
||||
}
|
||||
}
|
||||
if firstProgress(before, probeFleetChildEntered) != "true" {
|
||||
return "no fan-out child reached the provider, so no execution is proven to have happened"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// runActiveStoreConstruct drives one delegation and one fan-out under a ceiling
|
||||
// they share, and dies with all four populations standing. The delegation goes
|
||||
// first and hangs: it is what keeps the ceiling from emptying under the fan-out,
|
||||
// so the item refused admission stays refused rather than starting a moment
|
||||
// later and taking the arm's negative control with it.
|
||||
func runActiveStoreConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
|
||||
prompt := fanOutTurn(turn+1, taskSentinel+" "+fleetActiveSentinel)
|
||||
go func() { _ = ctrl.Run(ctx, prompt) }()
|
||||
if err := waitForPopulations(ctrl, prov); err != nil {
|
||||
return err
|
||||
}
|
||||
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
|
||||
// Whether each child reached the provider is known only here, and it is half
|
||||
// of what makes a silent store a finding rather than a missing record for
|
||||
// work that never ran.
|
||||
obs.Progress[probeFleetChildEntered] = []string{fmt.Sprint(prov.fleets.held(childEntered(childHang)))}
|
||||
obs.Progress[probeChildEntered] = []string{fmt.Sprint(prov.fleets.held(childEntered(childHold)))}
|
||||
if err := writeObservation(root, obs); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Exit(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForPopulations blocks until the journal holds all four states, then holds
|
||||
// still and asks again. A refusal is a state nothing announces, so the second
|
||||
// reading is what separates an item the scheduler is holding back from one that
|
||||
// was merely slow to be admitted.
|
||||
func waitForPopulations(ctrl *control.Controller, prov *scripted) error {
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if populationsStanding(ctrl, prov) {
|
||||
time.Sleep(2 * time.Second)
|
||||
if !populationsStanding(ctrl, prov) {
|
||||
return errUnexpected("a refusal that was still standing",
|
||||
journalStates(control.ExecutionHistory(ctrl.SessionPath())))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return errUnexpected("one death holding all four populations",
|
||||
journalStates(control.ExecutionHistory(ctrl.SessionPath())))
|
||||
}
|
||||
|
||||
func populationsStanding(ctrl *control.Controller, prov *scripted) bool {
|
||||
held := map[population]bool{}
|
||||
for _, e := range control.ExecutionHistory(ctrl.SessionPath()) {
|
||||
held[populationOf(e)] = true
|
||||
}
|
||||
return held[popSettled] && held[popRunning] && held[popRefused] && held[popLone] &&
|
||||
prov.fleets.held(childEntered(childHang)) && prov.fleets.held(childEntered(childHold))
|
||||
}
|
||||
|
||||
func journalStates(entries []execjournal.Entry) string {
|
||||
out := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
out = append(out, fmt.Sprintf("%s:started=%t,queued=%t,open=%t", e.ID, e.Started(), e.Queued(), e.Open()))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return orNone(join(out))
|
||||
}
|
||||
@@ -0,0 +1,345 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/config"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/state/memory"
|
||||
)
|
||||
|
||||
// arm is one death-and-resume run. Lever runs between the two processes and is
|
||||
// what separates the three compaction questions: nothing changed, only the
|
||||
// stable prefix changed, the covered conversation changed.
|
||||
type arm struct {
|
||||
name string
|
||||
asks string
|
||||
lever func(armRoot, Observation) error
|
||||
}
|
||||
|
||||
// appendsAfterFold names the arms whose construct phase adds one turn after
|
||||
// the fold. Every arm carrying a lever needs it: without it the projection is
|
||||
// already gone at the boundary for an unrelated reason, and the lever's own
|
||||
// effect cannot be read out of the result.
|
||||
func appendsAfterFold(name string) bool {
|
||||
return name != "exact" && name != armRefoldIntoBody && !graphArm(name) && !uiArm(name) &&
|
||||
!schedulerWaitArm(name) && !terminalArm(name) && !deriveArm(name) &&
|
||||
!loneTaskArm(name) && !cancelRoutingArm(name) && !lineageArm(name) && !activeStoreArm(name) &&
|
||||
!skillArm(name) && !ephemeralArm(name) && !hostExecArm(name)
|
||||
}
|
||||
|
||||
// armAppendAfterFold separates the two ways a projection can fail validation
|
||||
// after a restart: a covered prefix that no longer matches, and a projection
|
||||
// that covers the whole transcript, whose only cross-process validation is a
|
||||
// transcript version the reload does not carry.
|
||||
const armAppendAfterFold = "append-after-fold"
|
||||
|
||||
// armRefoldIntoBody folds twice. The second fold reads the provider-visible
|
||||
// view — stored body plus live tail — and must record a boundary in canonical
|
||||
// terms, which the body has no counterpart for.
|
||||
const armRefoldIntoBody = "refold-into-body"
|
||||
|
||||
// The tail arms separate two ways the live tail can move. tail-edit and
|
||||
// tail-truncate change the transcript past the fold, where the covered hash
|
||||
// cannot notice; tail-rewind takes the path a person actually drives, which
|
||||
// throws the projection away without asking whether coverage still holds.
|
||||
const (
|
||||
armTailEdit = "tail-edit"
|
||||
armTailTruncate = "tail-truncate"
|
||||
armTailRewind = "tail-rewind"
|
||||
// armCoveredRewind is the negative control the rewind path will need before
|
||||
// its invalidation can become conditional: rewinding below the fold removes
|
||||
// history the digest folded, and no coverage check may excuse that.
|
||||
armCoveredRewind = "covered-rewind"
|
||||
// armTodoIdentity moves host step identity while a fold is between the
|
||||
// model and it: identity A, fold, live tail, identity B, fold again.
|
||||
armTodoIdentity = "todo-identity"
|
||||
// armOpenDecision dies while a question is open. It asks what the host
|
||||
// believes afterwards, not whether one particular card comes back.
|
||||
armOpenDecision = "open-decision"
|
||||
// The three successors to an interrupted barrier. Each starts where
|
||||
// open-decision ends and differs only in what the next process does, which
|
||||
// is the whole question: when does a recorded interruption stop being the
|
||||
// context every later request carries?
|
||||
armIdleRestart = "interrupted-idle"
|
||||
armUnrelatedTurn = "interrupted-unrelated"
|
||||
armDirectAnswer = "interrupted-answered"
|
||||
)
|
||||
|
||||
// successorTurn is what the middle process says, empty when it only restarts.
|
||||
func successorTurn(arm string) (string, bool) {
|
||||
switch arm {
|
||||
case armIdleRestart:
|
||||
return "", true
|
||||
case armUnrelatedTurn:
|
||||
return marker(120) + " List the files you would look at first.", true
|
||||
case armDirectAnswer:
|
||||
return marker(121) + " Yes, go ahead with that — take the side below the fold.", true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func arms() []arm {
|
||||
return []arm{
|
||||
{name: "exact", asks: "nothing changed between the processes"},
|
||||
{name: armAppendAfterFold, asks: "nothing changed, and one turn was appended after the fold"},
|
||||
{name: armRefoldIntoBody, asks: "a second fold reaches into the body the first one stored"},
|
||||
{name: "system-swap", asks: "only the stable prefix changed, against the surviving baseline", lever: swapSystemPrefix},
|
||||
{name: armTailEdit, asks: "a message past the fold changed", lever: editTailRow},
|
||||
{name: armTailTruncate, asks: "the messages past the fold were dropped", lever: truncateTail},
|
||||
{name: armTailRewind, asks: "the same truncation, driven through the rewind a person uses"},
|
||||
{name: armCoveredRewind, asks: "a rewind that lands below the fold boundary"},
|
||||
{name: armTodoIdentity, asks: "host step identity moved across two folds and a live tail"},
|
||||
{name: armOpenDecision, asks: "the process died while a decision was open"},
|
||||
{name: armIdleRestart, asks: "an interrupted barrier, and nobody does anything"},
|
||||
{name: armUnrelatedTurn, asks: "an interrupted barrier, then a turn about something else"},
|
||||
{name: armDirectAnswer, asks: "an interrupted barrier, then a turn that reads like an answer to it"},
|
||||
{name: armGraphCompleted, asks: "a fan-out whose items had all settled when the process died"},
|
||||
{name: armGraphRunning, asks: "a fan-out with a child still executing when the process died"},
|
||||
{name: armGraphMixed, asks: "a fan-out holding completed, failed, adopted and running at once"},
|
||||
{name: armUIGraphMixed, asks: "which door each fact reaches the frontend through, after a death"},
|
||||
{name: armTaskCompleted, asks: "a lone delegation that finished, in a turn that closed"},
|
||||
{name: armTaskRunning, asks: "a lone delegation still executing when the process died"},
|
||||
{name: armTaskFgQueued, asks: "what a lone delegation is drawn as while the scheduler holds it back"},
|
||||
{name: armTaskBgQueued, asks: "a backgrounded delegation the ceiling refused, with its job already handed over"},
|
||||
{name: armTaskBgRunning, asks: "a backgrounded delegation executing inside its job"},
|
||||
{name: armCancelTool, asks: "whether a stop reaches an ordinary tool the turn is waiting on"},
|
||||
{name: armCancelBackground, asks: "whether a stop leaves work already handed to a job alone"},
|
||||
{name: armCancelHeadlessOwner, asks: "who owns the cancellation of a turn the controller never admitted"},
|
||||
{name: armLineageForeground, asks: "control: does a stop reach the tree the turn owns?"},
|
||||
{name: armLineageBackground, asks: "does it also reach the tree it handed away?"},
|
||||
{name: armLineageJobKill, asks: "control: does a job's own stop reach its tree?"},
|
||||
{name: armTaskFgQueuedCancel, asks: "a delegation stopped while the scheduler still held it back"},
|
||||
{name: armTaskBgQueuedCancel, asks: "the same, backgrounded: a job killed before it was ever admitted"},
|
||||
{name: armTaskFgRunningCancel, asks: "a delegation stopped while its child was executing"},
|
||||
{name: armTaskBgRunningCancel, asks: "the same, backgrounded: a job killed while its child ran"},
|
||||
{name: armWaitSlots, asks: "an item the session's total ceiling refused"},
|
||||
{name: armWaitWriters, asks: "an item the writer ceiling refused, with total capacity free"},
|
||||
{name: armWaitClaim, asks: "an item a path conflict refused, with both ceilings free"},
|
||||
{name: armWaitTransition, asks: "what a reported wait cause does when the thing it named stops holding the item"},
|
||||
{name: armTerminalAdopted, asks: "an item that never ran and still holds an answer"},
|
||||
{name: armTerminalSkippedDep, asks: "an item that never ran and holds none, because its dependency failed"},
|
||||
{name: armTerminalCancelled, asks: "an item a cancellation ended before it was admitted"},
|
||||
{name: armTerminalContext, asks: "whether a delivered answer's edge follows from what is already durable"},
|
||||
{name: armChildTerminal, asks: "whether the store keeps every terminal the graph draws"},
|
||||
{name: armDeriveSkipBoth, asks: "a skip whose two upstreams both ended without an answer"},
|
||||
{name: armDeriveSkipFlip, asks: "the same, with the two failures in the other order"},
|
||||
{name: armIdentitySemantics, asks: "what a node's model and effort report, across two producers and the store"},
|
||||
{name: armDeriveAnswered, asks: "a dependent whose upstreams answered, one by completing and one by adopting"},
|
||||
{name: armFleetActiveStore, asks: "whether the store can be asked about a fan-out item that is executing"},
|
||||
{name: armSkillQueued, asks: "whether a subagent skill asks the same scheduler as every other delegation"},
|
||||
{name: armSkillRunning, asks: "what names a subagent skill while its child is executing"},
|
||||
{name: armSkillCompleted, asks: "what a finished subagent skill leaves, through both entry points"},
|
||||
{name: armSkillCancel, asks: "what a stop reaches when a subagent skill is running"},
|
||||
{name: armReadOnlySkillQueued, asks: "whether the read-only skill entry point asks the same scheduler"},
|
||||
{name: armReadOnlySkillRunning, asks: "what names a read-only skill while its child is executing"},
|
||||
{name: armReadOnlySkillCompleted, asks: "what a finished read-only skill leaves behind, if anything"},
|
||||
{name: armEphemeralTask, asks: "can the execution authority account for what read_only_task draws?"},
|
||||
{name: armEphemeralSkill, asks: "the same of read_only_skill, the other ephemeral entry point"},
|
||||
{name: armHostRunning, asks: "what a slash-started subagent leaves when the process dies mid-execution"},
|
||||
{name: armHostQueued, asks: "what names slash-started work the scheduler is holding back"},
|
||||
{name: armHostCompleted, asks: "what a finished slash invocation leaves besides its answer"},
|
||||
{name: armHostCancel, asks: "what a stop reaches, and what history it leaves"},
|
||||
{name: armHeadlessCompleted, asks: "does the headless host surface leave the same provenance?"},
|
||||
{name: armHeadlessRunningCancel, asks: "the same, stopped through the context its caller holds"},
|
||||
{name: armHeadlessQueuedCancel, asks: "the same while the scheduler is still holding it back"},
|
||||
{name: "covered-mutation", asks: "the covered conversation changed, against the surviving baseline", lever: mutateCoveredRow},
|
||||
}
|
||||
}
|
||||
|
||||
// swapSystemPrefix saves a memory fact into this arm's home. The lever is a
|
||||
// real user action rather than a poked field: the saved-fact index folds into
|
||||
// the boot-time system prompt, so the next process composes a different one.
|
||||
func swapSystemPrefix(root armRoot, _ Observation) error {
|
||||
store := memory.StoreFor(config.RootsForHome(root.Home).MemoryUserDir(), root.Workspace)
|
||||
_, err := store.Save(memory.Memory{
|
||||
Name: "probe-prefix-lever",
|
||||
Title: "Probe prefix lever",
|
||||
Description: "A fact saved between the two processes so the composed system prompt differs.",
|
||||
Type: memory.TypeProject,
|
||||
Scope: memory.FactScopeProject,
|
||||
// Pinned, not relevant: only a pinned body rides the stable prefix. The
|
||||
// saved-fact index is retrieval-only and would move nothing.
|
||||
Activation: memory.ActivationPinned,
|
||||
Body: "The runtime-resume probe wrote this to move the stable prefix.",
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// mutateCoveredRow rewrites one provider-visible message inside the prefix the
|
||||
// projection folded, through the same rewrite save a rewind or a recovery
|
||||
// branch uses. This is the control arm: the digest's material changed, so the
|
||||
// projection must not be reused.
|
||||
func mutateCoveredRow(_ armRoot, before Observation) error {
|
||||
session, err := sessionstore.LoadSession(before.SessionPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msgs := session.Snapshot()
|
||||
covered := min(before.Sidecar.CoveredCount, len(msgs))
|
||||
target := -1
|
||||
for i := range covered {
|
||||
if msgs[i].Role == provider.RoleUser || msgs[i].Role == provider.RoleAssistant {
|
||||
target = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if target < 0 {
|
||||
return errUnexpected("a covered conversation row to mutate", covered)
|
||||
}
|
||||
msgs[target].Content += "\n[probe: covered row mutated]"
|
||||
session.Replace(msgs)
|
||||
return session.SaveRewrite(before.SessionPath)
|
||||
}
|
||||
|
||||
// editTailRow rewrites one message the projection does not claim to cover. The
|
||||
// covered hash cannot see it, so the projection should keep serving.
|
||||
func editTailRow(_ armRoot, before Observation) error {
|
||||
return rewriteSession(before, func(msgs []provider.Message, covered int) ([]provider.Message, error) {
|
||||
for i := covered; i < len(msgs); i++ {
|
||||
if msgs[i].Role == provider.RoleUser || msgs[i].Role == provider.RoleAssistant {
|
||||
msgs[i].Content += "\n[probe: tail row edited]"
|
||||
return msgs, nil
|
||||
}
|
||||
}
|
||||
return nil, errUnexpected("a live tail row to edit", covered)
|
||||
})
|
||||
}
|
||||
|
||||
// truncateTail drops everything past the fold boundary, leaving the covered
|
||||
// prefix exactly as the projection folded it.
|
||||
func truncateTail(_ armRoot, before Observation) error {
|
||||
return rewriteSession(before, func(msgs []provider.Message, covered int) ([]provider.Message, error) {
|
||||
return msgs[:covered], nil
|
||||
})
|
||||
}
|
||||
|
||||
// rewriteSession applies a transcript rewrite through the save a rewind or a
|
||||
// recovery branch uses, refusing an arm with no live tail to act on.
|
||||
func rewriteSession(before Observation, edit func([]provider.Message, int) ([]provider.Message, error)) error {
|
||||
session, err := sessionstore.LoadSession(before.SessionPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msgs := session.Snapshot()
|
||||
covered := before.Sidecar.CoveredCount
|
||||
if covered <= 0 || covered >= len(msgs) {
|
||||
return errUnexpected("a live tail past the fold", covered)
|
||||
}
|
||||
next, err := edit(msgs, covered)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
session.Replace(next)
|
||||
return session.SaveRewrite(before.SessionPath)
|
||||
}
|
||||
|
||||
func orchestrate(work, only, jsonOut string, keep bool) error {
|
||||
self, err := os.Executable()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if work == "" {
|
||||
work, err = os.MkdirTemp("", "runtime-resume-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !keep {
|
||||
defer os.RemoveAll(work)
|
||||
}
|
||||
}
|
||||
var results []armResult
|
||||
for _, a := range arms() {
|
||||
if only != "" && a.name != only {
|
||||
continue
|
||||
}
|
||||
result, err := runArm(self, work, a)
|
||||
if err != nil {
|
||||
return fmt.Errorf("arm %s: %w", a.name, err)
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
if len(results) == 0 {
|
||||
return fmt.Errorf("no arm matched %q", only)
|
||||
}
|
||||
fmt.Print(renderMatrix(results, work))
|
||||
return writeJSON(jsonOut, results)
|
||||
}
|
||||
|
||||
func runArm(self, work string, a arm) (armResult, error) {
|
||||
root := rootFor(filepath.Join(work, a.name))
|
||||
if err := root.requireFresh(); err != nil {
|
||||
return armResult{}, err
|
||||
}
|
||||
if err := root.create(); err != nil {
|
||||
return armResult{}, err
|
||||
}
|
||||
if err := spawn(self, "construct", root, a.name); err != nil {
|
||||
return armResult{}, fmt.Errorf("construct process: %w", err)
|
||||
}
|
||||
before, err := readObservation(root, "construct")
|
||||
if err != nil {
|
||||
return armResult{}, err
|
||||
}
|
||||
if a.lever != nil {
|
||||
if err := a.lever(root, before); err != nil {
|
||||
return armResult{}, fmt.Errorf("lever: %w", err)
|
||||
}
|
||||
}
|
||||
if _, ok := successorTurn(a.name); ok {
|
||||
if err := spawn(self, "successor", root, a.name); err != nil {
|
||||
return armResult{}, fmt.Errorf("successor process: %w", err)
|
||||
}
|
||||
}
|
||||
if err := spawn(self, "resume", root, a.name); err != nil {
|
||||
return armResult{}, fmt.Errorf("resume process: %w", err)
|
||||
}
|
||||
after, err := readObservation(root, "resume")
|
||||
if err != nil {
|
||||
return armResult{}, err
|
||||
}
|
||||
var extra *Observation
|
||||
if _, ok := successorTurn(a.name); ok {
|
||||
obs, err := readObservation(root, "successor")
|
||||
if err != nil {
|
||||
return armResult{}, err
|
||||
}
|
||||
extra = &obs
|
||||
} else if phase := extraPhase(a.name); phase != "" {
|
||||
obs, err := readObservation(root, phase)
|
||||
if err != nil {
|
||||
return armResult{}, err
|
||||
}
|
||||
extra = &obs
|
||||
}
|
||||
return classify(a, extra, before, after), nil
|
||||
}
|
||||
|
||||
// spawn runs one phase as a real child process and waits for it to exit. The
|
||||
// OS process is the measurement boundary: a nil-ed object can still be revived
|
||||
// by a package-level singleton, a live sink, or a cache, and none of those
|
||||
// survive here.
|
||||
func spawn(self, phase string, root armRoot, armName string) error {
|
||||
cmd := exec.Command(self, "-phase="+phase, "-root="+root.Dir, "-arm="+armName)
|
||||
cmd.Env = childEnv(root)
|
||||
cmd.Dir = root.Workspace
|
||||
cmd.Stdout = os.Stderr
|
||||
cmd.Stderr = os.Stderr
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func writeJSON(path string, results []armResult) error {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return nil
|
||||
}
|
||||
b, err := marshalResults(results)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, b, 0o644)
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/session/control"
|
||||
)
|
||||
|
||||
// Where a stop goes — and, first, whose stop it is. A controller owns the turns
|
||||
// it admits itself and cancels those; a synchronous Run belongs to the caller
|
||||
// that passed the context. Measuring the second with the first says nothing
|
||||
// about either.
|
||||
const (
|
||||
// armCancelTool is the control that says where a defect lives. If an
|
||||
// ordinary tool's context is cancelled and a delegation's is not, the fault
|
||||
// is in how a delegation derives its context; if neither is, it is upstream
|
||||
// of both.
|
||||
armCancelTool = "cancel-ordinary-tool"
|
||||
// armCancelBackground is the negative control, and the one that decides what
|
||||
// a fix may do: work already handed to a job has an owner of its own, and a
|
||||
// cancel that reached into it would take back the handoff.
|
||||
armCancelBackground = "cancel-background-handoff"
|
||||
// armCancelHeadlessOwner reads the other surface. A synchronous Run is never
|
||||
// admitted through the gate, so the controller holds no cancel for it and
|
||||
// the caller's own context holds the only one. Both halves are asserted: a
|
||||
// controller that started claiming this would be taking ownership from the
|
||||
// only holder that has it.
|
||||
armCancelHeadlessOwner = "cancel-headless-owner"
|
||||
)
|
||||
|
||||
func cancelRoutingArm(name string) bool {
|
||||
switch name {
|
||||
case armCancelTool, armCancelBackground, armCancelHeadlessOwner:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// guardedArm names the arms driven through a turn the controller admits itself,
|
||||
// which is the surface a person's Stop reaches.
|
||||
func guardedArm(name string) bool {
|
||||
return cancelTaskArm(name) || name == armCancelTool || name == armCancelBackground ||
|
||||
name == armSkillCancel || name == armHostCancel
|
||||
}
|
||||
|
||||
const (
|
||||
sleepSentinel = "PROBE-SLEEP"
|
||||
sleepCallID = "probe_sleep"
|
||||
// sleepSeconds outlasts the settling this probe waits through, so a shell
|
||||
// that ends promptly ended because something ended it.
|
||||
sleepSeconds = 30
|
||||
)
|
||||
|
||||
// sleepCall is an ordinary foreground tool that does nothing but wait. It is
|
||||
// the shell every turn can call, which is the point: it derives its context the
|
||||
// way every other tool does.
|
||||
func sleepCall() []provider.Chunk {
|
||||
args, _ := json.Marshal(map[string]any{
|
||||
"command": fmt.Sprintf("sleep %d", sleepSeconds),
|
||||
"description": "waits until something stops it",
|
||||
})
|
||||
return []provider.Chunk{{
|
||||
Type: provider.ChunkToolCall,
|
||||
ToolCall: &provider.ToolCall{ID: sleepCallID, Name: "bash", Arguments: string(args)},
|
||||
}}
|
||||
}
|
||||
|
||||
// runCancelRoutingConstruct drives the arm's work, stops it the way that
|
||||
// surface's owner stops it, and records what the stop reached. Nothing waits
|
||||
// for the work to end: whether it ends at all is the measurement.
|
||||
func runCancelRoutingConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
|
||||
sentinel := sleepSentinel
|
||||
if arm == armCancelBackground {
|
||||
sentinel = taskSentinel
|
||||
}
|
||||
prompt := fanOutTurn(turn+1, sentinel)
|
||||
ctx, cancelCaller := context.WithCancel(ctx)
|
||||
defer cancelCaller()
|
||||
if guardedArm(arm) {
|
||||
ctrl.Send(prompt)
|
||||
} else {
|
||||
go func() { _ = ctrl.Run(ctx, prompt) }()
|
||||
}
|
||||
if err := waitForCancelSubject(sink, prov, arm); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
|
||||
obs.Progress[probeStopReached] = stopReadings(ctrl, sink, prov, arm, cancelCaller)
|
||||
if err := writeObservation(root, obs); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Exit(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeStopReached carries what each stop reached into the observation. The key
|
||||
// names the probe so no row reads it as a status the kernel emitted.
|
||||
const probeStopReached = "probe:stop-reached"
|
||||
|
||||
// stopReadings issues this surface's stop and reports what it reached. The
|
||||
// headless arm issues both in order — the controller's, which owns nothing
|
||||
// there, and then the caller's, which owns everything.
|
||||
func stopReadings(ctrl *control.Controller, sink *graphSink, prov *scripted, arm string, cancelCaller context.CancelFunc) []string {
|
||||
// What the stop found before it was issued. Work already ended would make
|
||||
// every answer below meaningless, and this probe has made that mistake once.
|
||||
before := "live"
|
||||
if !workLive(sink, prov, arm) {
|
||||
before = "already gone"
|
||||
}
|
||||
ctrl.Cancel()
|
||||
first := "controller-cancel before=" + before + " " + reachedAfter(ctrl, sink, prov, arm)
|
||||
if arm != armCancelHeadlessOwner {
|
||||
return []string{first}
|
||||
}
|
||||
cancelCaller()
|
||||
return []string{first, "caller-cancel " + reachedAfter(ctrl, sink, prov, arm)}
|
||||
}
|
||||
|
||||
// reachedAfter waits out the settling this probe allows and reports what is
|
||||
// still live. The wait stays well below the shell's own sleep, so work that has
|
||||
// ended was ended by the stop rather than by finishing.
|
||||
func reachedAfter(ctrl *control.Controller, sink *graphSink, prov *scripted, arm string) string {
|
||||
stopped := time.Now()
|
||||
deadline := stopped.Add(8 * time.Second)
|
||||
for time.Now().Before(deadline) && workLive(sink, prov, arm) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
live := "gone"
|
||||
if workLive(sink, prov, arm) {
|
||||
live = "still running"
|
||||
}
|
||||
return fmt.Sprintf("turn=%s work=%s after=%s",
|
||||
turnStanding(ctrl, arm), live, time.Since(stopped).Round(time.Second))
|
||||
}
|
||||
|
||||
func workLive(sink *graphSink, prov *scripted, arm string) bool {
|
||||
if arm == armCancelBackground {
|
||||
return prov.fleets.held(childEntered(childHang)) && !prov.fleets.held(childLeft(childHang))
|
||||
}
|
||||
_, ended := sink.toolResult(sleepCallID)
|
||||
return !ended
|
||||
}
|
||||
|
||||
// turnStanding reports the gate only where the gate is the authority. A
|
||||
// synchronous Run is never admitted through it, so reading it there would say
|
||||
// "ended" about a turn it was never told about — which is how this probe first
|
||||
// mistook an ownership boundary for a cancellation defect.
|
||||
func turnStanding(ctrl *control.Controller, arm string) string {
|
||||
if !guardedArm(arm) {
|
||||
return "not admitted (caller-owned)"
|
||||
}
|
||||
if ctrl.Running() {
|
||||
return "still active"
|
||||
}
|
||||
return "ended"
|
||||
}
|
||||
|
||||
// waitForCancelSubject blocks until the work this arm is about to stop is
|
||||
// really under way: a stop that arrived before it would measure nothing.
|
||||
func waitForCancelSubject(sink *graphSink, prov *scripted, arm string) error {
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if arm == armCancelBackground {
|
||||
if prov.fleets.held(childEntered(childHang)) {
|
||||
return nil
|
||||
}
|
||||
} else if _, seen := sink.toolDispatched(sleepCallID); seen {
|
||||
// The shell starts with the dispatch, not with the event; a moment's
|
||||
// grace keeps the stop from racing its own subject.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
return nil
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return errUnexpected("work under way to stop", "nothing started")
|
||||
}
|
||||
|
||||
// cancelRoutingRows read the arm's own readings. The three arms want different
|
||||
// answers to the same question, which is why they run together: a fix that
|
||||
// cancels everything a session knows about passes one and fails two.
|
||||
func cancelRoutingRows(arm string, before Observation) []row {
|
||||
readings := before.Progress[probeStopReached]
|
||||
first, second := reading(readings, 0), reading(readings, 1)
|
||||
switch arm {
|
||||
case armCancelHeadlessOwner:
|
||||
return []row{
|
||||
ownershipRow("a controller stop on a turn it never admitted", first,
|
||||
"leaves the caller's work alone", stillRunning(first)),
|
||||
ownershipRow("the caller's own stop", second,
|
||||
"ends the work it owns", !stillRunning(second)),
|
||||
}
|
||||
case armCancelBackground:
|
||||
return []row{
|
||||
ownershipRow("the turn after a stop", first, "a cancelled turn ends",
|
||||
strings.Contains(first, "turn=ended")),
|
||||
ownershipRow("work already handed to a job", first,
|
||||
"a stop aimed at the turn leaves work the turn no longer owns", stillRunning(first)),
|
||||
}
|
||||
}
|
||||
return []row{
|
||||
ownershipRow("the turn after a stop", first, "a cancelled turn ends",
|
||||
strings.Contains(first, "turn=ended")),
|
||||
ownershipRow("the tool the turn was waiting on", first,
|
||||
"a stop reaches the work its turn is waiting on", !stillRunning(first)),
|
||||
}
|
||||
}
|
||||
|
||||
func ownershipRow(semantic, got, want string, ok bool) row {
|
||||
return row{
|
||||
Semantic: semantic, Authority: "the surface that owns this cancellation",
|
||||
Artifact: "none: in-process state", Reconstruction: want,
|
||||
Before: want, After: orNone(got), Verdict: held(ok),
|
||||
}
|
||||
}
|
||||
|
||||
func reading(readings []string, at int) string {
|
||||
if at < len(readings) {
|
||||
return readings[at]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func stillRunning(reading string) bool { return strings.Contains(reading, "work=still running") }
|
||||
@@ -0,0 +1,156 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// How a host can stand after dying with a question open. The names are the
|
||||
// outcomes, not a pass and its failures: two of them are designs a host may
|
||||
// legitimately choose, and only measurement says which one this is.
|
||||
const (
|
||||
outcomeResumable = "resumable" // the question is still there to answer
|
||||
outcomeInterrupted = "interrupted-explicit" // it is gone, and the host says a turn was cut
|
||||
outcomeLostSilent = "LOST-SILENT" // it is gone and nothing records that it existed
|
||||
)
|
||||
|
||||
// decisionRows classify what a restart inherited. The safety row comes first
|
||||
// because it is the only one with a single acceptable answer: whether the
|
||||
// decision survives is a design question, whether the effect it blocked stayed
|
||||
// blocked is not.
|
||||
func decisionRows(before, after Observation) []row {
|
||||
return []row{
|
||||
deferredRow(before, after),
|
||||
outcomeRow(before, after),
|
||||
barrierRow(before, after),
|
||||
modelContextRow(before, after),
|
||||
obligationRow(before, after),
|
||||
}
|
||||
}
|
||||
|
||||
// deferredRow asserts the write the question held back never ran — not while
|
||||
// the answer was pending, and not because a later process loaded the session.
|
||||
func deferredRow(before, after Observation) row {
|
||||
verdict := verdictHolds
|
||||
if before.Deferred.Executed || after.Deferred.Executed {
|
||||
verdict = verdictViolated
|
||||
}
|
||||
return row{
|
||||
Semantic: "the effect the decision held back", Authority: "the workspace",
|
||||
Artifact: before.Deferred.MarkerPath, Reconstruction: "the blocked write never runs, before or after the boundary",
|
||||
Before: ranOrNot(before), After: ranOrNot(after), Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func ranOrNot(o Observation) string {
|
||||
if o.Deferred.Executed {
|
||||
return "RAN"
|
||||
}
|
||||
return "did not run"
|
||||
}
|
||||
|
||||
// outcomeRow names which of the three standings the restart produced. It is
|
||||
// stated, not judged: a host that drops the question and records the
|
||||
// interruption is a defensible design, and one that drops both is not.
|
||||
func outcomeRow(before, after Observation) row {
|
||||
verdict := verdictHolds
|
||||
var outcome string
|
||||
switch {
|
||||
case len(after.Decisions) > 0:
|
||||
outcome = outcomeResumable
|
||||
case len(after.Interrupted) > 0:
|
||||
outcome = outcomeInterrupted
|
||||
case after.Obligation.InterruptionMarked > 0 || len(after.Obligation.UnansweredCalls) > 0:
|
||||
outcome = outcomeInterrupted
|
||||
default:
|
||||
outcome = outcomeLostSilent
|
||||
verdict = verdictViolated
|
||||
}
|
||||
return row{
|
||||
Semantic: "what the restart inherited", Authority: "approvalManager and the transcript",
|
||||
Artifact: "none for the question; the transcript for the obligation",
|
||||
Reconstruction: "resumable, interrupted-explicit, or nothing at all",
|
||||
Before: decisionSummary(before), After: outcome, Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func decisionSummary(o Observation) string {
|
||||
if len(o.Decisions) == 0 {
|
||||
return "no decision open"
|
||||
}
|
||||
var kinds []string
|
||||
for _, d := range o.Decisions {
|
||||
kinds = append(kinds, string(d.Kind))
|
||||
}
|
||||
return strings.Join(kinds, ", ") + " open"
|
||||
}
|
||||
|
||||
// barrierRow is the evidence behind an interrupted classification: a barrier
|
||||
// the host recorded before anyone could be asked, still open, with no owner in
|
||||
// this process waiting on it. It is deliberately not offered as answerable.
|
||||
func barrierRow(before, after Observation) row {
|
||||
return row{
|
||||
Semantic: "durable adjudication barriers", Authority: "the adjudication log",
|
||||
Artifact: "<stem>.adjudication.jsonl", Reconstruction: "open records with no live owner in this process",
|
||||
Before: barrierSummary(before), After: barrierSummary(after), Verdict: verdictStable,
|
||||
}
|
||||
}
|
||||
|
||||
func barrierSummary(o Observation) string {
|
||||
if len(o.Interrupted) == 0 {
|
||||
return "none"
|
||||
}
|
||||
var out []string
|
||||
for _, b := range o.Interrupted {
|
||||
out = append(out, b.Kind+":"+b.ID)
|
||||
}
|
||||
return join(out) + " (not answerable)"
|
||||
}
|
||||
|
||||
// modelContextRow is the other read surface: what the next request tells the
|
||||
// model. One block when an interruption stands, none when it does not, and it
|
||||
// must never arrive as something answerable.
|
||||
func modelContextRow(before, after Observation) row {
|
||||
verdict := verdictHolds
|
||||
if before.ModelSeesInterruption != expectedBlocks(before) || after.ModelSeesInterruption != expectedBlocks(after) {
|
||||
verdict = verdictViolated
|
||||
}
|
||||
return row{
|
||||
Semantic: "what the next request tells the model", Authority: "control, projected per request",
|
||||
Artifact: "none: derived, never stored", Reconstruction: "one block while an interruption stands, none otherwise",
|
||||
Before: modelContextSummary(before), After: modelContextSummary(after), Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func expectedBlocks(o Observation) int {
|
||||
if len(o.Interrupted) > 0 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func modelContextSummary(o Observation) string {
|
||||
return fmt.Sprintf("%d block(s) for %d interruption(s)", o.ModelSeesInterruption, len(o.Interrupted))
|
||||
}
|
||||
|
||||
// obligationRow shows the evidence behind that classification, so a reader can
|
||||
// see what the host still says rather than trusting the label.
|
||||
func obligationRow(before, after Observation) row {
|
||||
return row{
|
||||
Semantic: "what the transcript still owes", Authority: "the canonical transcript",
|
||||
Artifact: "<stem>.jsonl", Reconstruction: "tool calls with no result, and interruption records",
|
||||
Before: obligationSummary(before), After: obligationSummary(after), Verdict: verdictStable,
|
||||
}
|
||||
}
|
||||
|
||||
func obligationSummary(o Observation) string {
|
||||
return fmt.Sprintf("unanswered %s, interruption records %d",
|
||||
orNone(join(o.Obligation.UnansweredCalls)), o.Obligation.InterruptionMarked)
|
||||
}
|
||||
|
||||
func orNone(s string) string {
|
||||
if s == "" {
|
||||
return "none"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/agentgraph"
|
||||
"reasonix/internal/session/control"
|
||||
"reasonix/internal/state/execgraph"
|
||||
)
|
||||
|
||||
// The derivability arms. Everything before them asked what survives; these ask
|
||||
// what still has to be written down. A fact a restart can compute from what is
|
||||
// already durable needs no record of its own, and adding one would give it a
|
||||
// second owner for nothing.
|
||||
const (
|
||||
// armDeriveSkipBoth cuts a branch with two upstreams that both end without
|
||||
// an answer. Which of them the graph names is the question: the durable
|
||||
// facts show two, and the picture shows one.
|
||||
armDeriveSkipBoth = "derive-skip-both"
|
||||
// armDeriveSkipFlip is the same graph with the two failures in the other
|
||||
// order. If the named cause moves while the durable facts do not, the cause
|
||||
// is history rather than a consequence of the end state.
|
||||
armDeriveSkipFlip = "derive-skip-flip"
|
||||
// armDeriveAnswered is the negative control and the context sample at once:
|
||||
// two upstreams that both answer, one by completing and one by adopting, so
|
||||
// the dependent must run and both deliveries must be accounted for.
|
||||
armDeriveAnswered = "derive-answered"
|
||||
)
|
||||
|
||||
func deriveArm(name string) bool {
|
||||
switch name {
|
||||
case armDeriveSkipBoth, armDeriveSkipFlip, armDeriveAnswered:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func deriveSentinels(arm string) string {
|
||||
if arm == armDeriveAnswered {
|
||||
return fleetWarmSentinel + " " + fleetDeriveSentinel
|
||||
}
|
||||
return fleetDeriveSentinel
|
||||
}
|
||||
|
||||
// runDeriveConstruct lets the fan-out settle and dies before the turn ends, the
|
||||
// way every terminal arm does: a skip is published only in the closing delta.
|
||||
func runDeriveConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
|
||||
runCtx, cancelRun := context.WithCancel(ctx)
|
||||
defer cancelRun()
|
||||
go func() { _ = ctrl.Run(runCtx, fanOutTurn(turn+1, deriveSentinels(arm))) }()
|
||||
if err := waitForDerive(sink, arm, prov); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeObservation(root, capture("construct", arm, bootSystem, ctrl, sink, root)); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Exit(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForDerive holds until the arm's shape is on the graph. The skip arms
|
||||
// release their second failure once the first has been recorded, so which
|
||||
// upstream the fan-out names is decided rather than raced.
|
||||
func waitForDerive(sink *graphSink, arm string, prov *scripted) error {
|
||||
if arm == armDeriveAnswered {
|
||||
// Waiting on a count would be satisfied by the warm-up alone. The
|
||||
// delivery edge is what the arm compares, so it waits for that.
|
||||
return waitForShape(sink, func(g agentgraph.Graph) bool {
|
||||
return len(contextTargets(g)) > 0 && len(nodesInState(g, agentgraph.StateAdopted)) > 0
|
||||
}, "an adopted upstream and a delivered answer")
|
||||
}
|
||||
// The first failure decides which upstream is named — the fan-out cuts the
|
||||
// branch when it processes that result. Only then is the second released,
|
||||
// so the order is constructed rather than raced.
|
||||
if err := waitForShape(sink, func(g agentgraph.Graph) bool {
|
||||
return len(nodesInState(g, agentgraph.StateFailed)) > 0
|
||||
}, "the first upstream to fail"); err != nil {
|
||||
return err
|
||||
}
|
||||
prov.releaseOnce()
|
||||
// A skip is published only in the closing delta, so the group has to end
|
||||
// before the state the arm compares exists at all.
|
||||
return waitForShape(sink, func(g agentgraph.Graph) bool {
|
||||
return len(nodesInState(g, agentgraph.StateSkipped)) > 0 &&
|
||||
len(nodesInState(g, agentgraph.StateFailed)) >= 2
|
||||
}, "both upstreams failed and the dependent skipped")
|
||||
}
|
||||
|
||||
// contextTargets are the items the graph shows receiving an upstream answer.
|
||||
func contextTargets(g agentgraph.Graph) []string {
|
||||
var out []string
|
||||
for _, e := range g.Edges {
|
||||
if e.Kind == agentgraph.Context {
|
||||
out = append(out, e.To)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func waitForShape(sink *graphSink, ok func(agentgraph.Graph) bool, want string) error {
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
graph, _ := sink.snapshot()
|
||||
if ok(graph) {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
graph, _ := sink.snapshot()
|
||||
return errUnexpected(want, waitStates(graph))
|
||||
}
|
||||
|
||||
// answeredFromDurable is Answered() computed the way a restart would have to:
|
||||
// an adoption says so on its own opening, and everything else is the terminal
|
||||
// the sub-agent store kept, joined by the execution id the child records.
|
||||
func answeredFromDurable(o Observation) map[string]bool {
|
||||
terminal := map[string]string{}
|
||||
for _, f := range o.Children.Facts {
|
||||
if f.ParentToolCallID != "" {
|
||||
terminal[f.ParentToolCallID] = f.Status
|
||||
}
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for _, e := range o.Executions {
|
||||
out[e.ID] = e.Disposition == "adopted" || terminal[e.ID] == string(sessionstore.SubagentCompleted)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// impliedSkipped is the state a restart would derive: an item the orchestration
|
||||
// released without ever starting it, ordered behind something that did not
|
||||
// answer. Nothing here reads a disposition, because that is what is missing.
|
||||
func impliedSkipped(o Observation) string {
|
||||
answered := answeredFromDurable(o)
|
||||
var out []string
|
||||
for _, e := range o.Executions {
|
||||
if e.Started() || e.Queued() || e.SettledAt.IsZero() || len(e.DependsOn) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, up := range e.DependsOn {
|
||||
if !answered[up] {
|
||||
out = append(out, e.ID)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
// impliedSkipCauses are every upstream that could have been named, and the two
|
||||
// rules that would pick one: declaration order, and the earliest release. A
|
||||
// cause is derivable only if some rule always agrees with the picture.
|
||||
func impliedSkipCauses(o Observation) (all, byOrder, bySettle string) {
|
||||
answered := answeredFromDurable(o)
|
||||
byID := map[string]int{}
|
||||
for i, e := range o.Executions {
|
||||
byID[e.ID] = i
|
||||
}
|
||||
var every, first, earliest []string
|
||||
for _, e := range o.Executions {
|
||||
if e.Started() || e.Queued() || e.SettledAt.IsZero() || len(e.DependsOn) == 0 {
|
||||
continue
|
||||
}
|
||||
var unanswered []string
|
||||
for _, up := range e.DependsOn {
|
||||
if !answered[up] {
|
||||
unanswered = append(unanswered, up)
|
||||
}
|
||||
}
|
||||
if len(unanswered) == 0 {
|
||||
continue
|
||||
}
|
||||
every = append(every, e.ID+"<-{"+strings.Join(unanswered, ",")+"}")
|
||||
first = append(first, e.ID+"<-"+unanswered[0])
|
||||
pick := unanswered[0]
|
||||
for _, up := range unanswered[1:] {
|
||||
if i, j := byID[up], byID[pick]; o.Executions[i].SettledAt.Before(o.Executions[j].SettledAt) {
|
||||
pick = up
|
||||
}
|
||||
}
|
||||
earliest = append(earliest, e.ID+"<-"+pick)
|
||||
}
|
||||
sort.Strings(every)
|
||||
sort.Strings(first)
|
||||
sort.Strings(earliest)
|
||||
return join(every), join(first), join(earliest)
|
||||
}
|
||||
|
||||
// actualSkipCause is the upstream the fan-out actually named, read off the node
|
||||
// the graph settled. It is the picture's own answer, not a derivation.
|
||||
func actualSkipCause(o Observation) string {
|
||||
var out []string
|
||||
for _, n := range o.Graph.Nodes {
|
||||
if n.State == agentgraph.StateSkipped && n.Err != "" {
|
||||
out = append(out, n.ID+": "+n.Err)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return orNone(join(out))
|
||||
}
|
||||
|
||||
// impliedContextFromDurable predicts the delivery edges from durable facts
|
||||
// alone: a started item receives an answer from every upstream that answered.
|
||||
func impliedContextFromDurable(o Observation) string {
|
||||
answered := answeredFromDurable(o)
|
||||
var out []string
|
||||
for _, e := range o.Executions {
|
||||
if !e.Started() {
|
||||
continue
|
||||
}
|
||||
for _, up := range e.DependsOn {
|
||||
if answered[up] {
|
||||
out = append(out, e.ID+"<-"+up)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
// deriveRows are the three questions, kept apart. A state that derives says
|
||||
// nothing about whether the cause does, and one arm's agreement says nothing
|
||||
// about a rule that has only ever seen one sample.
|
||||
func deriveRows(arm string, before, after Observation) []row {
|
||||
rows := []row{
|
||||
joinRow(before, after),
|
||||
skipStateRow(before, after),
|
||||
contextDeriveRow(before, after),
|
||||
}
|
||||
if arm != armDeriveAnswered {
|
||||
rows = append(rows, skipCauseRow(before, after))
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
// joinRow is what every derivation below stands on: the child records name the
|
||||
// execution they ran for, so a restart can ask the store what each item ended
|
||||
// as. Without it none of the other rows could be computed at all.
|
||||
func joinRow(before, after Observation) row {
|
||||
return valueRow("executions the store can be asked about", "child metadata's parent tool-call id",
|
||||
"subagents/<ref>.meta.json", "joined to the journal by execution id",
|
||||
joinedExecutions(before), joinedExecutions(after), true)
|
||||
}
|
||||
|
||||
func joinedExecutions(o Observation) string {
|
||||
var out []string
|
||||
for _, f := range o.Children.Facts {
|
||||
if f.ParentToolCallID != "" {
|
||||
out = append(out, f.ParentToolCallID+"="+f.Status)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
// skipStateRow compares the state a restart would derive against the one the
|
||||
// picture drew. Agreement means StateSkipped needs no record of its own.
|
||||
func skipStateRow(before, after Observation) row {
|
||||
actual := join(nodesInState(graphOf(before), agentgraph.StateSkipped))
|
||||
implied := impliedSkipped(after)
|
||||
verdict := verdictHolds
|
||||
if implied != actual {
|
||||
verdict = verdictViolated
|
||||
}
|
||||
return row{
|
||||
Semantic: "skipped state: derived against drawn", Authority: "the journal joined to the store",
|
||||
Artifact: "<stem>.execution.jsonl + subagents/<ref>.meta.json",
|
||||
Reconstruction: "settled, never started, ordered behind something that did not answer",
|
||||
Before: "drawn " + orNone(actual), After: "derived " + orNone(implied), Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
// skipCauseRow keeps the harder question separate, and reads it three ways: the
|
||||
// upstream the picture named, the two rules this file works out on its own, and
|
||||
// what the production fold concludes. The independence matters — a probe that
|
||||
// only asked the implementation would agree with it by construction.
|
||||
func skipCauseRow(before, after Observation) row {
|
||||
all, byOrder, bySettle := impliedSkipCauses(after)
|
||||
return row{
|
||||
Semantic: "skipped cause: what the picture named, what the facts permit",
|
||||
Authority: "the fan-out's own choice against the durable upstream set",
|
||||
Artifact: "none: the reason text is not recorded",
|
||||
Reconstruction: "declaration order, or earliest released, over the upstreams that did not answer",
|
||||
Before: actualSkipCause(before),
|
||||
After: "candidates " + orNone(all) + " | by-order " + orNone(byOrder) +
|
||||
" | by-settle " + orNone(bySettle) + " | rebuild " + orNone(rebuiltSkipCauses(after)),
|
||||
Verdict: verdictStable,
|
||||
}
|
||||
}
|
||||
|
||||
// rebuiltSkipCauses is what the production fold names, asked the same question
|
||||
// through its own code path.
|
||||
func rebuiltSkipCauses(o Observation) string {
|
||||
children := make([]execgraph.ChildOutcome, 0, len(o.Children.Facts))
|
||||
for _, f := range o.Children.Facts {
|
||||
children = append(children, execgraph.ChildOutcome{
|
||||
Execution: f.ParentToolCallID, Ref: f.Ref, Status: f.Status,
|
||||
})
|
||||
}
|
||||
rebuilt := execgraph.Rebuild(o.Executions, children, nil)
|
||||
var out []string
|
||||
for _, n := range rebuilt.Graph.Nodes {
|
||||
if n.State != agentgraph.StateSkipped {
|
||||
continue
|
||||
}
|
||||
if cause := execgraph.SkipCause(o.Executions, children, n.ID); cause != "" {
|
||||
out = append(out, n.ID+"<-"+cause)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
// contextDeriveRow asks the same question of the delivery edge, now that a
|
||||
// restart can tell which upstreams answered.
|
||||
func contextDeriveRow(before, after Observation) row {
|
||||
actual, implied := contextEdges(before), impliedContextFromDurable(after)
|
||||
verdict := verdictHolds
|
||||
if implied != actual {
|
||||
verdict = verdictViolated
|
||||
}
|
||||
return row{
|
||||
Semantic: "context edges: derived against drawn", Authority: "the journal joined to the store",
|
||||
Artifact: "<stem>.execution.jsonl + subagents/<ref>.meta.json",
|
||||
Reconstruction: "a started item receives from every upstream that answered",
|
||||
Before: "drawn " + orNone(actual), After: "derived " + orNone(implied), Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
// deriveArmInvalid refuses an arm that did not reach the shape it compares.
|
||||
func deriveArmInvalid(arm string, before Observation) string {
|
||||
if arm == armDeriveAnswered {
|
||||
if len(nodesInState(graphOf(before), agentgraph.StateAdopted)) == 0 {
|
||||
return "no item was adopted, so the arm never had an answered upstream that did not run"
|
||||
}
|
||||
if contextEdges(before) == "" {
|
||||
return "no answer was delivered, so there is no context edge to derive"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
if len(nodesInState(graphOf(before), agentgraph.StateSkipped)) == 0 {
|
||||
return "no dependent was skipped; the fan-out ended as " + orNone(terminalSummary(before))
|
||||
}
|
||||
if len(nodesInState(graphOf(before), agentgraph.StateFailed)) < 2 {
|
||||
return "only one upstream ended without an answer, so the cause was never ambiguous"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/agentgraph"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/session/control"
|
||||
)
|
||||
|
||||
// The two ephemeral entry points, read against the execution read model's own
|
||||
// authority: the snapshot is what a reader starts from and what a gap restores,
|
||||
// and the stream is a projection of the same facts. A node only the stream has
|
||||
// is therefore not a lighter node but one the authority cannot account for, and
|
||||
// these arms ask whether either entry point produces one.
|
||||
const (
|
||||
armEphemeralTask = "ephemeral-task-authority"
|
||||
armEphemeralSkill = "ephemeral-skill-authority"
|
||||
)
|
||||
|
||||
func ephemeralArm(name string) bool {
|
||||
return name == armEphemeralTask || name == armEphemeralSkill
|
||||
}
|
||||
|
||||
const (
|
||||
ephemeralSentinel = "PROBE-EPHEMERAL"
|
||||
// ephemeralCallID is the call the model makes. Both entry points use it, so
|
||||
// the two arms differ by which tool ran and by nothing else.
|
||||
ephemeralCallID = "probe_ephemeral"
|
||||
)
|
||||
|
||||
// ephemeralCall dispatches the entry point this arm measures. Its child hangs,
|
||||
// because every row here is about a run that is still happening: an ephemeral
|
||||
// execution that has ended owes no reader anything.
|
||||
func ephemeralCall(arm string) []provider.Chunk {
|
||||
name, args := "read_only_task", map[string]any{
|
||||
"prompt": childHang + " ephemeral research", "description": "still executing at death",
|
||||
}
|
||||
if arm == armEphemeralSkill {
|
||||
name, args = "read_only_skill", map[string]any{
|
||||
"name": probeSkillName, "arguments": childHang + " ephemeral research",
|
||||
}
|
||||
}
|
||||
raw, _ := json.Marshal(args)
|
||||
return []provider.Chunk{{
|
||||
Type: provider.ChunkToolCall,
|
||||
ToolCall: &provider.ToolCall{ID: ephemeralCallID, Name: name, Arguments: string(raw)},
|
||||
}}
|
||||
}
|
||||
|
||||
// runEphemeralConstruct drives one ephemeral run and reads three surfaces at the
|
||||
// same instant, while its child is inside the provider: what the stream drew,
|
||||
// what the authority holds, and what a reader is left with once a gap makes it
|
||||
// start from the authority again.
|
||||
func runEphemeralConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
|
||||
go func() { _ = ctrl.Run(ctx, fanOutTurn(turn+1, ephemeralSentinel)) }()
|
||||
if err := waitForEphemeral(prov); err != nil {
|
||||
return err
|
||||
}
|
||||
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
|
||||
obs.Progress[probeChildEntered] = []string{fmt.Sprint(prov.fleets.held(childEntered(childHang)))}
|
||||
// One instant, three readings. Taken here because two of them cannot be
|
||||
// taken later at all: the authority is recomputed per call, and what a gap
|
||||
// leaves is a claim about the moment the gap happens.
|
||||
live, _ := sink.snapshot()
|
||||
authority := ctrl.ExecutionGraph()
|
||||
obs.Progress[probeEphemeralLive] = []string{ephemeralNodes(live)}
|
||||
obs.Progress[probeEphemeralAuthority] = []string{ephemeralNodes(authority.Graph)}
|
||||
obs.Progress[probeEphemeralAfterGap] = []string{ephemeralNodes(afterGap(authority.Graph))}
|
||||
if err := writeObservation(root, obs); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Exit(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// The three readings one instant produces. They ride the observation because
|
||||
// none of them survives the process, and the arm's whole question is what a
|
||||
// reader could see while the run was happening.
|
||||
const (
|
||||
probeEphemeralLive = "probe:ephemeral-live"
|
||||
probeEphemeralAuthority = "probe:ephemeral-authority"
|
||||
probeEphemeralAfterGap = "probe:ephemeral-after-gap"
|
||||
)
|
||||
|
||||
// afterGap is what a view holds once a dropped stream makes it start from the
|
||||
// authority again: the snapshot, and only the snapshot. A resync replaces the
|
||||
// folded state rather than merging into it — that is what makes the snapshot the
|
||||
// authority — so whatever the fold held and it does not has left the screen.
|
||||
func afterGap(authority agentgraph.Graph) agentgraph.Graph { return authority }
|
||||
|
||||
func waitForEphemeral(prov *scripted) error {
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if prov.fleets.held(childEntered(childHang)) {
|
||||
// Let the picture settle: a reading taken the instant the child
|
||||
// arrives could miss a delta that was already on its way.
|
||||
time.Sleep(time.Second)
|
||||
return nil
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return errUnexpected("an ephemeral child inside the provider", "it never reached one")
|
||||
}
|
||||
|
||||
// ephemeralNodes renders the nodes belonging to this arm's run, worker and group
|
||||
// alike: a group card drawn for work the authority cannot account for is the
|
||||
// same defect as a worker one.
|
||||
func ephemeralNodes(g agentgraph.Graph) string {
|
||||
var out []string
|
||||
for _, n := range g.Nodes {
|
||||
if strings.HasPrefix(n.ID, ephemeralCallID) {
|
||||
out = append(out, n.ID+":"+orDash(string(n.State)))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return orNone(join(out))
|
||||
}
|
||||
|
||||
func ephemeralRows(arm string, before, after Observation) []row {
|
||||
return []row{
|
||||
ephemeralLiveRow(before),
|
||||
ephemeralAuthorityRow(before),
|
||||
ephemeralGapRow(before),
|
||||
ephemeralRestartRow(before, after),
|
||||
ephemeralDurableRow(before, after),
|
||||
ephemeralContractRow(arm, before),
|
||||
}
|
||||
}
|
||||
|
||||
const authorityName = "Controller.ExecutionGraph (the snapshot a reader starts from)"
|
||||
|
||||
func ephemeralLiveRow(before Observation) row {
|
||||
drawn := firstProgress(before, probeEphemeralLive)
|
||||
return row{
|
||||
Semantic: "what the stream drew while it ran", Authority: liveAuthority,
|
||||
Artifact: "none observed", Reconstruction: "in-process only",
|
||||
Before: drawn, After: "—", Verdict: presenceVerdict(drawn != "none"),
|
||||
}
|
||||
}
|
||||
|
||||
func ephemeralAuthorityRow(before Observation) row {
|
||||
held := firstProgress(before, probeEphemeralAuthority)
|
||||
return row{
|
||||
Semantic: "what the authority held at the same instant", Authority: authorityName,
|
||||
Artifact: "<stem>.execution.jsonl + subagents/<ref>.meta.json",
|
||||
Reconstruction: "execgraph.Rebuild over durable facts, recomputed on demand",
|
||||
Before: held, After: "—", Verdict: presenceVerdict(held != "none"),
|
||||
}
|
||||
}
|
||||
|
||||
// ephemeralGapRow is the one that decides. A reader that loses the stream starts
|
||||
// from the authority again, so whatever the fold held and the authority does not
|
||||
// leaves the screen — while the child it named is still holding a slot.
|
||||
func ephemeralGapRow(before Observation) row {
|
||||
left := firstProgress(before, probeEphemeralAfterGap)
|
||||
return row{
|
||||
Semantic: "what a reader is left with after a gap", Authority: authorityName,
|
||||
Artifact: "none: the fold is replaced, never merged",
|
||||
Reconstruction: "a resync starts from the authority",
|
||||
Before: left, After: "—", Verdict: presenceVerdict(left != "none"),
|
||||
}
|
||||
}
|
||||
|
||||
func ephemeralRestartRow(before, after Observation) row {
|
||||
return row{
|
||||
Semantic: "what a restart can say about it", Authority: rebuildAuthority,
|
||||
Artifact: "<stem>.execution.jsonl",
|
||||
Reconstruction: "execgraph.Rebuild in the next process",
|
||||
Before: "executing inside the provider",
|
||||
After: ephemeralNodes(rebuiltGraph(after).Graph),
|
||||
Verdict: presenceVerdict(ephemeralNodes(rebuiltGraph(after).Graph) != "none"),
|
||||
}
|
||||
}
|
||||
|
||||
// ephemeralDurableRow is the contract both entry points already promise. It is
|
||||
// read rather than assumed, because a convergence that made one of them visible
|
||||
// by recording it would satisfy the rows above and break this one.
|
||||
func ephemeralDurableRow(before, after Observation) row {
|
||||
return presenceRow("journal and store", crossAuthority,
|
||||
"<stem>.execution.jsonl, subagents/<ref>.meta.json",
|
||||
"what an ephemeral entry point promises to leave: nothing",
|
||||
ephemeralDurable(before), ephemeralDurable(after))
|
||||
}
|
||||
|
||||
func ephemeralDurable(o Observation) []string {
|
||||
var out []string
|
||||
for _, e := range o.Executions {
|
||||
if strings.HasPrefix(e.ID, ephemeralCallID) {
|
||||
out = append(out, "journal:"+e.ID)
|
||||
}
|
||||
}
|
||||
for _, f := range o.Children.Facts {
|
||||
if strings.HasPrefix(f.ParentToolCallID, ephemeralCallID) {
|
||||
out = append(out, "store:"+f.ParentToolCallID+"="+f.Status)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// ephemeralContractRow states the authority contract as a conjunction. A node
|
||||
// the stream drew, an authority that cannot account for it, and a child still
|
||||
// executing: that is not a lighter kind of node, it is one that leaves the
|
||||
// screen at the next gap while the work it named goes on holding a slot.
|
||||
func ephemeralContractRow(arm string, before Observation) row {
|
||||
live := firstProgress(before, probeEphemeralLive) != "none"
|
||||
authority := firstProgress(before, probeEphemeralAuthority) != "none"
|
||||
running := firstProgress(before, probeChildEntered) == "true"
|
||||
answer, verdict := "no node was drawn: nothing claims what the authority cannot hold", verdictHolds
|
||||
switch {
|
||||
case !running:
|
||||
answer, verdict = "the child never reached the provider", verdictNotMeasured
|
||||
case live && !authority:
|
||||
answer, verdict = "drawn by the stream, absent from the authority, and still executing", verdictViolated
|
||||
case live && authority:
|
||||
answer, verdict = "drawn, and the authority accounts for it", verdictHolds
|
||||
}
|
||||
return row{
|
||||
Semantic: "a node the authority cannot account for", Authority: crossAuthority,
|
||||
Artifact: "none, which is the point",
|
||||
Reconstruction: "the stream and the snapshot, read at one instant",
|
||||
Before: answer, After: "—", Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
// ephemeralArmInvalid reports the premise this arm could not establish.
|
||||
func ephemeralArmInvalid(before Observation) string {
|
||||
if firstProgress(before, probeChildEntered) != "true" {
|
||||
return "the ephemeral child never reached the provider, so nothing was executing at the reading"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,641 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/agentgraph"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/session/control"
|
||||
)
|
||||
|
||||
// The three fan-out arms. Each dies at a different point in one fleet's life,
|
||||
// because "what survives" has no single answer: an item that finished, one
|
||||
// still executing, and one the graph knows about without ever running are
|
||||
// carried by different host state, and only separating them says which.
|
||||
const (
|
||||
// armGraphCompleted lets the fleet finish and the turn close before the
|
||||
// process exits. Nothing is interrupted, so a loss here is not about
|
||||
// interruption at all — it says execution provenance itself is ephemeral.
|
||||
armGraphCompleted = "graph-completed"
|
||||
// armGraphRunning kills the process with a child mid-execution. It asks the
|
||||
// one question a completed fleet cannot: what does the new process believe
|
||||
// about work whose owner is gone?
|
||||
armGraphRunning = "graph-running"
|
||||
// armGraphMixed carries every state at once — completed, running, adopted,
|
||||
// failed, skipped — plus a read grant, a write grant, and a dependency edge.
|
||||
// It is the density arm: one death, every semantic the graph claims to hold.
|
||||
armGraphMixed = "graph-mixed"
|
||||
)
|
||||
|
||||
func graphArm(name string) bool {
|
||||
return name == armGraphCompleted || name == armGraphRunning || name == armGraphMixed
|
||||
}
|
||||
|
||||
// Child sentinels. A child's prompt decides what its run does, and the probe
|
||||
// reads the prompt off the request's last user message rather than scanning the
|
||||
// whole conversation: the parent's own transcript quotes every fleet argument,
|
||||
// so a scan would make the parent answer as its own children.
|
||||
const (
|
||||
childDone = "PROBE-CHILD-DONE"
|
||||
childHang = "PROBE-CHILD-HANG"
|
||||
childFail = "PROBE-CHILD-FAIL"
|
||||
)
|
||||
|
||||
// Parent sentinels, one per fleet the arm dispatches.
|
||||
const (
|
||||
fleetWarmSentinel = "PROBE-FLEET-WARM"
|
||||
fleetMixedSentinel = "PROBE-FLEET-MIXED"
|
||||
fleetPairSentinel = "PROBE-FLEET-PAIR"
|
||||
// The scheduler arms. Holder fills a ceiling and never ends; refused asks
|
||||
// for admission the scheduler must deny. Two sentinels rather than one
|
||||
// because the claim and transition arms need the holder to be already
|
||||
// admitted before the second fleet is dispatched.
|
||||
fleetHolderSentinel = "PROBE-FLEET-HOLDER"
|
||||
fleetRefusedSentinel = "PROBE-FLEET-REFUSED"
|
||||
// The terminal arms. Their fan-outs run to completion, because a skip is
|
||||
// published only in the closing delta; the turn is then held open so the
|
||||
// process still dies mid-turn.
|
||||
fleetTerminalSentinel = "PROBE-FLEET-TERMINAL"
|
||||
// Named so it is not contained in PROBE-PARALLEL-IDENTITY. Sentinels are
|
||||
// matched by substring, so one inside another dispatches a fan-out its arm
|
||||
// never asked for — this pair silently ran cancelParallel in the identity
|
||||
// arm, whose rows then read a scenario nobody wrote.
|
||||
parallelSentinel = "PROBE-PARALLEL-CANCEL"
|
||||
fleetOutcomesSentinel = "PROBE-FLEET-OUTCOMES"
|
||||
fleetDeriveSentinel = "PROBE-FLEET-DERIVE"
|
||||
fleetIdentitySentinel = "PROBE-FLEET-IDENTITY"
|
||||
parallelIdentity = "PROBE-PARALLEL-IDENTITY"
|
||||
// The active-store arm. Its fan-out shares one ceiling with a lone
|
||||
// delegation, so the four populations it needs are reached in one death.
|
||||
fleetActiveSentinel = "PROBE-FLEET-ACTIVE"
|
||||
)
|
||||
|
||||
// childHold is a child that reports holding its slot before it blocks, so a
|
||||
// later fleet can be dispatched knowing the ceiling is already occupied. Timing
|
||||
// it any other way races the scheduler.
|
||||
const (
|
||||
childHold = "PROBE-CHILD-HOLD"
|
||||
// childRelease finishes only when the arm frees it, which is how capacity
|
||||
// is returned while a refusal is still standing.
|
||||
childRelease = "PROBE-CHILD-RELEASE"
|
||||
// childFailLate fails only once the arm frees it, so two failures in one
|
||||
// fan-out have a decided order instead of a raced one. Its wording shares no
|
||||
// prefix with childFail: the sentinels are matched by substring, so a name
|
||||
// containing another's makes the shorter one win and the delay disappear.
|
||||
childFailLate = "PROBE-CHILD-DELAYEDFAIL"
|
||||
)
|
||||
|
||||
// probeClaimPath is the write path the claim arm overlaps. The refused writer
|
||||
// asks for a path inside it, which is a conflict no ceiling change releases.
|
||||
const (
|
||||
probeClaimPath = "probe-claim"
|
||||
probeClaimInner = "probe-claim/inner"
|
||||
)
|
||||
|
||||
// subagentRefPattern finds the references an earlier fleet's aggregate reported.
|
||||
// The mixed arm adopts one of them, which is the only way to reach StateAdopted
|
||||
// through the tool's own contract rather than by constructing the state.
|
||||
var subagentRefPattern = regexp.MustCompile(`sa_[A-Za-z0-9_-]+`)
|
||||
|
||||
// childScript answers one delegated run. Hanging is expressed as a context read
|
||||
// so the child holds its scheduler slot until the process dies — a sleep long
|
||||
// enough to look the same would still be a race against the probe's deadline.
|
||||
func (s *scripted) childScript(ctx context.Context, sentinel string) []provider.Chunk {
|
||||
// Reaching here is the child's own proof that it started, and returning is
|
||||
// its proof that something let it go: an arm asking whether a cancellation
|
||||
// reached a running child has no other way to know.
|
||||
s.fleets.first(childEntered(sentinel))
|
||||
defer s.fleets.first(childLeft(sentinel))
|
||||
switch sentinel {
|
||||
case childHold:
|
||||
// Reaching a provider call means the slot is already held, which is the
|
||||
// only moment a later fleet can be dispatched against a full ceiling
|
||||
// without racing it.
|
||||
s.holding.Do(func() { close(s.held) })
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
case childRelease:
|
||||
<-s.release
|
||||
return append(text("Child released."), done())
|
||||
case childFailLate:
|
||||
<-s.release
|
||||
return []provider.Chunk{{Type: provider.ChunkError, Err: fmt.Errorf("probe child failed on release")}}
|
||||
case childHang:
|
||||
<-ctx.Done()
|
||||
// The context closed, and this is the only place that can say so: a
|
||||
// caller that stopped waiting leaves the same absence behind.
|
||||
s.fleets.first(childCtxDone(sentinel))
|
||||
return nil
|
||||
case childFail:
|
||||
return []provider.Chunk{{Type: provider.ChunkError, Err: fmt.Errorf("probe child failed on purpose")}}
|
||||
default:
|
||||
return append(text("Child answered."), done())
|
||||
}
|
||||
}
|
||||
|
||||
// askedInPrompt reports whether this request's own user turns carry a sentinel.
|
||||
// The user side only: a fleet's arguments live in an assistant message, so a
|
||||
// whole-conversation scan makes a parent answer as one of its own children.
|
||||
// Every user turn, not the last: the host appends its own notes after the
|
||||
// prompt, and the last user message in a request is frequently one of those.
|
||||
func askedInPrompt(req provider.Request, sentinel string) bool {
|
||||
for _, m := range req.Messages {
|
||||
if m.Role == provider.RoleUser && strings.Contains(m.Content, sentinel) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// childSentinel reports which child script this request is, empty when the
|
||||
// request is a parent turn.
|
||||
func childSentinel(req provider.Request) string {
|
||||
for _, sentinel := range []string{childDone, childHang, childFail, childHold, childRelease, childFailLate} {
|
||||
if askedInPrompt(req, sentinel) {
|
||||
return sentinel
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// fanOutTurnRecorded reports whether the dispatching turn is in these messages.
|
||||
// The probe reads its own sentinel rather than counting rows: a turn appended
|
||||
// at some later point would still be there, and only the prompt says whether
|
||||
// this is the turn the fan-out was opened in.
|
||||
func fanOutTurnRecorded(msgs []provider.Message) bool {
|
||||
for _, m := range msgs {
|
||||
if strings.Contains(m.Content, fleetPairSentinel) || strings.Contains(m.Content, fleetMixedSentinel) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fleetCall renders one fleet dispatch. Nothing closes the round it opens: all
|
||||
// three arms die inside the turn, so the host's final-answer readiness check
|
||||
// never runs and the arms measure a process boundary rather than a task list.
|
||||
func fleetCall(id string, tasks []map[string]any) []provider.Chunk {
|
||||
args, _ := json.Marshal(map[string]any{"tasks": tasks})
|
||||
return []provider.Chunk{{
|
||||
Type: provider.ChunkToolCall,
|
||||
ToolCall: &provider.ToolCall{ID: id, Name: "fleet", Arguments: string(args)},
|
||||
}}
|
||||
}
|
||||
|
||||
// warmFleet finishes two children so a later fleet has a completed reference to
|
||||
// adopt. Both are read-only: nothing about the warm-up is under measurement.
|
||||
func warmFleet() []provider.Chunk {
|
||||
return fleetCall("probe_fleet_warm", []map[string]any{
|
||||
{"id": "w1", "prompt": childDone + " warm one", "description": "warm one", "read_only": true},
|
||||
{"id": "w2", "prompt": childDone + " warm two", "description": "warm two", "read_only": true},
|
||||
})
|
||||
}
|
||||
|
||||
// pairFleet is the two-item shape both single-state arms use. The running arm
|
||||
// dies with the second item still executing; the completed arm lets both
|
||||
// finish, so the two differ by when the process exits and nothing else.
|
||||
func pairFleet(second string) []provider.Chunk {
|
||||
return fleetCall("probe_fleet_pair", []map[string]any{
|
||||
{"id": "p1", "prompt": childDone + " pair one", "description": "pair one", "read_only": true},
|
||||
{"id": "p2", "prompt": second + " pair two", "description": "pair two", "read_only": true},
|
||||
})
|
||||
}
|
||||
|
||||
// mixedFleet is the density arm's dispatch: one item per terminal the graph can
|
||||
// record, plus one still running, and the two grants told apart. The write item
|
||||
// declares a path so preflight admits it beside the readers; it never writes,
|
||||
// because the grant is the envelope under measurement, not the effect.
|
||||
func mixedFleet(adoptRef string) []provider.Chunk {
|
||||
return fleetCall("probe_fleet_mixed", []map[string]any{
|
||||
// Model and effort are named so the identity row measures something: two
|
||||
// nodes agreeing that a field is empty proves nothing about recovering it.
|
||||
{"id": "m1", "prompt": childDone + " mixed one", "description": "completes", "read_only": true,
|
||||
"model": probeModelRef, "effort": "high"},
|
||||
{"id": "m2", "prompt": childFail + " mixed two", "description": "fails", "read_only": true},
|
||||
{"id": "m3", "prompt": childDone + " mixed three", "description": "skipped by m2", "read_only": true, "depends_on": []string{"m2"}},
|
||||
{"id": "m4", "adopt_ref": adoptRef, "description": "adopted"},
|
||||
{"id": "m5", "prompt": childHang + " mixed five", "description": "running at death", "write_paths": []string{"probe-fanout-out"}},
|
||||
})
|
||||
}
|
||||
|
||||
// opens reports whether this round is the one that dispatches that fleet.
|
||||
func (s *scripted) opens(req provider.Request, sentinel string) bool {
|
||||
return askedInPrompt(req, sentinel) && s.fleets.first(sentinel)
|
||||
}
|
||||
|
||||
// opensRun is the same for the fan-out an arm is built around: the terminal and
|
||||
// derive arms hold the turn open once theirs has gone out.
|
||||
func (s *scripted) opensRun(req provider.Request, sentinel string) bool {
|
||||
if !s.opens(req, sentinel) {
|
||||
return false
|
||||
}
|
||||
s.opened.Store(true)
|
||||
return true
|
||||
}
|
||||
|
||||
// fanOut answers a parent turn that opens a fleet, at most once per sentinel.
|
||||
// The sentinel is read off the current user turn, and the latch is what stops
|
||||
// the round that receives the aggregate from dispatching the same fleet again.
|
||||
func (s *scripted) fanOut(req provider.Request) ([]provider.Chunk, bool) {
|
||||
switch {
|
||||
case s.opens(req, fleetWarmSentinel):
|
||||
return append(warmFleet(), done()), true
|
||||
case s.opens(req, fleetPairSentinel):
|
||||
if s.arm == armWaitSlots {
|
||||
return append(slotsFleet(), done()), true
|
||||
}
|
||||
second := childDone
|
||||
if s.arm == armGraphRunning {
|
||||
second = childHang
|
||||
}
|
||||
return append(pairFleet(second), done()), true
|
||||
case s.opens(req, fleetSettledSentinel):
|
||||
return append(settledFleet(), done()), true
|
||||
case s.opens(req, fleetLiveSentinel):
|
||||
return append(liveFleet(), done()), true
|
||||
case s.opens(req, fleetMixedSentinel):
|
||||
if s.arm == armTerminalAdopted {
|
||||
return append(adoptOnlyFleet(adoptableRef(req)), done()), true
|
||||
}
|
||||
if s.arm == armUIGraphMixed {
|
||||
return append(uiMixedFleet(adoptableRef(req)), done()), true
|
||||
}
|
||||
return append(mixedFleet(adoptableRef(req)), done()), true
|
||||
case s.opensRun(req, fleetIdentitySentinel):
|
||||
return append(identityFleet(adoptableRef(req)), done()), true
|
||||
case s.opens(req, parallelIdentity):
|
||||
return append(identityParallel(), done()), true
|
||||
case s.opensRun(req, fleetDeriveSentinel):
|
||||
return append(deriveFleet(s.arm, adoptableRef(req)), done()), true
|
||||
case s.opensRun(req, fleetOutcomesSentinel):
|
||||
return append(outcomesFleet(), done()), true
|
||||
case s.opensRun(req, fleetTerminalSentinel):
|
||||
return append(terminalFleet(s.arm), done()), true
|
||||
case s.opensRun(req, parallelSentinel):
|
||||
return append(cancelParallel(), done()), true
|
||||
case s.fleets.held(taskSentinel) && s.opens(req, fleetActiveSentinel):
|
||||
// The delegation goes first and its own child says when the slot is
|
||||
// taken. Dispatching before that lets the fan-out take the whole
|
||||
// ceiling, and the arm loses the entry point it exists to compare.
|
||||
<-s.held
|
||||
return append(activeStoreFleet(), done()), true
|
||||
case s.opens(req, fleetHolderSentinel):
|
||||
return append(holderFleet(s.arm), done()), true
|
||||
case s.opens(req, fleetRefusedSentinel):
|
||||
// The holder's own child says when its ceiling is occupied. Dispatching
|
||||
// before that lets the refused item win the race and start.
|
||||
<-s.held
|
||||
return append(refusedFleet(s.arm), done()), true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// holderFleet occupies the ceiling this arm measures. The slots arm needs no
|
||||
// holder — its refused item is in the same fleet as the runs that fill the
|
||||
// ceiling — so only the arms that need one dispatch it in the background.
|
||||
func holderFleet(arm string) []provider.Chunk {
|
||||
tasks := []map[string]any{
|
||||
{"id": "h1", "prompt": childHold + " holder", "description": "holds the ceiling", "write_paths": []string{probeClaimPath}},
|
||||
{"id": "h2", "prompt": childHang + " filler", "description": "filler", "read_only": true},
|
||||
}
|
||||
if arm == armWaitTransition {
|
||||
// The second run is what the arm later releases: freeing total capacity
|
||||
// while the writer ceiling stays full is how the blocker changes.
|
||||
tasks[1] = map[string]any{"id": "h2", "prompt": childRelease + " releasable", "description": "released mid-wait", "read_only": true}
|
||||
}
|
||||
if bothHoldersReport(arm) {
|
||||
tasks[1] = map[string]any{"id": "h2", "prompt": childHold + " second holder", "description": "holds the ceiling", "read_only": true}
|
||||
}
|
||||
return backgroundFleetCall("probe_fleet_holder", tasks)
|
||||
}
|
||||
|
||||
// bothHoldersReport names the arms that leave one slot free. Either holder may
|
||||
// win it, so both report holding, and the refused delegation's own child is then
|
||||
// the only arrival that means anything — a filler that hung under the same
|
||||
// sentinel would be read as the subject having started.
|
||||
func bothHoldersReport(arm string) bool {
|
||||
return queuedTaskArm(arm) || queuedSkillArm(arm) || queuedHostArm(arm)
|
||||
}
|
||||
|
||||
// refusedFleet asks for admission the scheduler must deny, with every check
|
||||
// above this arm's cause already cleared.
|
||||
func refusedFleet(arm string) []provider.Chunk {
|
||||
refused := map[string]any{"id": "r1", "prompt": childHang + " refused", "description": "refused admission"}
|
||||
switch arm {
|
||||
case armWaitWriters:
|
||||
// Disjoint from the holder's path, so only the writer ceiling can
|
||||
// refuse it.
|
||||
refused["write_paths"] = []string{"probe-writers-own"}
|
||||
case armWaitClaim:
|
||||
refused["write_paths"] = []string{probeClaimInner}
|
||||
default:
|
||||
refused["write_paths"] = []string{"probe-transition-own"}
|
||||
}
|
||||
return fleetCall("probe_fleet_refused", []map[string]any{
|
||||
refused,
|
||||
{"id": "r2", "prompt": childHang + " sibling", "description": "sibling", "read_only": true},
|
||||
})
|
||||
}
|
||||
|
||||
// terminalFleet runs to completion holding the disposition its arm measures.
|
||||
// The skipped arm cuts a branch with a failure; the context arm delivers a real
|
||||
// answer across an ordering edge, which is the only way a context edge appears.
|
||||
func terminalFleet(arm string) []provider.Chunk {
|
||||
upstream := map[string]any{"id": "up", "prompt": childDone + " upstream", "description": "upstream", "read_only": true}
|
||||
if arm == armTerminalSkippedDep {
|
||||
upstream["prompt"] = childFail + " upstream"
|
||||
}
|
||||
return fleetCall("probe_fleet_terminal", []map[string]any{
|
||||
upstream,
|
||||
{"id": "down", "prompt": childDone + " downstream", "description": "downstream", "read_only": true, "depends_on": []string{"up"}},
|
||||
})
|
||||
}
|
||||
|
||||
// cancelParallel opens more items than the session admits at once, so the arm
|
||||
// can interrupt the group while one has started and another has not. That split
|
||||
// is what parallel_tasks reports as cancelled against skipped.
|
||||
func cancelParallel() []provider.Chunk {
|
||||
args, _ := json.Marshal(map[string]any{"tasks": []map[string]any{
|
||||
{"prompt": childHang + " first", "description": "started"},
|
||||
{"prompt": childHang + " second", "description": "never started"},
|
||||
{"prompt": childHang + " third", "description": "never started"},
|
||||
}})
|
||||
return []provider.Chunk{{
|
||||
Type: provider.ChunkToolCall,
|
||||
ToolCall: &provider.ToolCall{ID: "probe_parallel", Name: "parallel_tasks", Arguments: string(args)},
|
||||
}}
|
||||
}
|
||||
|
||||
// adoptOnlyFleet runs to completion with one item adopted and one that ran.
|
||||
// Nothing hangs: the arm compares two settled items whose only difference is
|
||||
// that one executed, which a fan-out still in flight could not show.
|
||||
func adoptOnlyFleet(adoptRef string) []provider.Chunk {
|
||||
return fleetCall("probe_fleet_terminal", []map[string]any{
|
||||
{"id": "a1", "adopt_ref": adoptRef, "description": "adopted"},
|
||||
{"id": "a2", "prompt": childDone + " ran", "description": "ran", "read_only": true},
|
||||
})
|
||||
}
|
||||
|
||||
// outcomesFleet reaches all three executed terminals in one group: one item
|
||||
// completes, one fails, and two are still running when the arm interrupts. The
|
||||
// store's record of each is then compared under identical conditions.
|
||||
func outcomesFleet() []provider.Chunk {
|
||||
return fleetCall("probe_fleet_outcomes", []map[string]any{
|
||||
{"id": "o1", "prompt": childDone + " completes", "description": "completes", "read_only": true},
|
||||
{"id": "o2", "prompt": childFail + " fails", "description": "fails", "read_only": true},
|
||||
{"id": "o3", "prompt": childHang + " cancelled", "description": "cancelled", "read_only": true},
|
||||
{"id": "o4", "prompt": childHang + " cancelled too", "description": "cancelled too", "read_only": true},
|
||||
})
|
||||
}
|
||||
|
||||
// identityFleet names a worker identity four ways: not at all, by model, by
|
||||
// effort, and not at all because nothing runs. What the graph then shows for
|
||||
// each is what says which layer of the resolution it is reporting.
|
||||
func identityFleet(adoptRef string) []provider.Chunk {
|
||||
return fleetCall("probe_fleet_identity", []map[string]any{
|
||||
{"id": "i1", "prompt": childDone + " default", "description": "inherits", "read_only": true},
|
||||
{"id": "i2", "prompt": childDone + " override", "description": "names a model", "read_only": true,
|
||||
"model": probeAltModelRef, "effort": "high"},
|
||||
{"id": "i3", "prompt": childDone + " effort only", "description": "names an effort", "read_only": true,
|
||||
"effort": "low"},
|
||||
{"id": "i4", "adopt_ref": adoptRef, "description": "runs nothing"},
|
||||
})
|
||||
}
|
||||
|
||||
// identityParallel asks the same of the other producer, which resolves through
|
||||
// a shorter chain: it has no profile layer to consult.
|
||||
func identityParallel() []provider.Chunk {
|
||||
args, _ := json.Marshal(map[string]any{"tasks": []map[string]any{
|
||||
{"prompt": childDone + " p default", "description": "inherits"},
|
||||
{"prompt": childDone + " p override", "description": "names a model",
|
||||
"model": probeAltModelRef, "effort": "high"},
|
||||
}})
|
||||
return []provider.Chunk{{
|
||||
Type: provider.ChunkToolCall,
|
||||
ToolCall: &provider.ToolCall{ID: "probe_parallel_identity", Name: "parallel_tasks", Arguments: string(args)},
|
||||
}}
|
||||
}
|
||||
|
||||
// deriveFleet orders one dependent behind two upstreams. The skip arms end both
|
||||
// upstreams without an answer, one immediately and one on release, so which of
|
||||
// them the fan-out names is decided by the order rather than by a race; the
|
||||
// answered arm gives the dependent one completed and one adopted upstream, so
|
||||
// it must run and both deliveries must be accounted for.
|
||||
func deriveFleet(arm, adoptRef string) []provider.Chunk {
|
||||
first, second := childFail+" first", childFailLate+" second"
|
||||
if arm == armDeriveSkipFlip {
|
||||
first, second = childFailLate+" first", childFail+" second"
|
||||
}
|
||||
up := []map[string]any{
|
||||
{"id": "a", "prompt": first, "description": "upstream a", "read_only": true},
|
||||
{"id": "b", "prompt": second, "description": "upstream b", "read_only": true},
|
||||
}
|
||||
if arm == armDeriveAnswered {
|
||||
up = []map[string]any{
|
||||
{"id": "a", "prompt": childDone + " first", "description": "upstream a", "read_only": true},
|
||||
{"id": "b", "adopt_ref": adoptRef, "description": "upstream b"},
|
||||
}
|
||||
}
|
||||
return fleetCall("probe_fleet_derive", append(up, map[string]any{
|
||||
"id": "c", "prompt": childDone + " dependent", "description": "dependent",
|
||||
"read_only": true, "depends_on": []string{"a", "b"},
|
||||
}))
|
||||
}
|
||||
|
||||
// activeStoreFleet reaches three fan-out populations under one ceiling. The
|
||||
// first item settles and frees the slot the other two then contend for, so one
|
||||
// executes and one is refused — which of them is the scheduler's to decide, and
|
||||
// the arm reads the populations off the journal rather than naming them here.
|
||||
func activeStoreFleet() []provider.Chunk {
|
||||
return fleetCall(activeFleetCallID, []map[string]any{
|
||||
{"id": "a1", "prompt": childDone + " settles", "description": "settles", "read_only": true},
|
||||
{"id": "a2", "prompt": childHang + " holds", "description": "contends for the freed slot",
|
||||
"read_only": true, "depends_on": []string{"a1"}},
|
||||
{"id": "a3", "prompt": childHang + " contends", "description": "contends for the freed slot",
|
||||
"read_only": true, "depends_on": []string{"a1"}},
|
||||
})
|
||||
}
|
||||
|
||||
// slotsFleet fills the total ceiling and asks for one more, all read-only: a
|
||||
// reader clears every writer check by not being a writer.
|
||||
func slotsFleet() []provider.Chunk {
|
||||
return fleetCall("probe_fleet_refused", []map[string]any{
|
||||
{"id": "s1", "prompt": childHang + " one", "description": "fills a slot", "read_only": true},
|
||||
{"id": "s2", "prompt": childHang + " two", "description": "fills a slot", "read_only": true},
|
||||
{"id": "s3", "prompt": childHang + " three", "description": "refused admission", "read_only": true},
|
||||
})
|
||||
}
|
||||
|
||||
// backgroundFleetCall dispatches a fleet that returns a job id at once. The
|
||||
// holder has to keep holding while a later fleet is dispatched, which a
|
||||
// foreground call cannot do: it does not return until every item has settled.
|
||||
func backgroundFleetCall(id string, tasks []map[string]any) []provider.Chunk {
|
||||
args, _ := json.Marshal(map[string]any{"tasks": tasks, "run_in_background": true})
|
||||
return []provider.Chunk{{
|
||||
Type: provider.ChunkToolCall,
|
||||
ToolCall: &provider.ToolCall{ID: id, Name: "fleet", Arguments: string(args)},
|
||||
}}
|
||||
}
|
||||
|
||||
// adoptableRef picks a reference the warm-up fleet reported. It reads the
|
||||
// conversation the model reads: the aggregate names each child's reference, and
|
||||
// an arm that cannot find one has not established its premise.
|
||||
func adoptableRef(req provider.Request) string {
|
||||
for _, m := range slices.Backward(req.Messages) {
|
||||
if found := subagentRefPattern.FindAllString(m.Content, -1); len(found) > 0 {
|
||||
return found[len(found)-1]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// fanOutTurn is the prompt that opens one fleet.
|
||||
func fanOutTurn(n int, sentinel string) string {
|
||||
return fmt.Sprintf("%s %s: dispatch the fan-out.", marker(n), sentinel)
|
||||
}
|
||||
|
||||
// fanOutSentinels is what this arm's one prompt carries. The mixed arm names
|
||||
// both of its fleets in the same turn: the warm-up has to finish before a
|
||||
// reference can be adopted, and a fleet's aggregate returns into the same turn
|
||||
// that dispatched it, so the second round dispatches the mixed fleet without
|
||||
// the turn ever closing.
|
||||
func fanOutSentinels(arm string) string {
|
||||
if arm == armUIGraphMixed {
|
||||
return fleetWarmSentinel + " " + fleetSettledSentinel + " " + fleetMixedSentinel
|
||||
}
|
||||
if arm == armGraphMixed {
|
||||
return fleetWarmSentinel + " " + fleetMixedSentinel
|
||||
}
|
||||
return fleetPairSentinel
|
||||
}
|
||||
|
||||
// runFanOutConstruct drives the arm's fan-out and exits while the turn is still
|
||||
// open. Every arm dies mid-turn, including the completed one: what separates
|
||||
// them is which states the graph holds at that moment, and a clean shutdown
|
||||
// would cancel a running child and settle its node — the one thing a process
|
||||
// that dies mid-fan-out does not do.
|
||||
func runFanOutConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, turn int) error {
|
||||
go func() { _ = ctrl.Run(ctx, fanOutTurn(turn+1, fanOutSentinels(arm))) }()
|
||||
if err := waitForFanOut(sink, arm); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeObservation(root, capture("construct", arm, bootSystem, ctrl, sink, root)); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Exit(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForFanOut blocks until the graph shows the shape this arm was built to
|
||||
// die with. Waiting on the shape rather than on a duration is what keeps the
|
||||
// arm honest: a timer that fired early would report a smaller fan-out as if
|
||||
// that were what the restart inherited.
|
||||
func waitForFanOut(sink *graphSink, arm string) error {
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
graph, _ := sink.snapshot()
|
||||
if fanOutReady(graph, arm) {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
graph, _ := sink.snapshot()
|
||||
return errUnexpected("a fan-out holding "+wantedText(arm), nodeStates(graph))
|
||||
}
|
||||
|
||||
// fanOutReady is the arm's premise expressed as a graph shape. The completed
|
||||
// arm waits for every worker to settle, which is the opposite condition from
|
||||
// the other two rather than a weaker version of it.
|
||||
func fanOutReady(g agentgraph.Graph, arm string) bool {
|
||||
if arm == armGraphCompleted {
|
||||
return allWorkersSettled(g)
|
||||
}
|
||||
return holdsAll(g, wantedStates(arm))
|
||||
}
|
||||
|
||||
// wantedStates are the states a mid-flight arm must hold at once. Skipped is
|
||||
// not among them and cannot be: a fan-out publishes a skip only in its closing
|
||||
// outcome delta, so nothing reads skipped until the group has ended and nothing
|
||||
// is running. An item whose dependency already failed sits at pending until
|
||||
// then, which is what the mixed arm dies holding.
|
||||
func wantedStates(arm string) []agentgraph.NodeState {
|
||||
want := []agentgraph.NodeState{agentgraph.StateCompleted, agentgraph.StateRunning}
|
||||
if arm == armGraphMixed || arm == armUIGraphMixed {
|
||||
want = append(want, agentgraph.StateFailed, agentgraph.StateAdopted)
|
||||
}
|
||||
// The UI arm dies holding a settled fan-out beside a live one, so it also
|
||||
// reaches the two states the mixed arm cannot: a skip, which is published
|
||||
// only when a group closes, and an item a ceiling never admitted.
|
||||
if arm == armUIGraphMixed {
|
||||
want = append(want, agentgraph.StateSkipped, agentgraph.StatePending, agentgraph.StateQueued)
|
||||
}
|
||||
return want
|
||||
}
|
||||
|
||||
func wantedText(arm string) string {
|
||||
if arm == armGraphCompleted {
|
||||
return "every worker settled"
|
||||
}
|
||||
return statesText(wantedStates(arm))
|
||||
}
|
||||
|
||||
// allWorkersSettled reports whether every worker reached a terminal state. An
|
||||
// empty state is not terminal here even though NodeState.Terminal calls it one:
|
||||
// a node declared before it ran carries no state at all, and reading that as
|
||||
// settled would let the arm die before the fan-out started.
|
||||
func allWorkersSettled(g agentgraph.Graph) bool {
|
||||
seen := false
|
||||
for _, n := range g.Nodes {
|
||||
if n.Kind != agentgraph.KindWorker {
|
||||
continue
|
||||
}
|
||||
seen = true
|
||||
if n.State == "" || !n.State.Terminal() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return seen
|
||||
}
|
||||
|
||||
// holdsAll reports whether the graph shows at least one worker in each state.
|
||||
// Group nodes are excluded: a group is the call that owns the fan-out, and its
|
||||
// own running state would satisfy the running requirement without any child
|
||||
// being in flight.
|
||||
func holdsAll(g agentgraph.Graph, want []agentgraph.NodeState) bool {
|
||||
for _, state := range want {
|
||||
found := false
|
||||
for _, n := range g.Nodes {
|
||||
if n.Kind == agentgraph.KindWorker && n.State == state {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func statesText(states []agentgraph.NodeState) string {
|
||||
out := make([]string, 0, len(states))
|
||||
for _, s := range states {
|
||||
out = append(out, string(s))
|
||||
}
|
||||
return join(out)
|
||||
}
|
||||
|
||||
func nodeStates(g agentgraph.Graph) string {
|
||||
out := make([]string, 0, len(g.Nodes))
|
||||
for _, n := range g.Nodes {
|
||||
out = append(out, n.ID+":"+string(n.State))
|
||||
}
|
||||
return orNone(join(out))
|
||||
}
|
||||
@@ -0,0 +1,646 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/agentgraph"
|
||||
"reasonix/internal/runtime/delegation"
|
||||
"reasonix/internal/state/execgraph"
|
||||
"reasonix/internal/state/execjournal"
|
||||
)
|
||||
|
||||
// How a run graph can stand after the process that built it exits. Three are
|
||||
// designs a host may hold; two are defects, and not the same defect. Losing the
|
||||
// graph costs a reader an explanation; a node still drawn as running with no
|
||||
// owner costs them the truth — reviving actionable state is more dangerous than
|
||||
// dropping it, and a running node is actionable state.
|
||||
const (
|
||||
graphExact = "reconstructed-exact" // nodes, edges, grants, waits, states all back
|
||||
graphInterrupted = "interrupted-explicit" // the fan-out is legible and the live child is marked cut
|
||||
graphLossy = "reconstructed-lossy" // it is back, minus semantics nothing can re-derive
|
||||
graphLostSilent = "LOST-SILENT" // gone, with no provenance that it ever existed
|
||||
graphGhost = "GHOST-RUNNING" // still says running, with no live owner anywhere
|
||||
graphNotMeasured = "not-measured" // the construct phase never established the shape
|
||||
)
|
||||
|
||||
// graphRows classify what a restart inherited from a fan-out. There are two
|
||||
// outcome rows rather than one because a fan-out that died mid-flight holds two
|
||||
// populations with different answers: the items that had already settled, whose
|
||||
// answers the store owns, and the one still executing, which nothing on disk
|
||||
// has ever been told about. One label over both reports the better half.
|
||||
func graphRows(before, after Observation) []row {
|
||||
return []row{
|
||||
ghostRow(before, after),
|
||||
dispatchTurnRow(before, after),
|
||||
settledOutcomeRow(before, after),
|
||||
interruptedOutcomeRow(before, after),
|
||||
graphStructureRow(before, after),
|
||||
graphSemanticsRow(before, after),
|
||||
childProvenanceRow(before, after),
|
||||
executionJournalRow(before, after),
|
||||
rebuildStateRow(before, after),
|
||||
rebuildTopologyRow(before, after),
|
||||
rebuildEnvelopeRow(before, after),
|
||||
rebuildInterruptedRow(before, after),
|
||||
rebuildIdentityRow(before, after),
|
||||
executionTopologyRow(before, after),
|
||||
executionKindRow(before, after),
|
||||
executionContextRow(before, after),
|
||||
graphObligationRow(before, after),
|
||||
}
|
||||
}
|
||||
|
||||
// runningWorkers are the nodes that were executing when the process died. They
|
||||
// are the arm's premise: an arm that establishes none has measured a fan-out
|
||||
// nobody interrupted, whatever else it recorded.
|
||||
func runningWorkers(o Observation) []string {
|
||||
var out []string
|
||||
for _, n := range o.Graph.Nodes {
|
||||
if n.Kind == agentgraph.KindWorker && n.State == agentgraph.StateRunning {
|
||||
out = append(out, n.ID)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// executedWorkers are the items that ran to a terminal state. Adopted nodes are
|
||||
// excluded on purpose: nothing executed under them, so the store owes them no
|
||||
// record and counting them would read a correct store as incomplete.
|
||||
func executedWorkers(o Observation) []string {
|
||||
var out []string
|
||||
for _, n := range o.Graph.Nodes {
|
||||
if n.Kind != agentgraph.KindWorker || n.State == "" || n.State == agentgraph.StateAdopted {
|
||||
continue
|
||||
}
|
||||
if n.State.Terminal() {
|
||||
out = append(out, n.ID)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ghostRow asks the one question with a single acceptable answer: does anything
|
||||
// the new process can read still present this work as running? A graph node and
|
||||
// a store record are both such a claim, and neither has an owner behind it —
|
||||
// the goroutine that would have settled them died with the first process.
|
||||
func ghostRow(before, after Observation) row {
|
||||
ghosts := append(runningWorkers(after), childrenInState(after, string(sessionstore.SubagentRunning))...)
|
||||
verdict := verdictHolds
|
||||
if len(ghosts) > 0 {
|
||||
verdict = verdictViolated
|
||||
}
|
||||
return row{
|
||||
Semantic: "work still presented as running", Authority: "the graph and the sub-agent store",
|
||||
Artifact: "subagents/<ref>.meta.json",
|
||||
Reconstruction: "nothing may read as running once its owner is gone",
|
||||
Before: orNone(join(runningWorkers(before))) + " running at death",
|
||||
After: orNone(join(ghosts)), Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
// dispatchTurnRow is the row underneath every other loss here: whether the turn
|
||||
// that opened the fan-out reached the transcript at all. A turn is appended
|
||||
// when it ends, so a process that dies inside one takes the request with it —
|
||||
// and a graph reconstruction has nothing to attach to even in principle.
|
||||
func dispatchTurnRow(before, after Observation) row {
|
||||
verdict := verdictPersisted
|
||||
switch {
|
||||
case !before.FanOutTurn:
|
||||
verdict = verdictNotMeasured
|
||||
case !after.FanOutTurn:
|
||||
verdict = verdictLost
|
||||
}
|
||||
return row{
|
||||
Semantic: "the turn that dispatched the fan-out", Authority: "session transcript + event log",
|
||||
Artifact: "<stem>.jsonl, <stem>.events.jsonl", Reconstruction: "appended when the turn ends",
|
||||
Before: recordedText(before), After: recordedText(after), Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func recordedText(o Observation) string {
|
||||
if o.FanOutTurn {
|
||||
return fmt.Sprintf("in the transcript (%d rows)", o.Transcript.Messages)
|
||||
}
|
||||
return fmt.Sprintf("absent (%d rows)", o.Transcript.Messages)
|
||||
}
|
||||
|
||||
// settledOutcomeRow classifies the items that had already finished. Their
|
||||
// answers are what a later turn reads back, so this is the row that decides
|
||||
// whether a fan-out's history is durable at all.
|
||||
func settledOutcomeRow(before, after Observation) row {
|
||||
outcome, verdict := settledOutcome(before, after)
|
||||
return row{
|
||||
Semantic: "items that had settled before the death", Authority: "the graph fold and the sub-agent store",
|
||||
Artifact: "subagents/<ref>.meta.json",
|
||||
Reconstruction: "one durable child record per item that executed",
|
||||
Before: fmt.Sprintf("%d executed: %s", len(executedWorkers(before)), graphSummary(before)),
|
||||
After: outcome, Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func settledOutcome(before, after Observation) (string, string) {
|
||||
executed := len(executedWorkers(before))
|
||||
switch {
|
||||
case len(before.Graph.Nodes) == 0:
|
||||
return graphNotMeasured, verdictNotMeasured
|
||||
case graphIdentical(before, after):
|
||||
return graphExact, verdictExact
|
||||
case executed == 0:
|
||||
return graphNotMeasured, verdictNotMeasured
|
||||
case len(after.Children.Facts) == 0:
|
||||
return graphLostSilent, verdictViolated
|
||||
case len(after.Children.Facts) < executed:
|
||||
return fmt.Sprintf("%s (%d of %d children)", graphLossy, len(after.Children.Facts), executed), verdictLossy
|
||||
default:
|
||||
return graphLossy, verdictLossy
|
||||
}
|
||||
}
|
||||
|
||||
// interruptedOutcomeRow classifies the item that was still executing. This is
|
||||
// the row the arm exists for: the graph is gone either way, and what separates
|
||||
// a defensible design from a durable gap is whether anything the new process
|
||||
// can read still says that work was opened and cut.
|
||||
func interruptedOutcomeRow(before, after Observation) row {
|
||||
outcome, verdict := interruptedOutcome(before, after)
|
||||
return row{
|
||||
Semantic: "the item still executing at the death", Authority: "the sub-agent store and the transcript",
|
||||
Artifact: "subagents/<ref>.meta.json, <stem>.jsonl",
|
||||
Reconstruction: "an interrupted child record, or an unsettled call the transcript still owes",
|
||||
Before: orNone(join(runningWorkers(before))), After: outcome, Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func interruptedOutcome(before, after Observation) (string, string) {
|
||||
switch {
|
||||
case len(before.Graph.Nodes) == 0 || len(runningWorkers(before)) == 0:
|
||||
return graphNotMeasured, verdictNotMeasured
|
||||
case len(runningWorkers(after)) > 0 || len(childrenInState(after, string(sessionstore.SubagentRunning))) > 0:
|
||||
return graphGhost, verdictViolated
|
||||
case len(after.InterruptedExecutions) > 0:
|
||||
return graphInterrupted, verdictLossy
|
||||
case len(childrenInState(after, string(sessionstore.SubagentInterrupted))) > 0:
|
||||
return graphInterrupted, verdictLossy
|
||||
case len(after.Obligation.UnansweredCalls) > 0 || after.Obligation.InterruptionMarked > 0:
|
||||
return graphInterrupted, verdictLossy
|
||||
default:
|
||||
return graphLostSilent, verdictViolated
|
||||
}
|
||||
}
|
||||
|
||||
// graphIdentical compares the whole graph, not its node count. Two graphs with
|
||||
// the same nodes and different grants are not the same graph, and a comparison
|
||||
// that only counted would have called that exact.
|
||||
func graphIdentical(before, after Observation) bool {
|
||||
return graphNodes(before) == graphNodes(after) &&
|
||||
graphEdges(before) == graphEdges(after) &&
|
||||
graphMap(before.Graph.Grants) == graphMap(after.Graph.Grants) &&
|
||||
graphMap(before.Graph.Waits) == graphMap(after.Graph.Waits)
|
||||
}
|
||||
|
||||
// graphStructureRow is the shape a reader asks for: which nodes, and the typed
|
||||
// edges between them. Spawn, depends, context and adopt are separate facts, and
|
||||
// a restart that kept the nodes while losing the edges kept a list, not a graph.
|
||||
func graphStructureRow(before, after Observation) row {
|
||||
return valueRow("run graph edges", "event sink (GraphDelta)",
|
||||
"none observed", "fold GraphDelta events, if any survive",
|
||||
graphEdges(before), graphEdges(after), false)
|
||||
}
|
||||
|
||||
// graphSemanticsRow is the authority envelope and the reason a node waited.
|
||||
// They are called out apart from the structure because nothing else in the
|
||||
// system records them: a child's persisted metadata carries its model and its
|
||||
// profile, and says nothing about what it was allowed to touch.
|
||||
func graphSemanticsRow(before, after Observation) row {
|
||||
return valueRow("run graph grants and waits", "event sink (GraphDelta)",
|
||||
"none observed", "no durable source carries an authority envelope",
|
||||
graphEnvelope(before), graphEnvelope(after), false)
|
||||
}
|
||||
|
||||
func graphEnvelope(o Observation) string {
|
||||
grants, waits := graphMap(o.Graph.Grants), graphMap(o.Graph.Waits)
|
||||
if grants == "" && waits == "" {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("grants[%s] waits[%s]", orNone(grants), orNone(waits))
|
||||
}
|
||||
|
||||
// childProvenanceRow is what the store still owns, which is the evidence behind
|
||||
// an interrupted classification. It is reported as its own row so a reader can
|
||||
// see the reconstruction the label stands on rather than trusting the label.
|
||||
func childProvenanceRow(before, after Observation) row {
|
||||
return valueRow("delegated runs the store owns", "SubagentStore, swept at boot",
|
||||
"subagents/<ref>.meta.json", "ListSubagentsByParent; CleanupStaleRunning marks stale running children interrupted",
|
||||
childSummary(before), childSummary(after), true)
|
||||
}
|
||||
|
||||
// graphObligationRow is the last fallback a reader has: a fleet call the
|
||||
// transcript records with no result is the host saying a fan-out was opened and
|
||||
// never settled, even when nothing else remembers it.
|
||||
func graphObligationRow(before, after Observation) row {
|
||||
return row{
|
||||
Semantic: "fan-out calls the transcript never settled", Authority: "the canonical transcript",
|
||||
Artifact: "<stem>.jsonl", Reconstruction: "tool calls with no result",
|
||||
Before: orNone(join(before.Obligation.UnansweredCalls)),
|
||||
After: orNone(join(after.Obligation.UnansweredCalls)), Verdict: verdictStable,
|
||||
}
|
||||
}
|
||||
|
||||
// rebuiltGraph is what the production fold makes of this session's durable
|
||||
// facts, join included: a record names its own execution, and an older one is
|
||||
// aliased to its parent call only where the journal opened one by that id.
|
||||
func rebuiltGraph(o Observation) execgraph.Result {
|
||||
opened := make(map[string]bool, len(o.Executions))
|
||||
for _, e := range o.Executions {
|
||||
opened[e.ID] = true
|
||||
}
|
||||
children := make([]execgraph.ChildOutcome, 0, len(o.Children.Facts))
|
||||
for _, f := range o.Children.Facts {
|
||||
identity := delegation.ResolveExecutionIdentity(
|
||||
sessionstore.SubagentMeta{ExecutionID: f.ExecutionID, ParentToolCallID: f.ParentToolCallID},
|
||||
func(id string) bool { return opened[id] })
|
||||
children = append(children, execgraph.ChildOutcome{
|
||||
Execution: identity.Execution, Ref: f.Ref, Status: f.Status,
|
||||
})
|
||||
}
|
||||
return execgraph.Rebuild(o.Executions, children, nil)
|
||||
}
|
||||
|
||||
// rebuildStateRow compares the states the fold produces against the ones the
|
||||
// dying process drew. Nodes whose owner disappeared are excluded and reported
|
||||
// by their own row: putting those back as running is the one outcome this
|
||||
// whole line of work exists to prevent.
|
||||
func rebuildStateRow(before, after Observation) row {
|
||||
rebuilt := rebuiltGraph(after)
|
||||
cut := map[string]bool{}
|
||||
for _, i := range rebuilt.Interrupted {
|
||||
cut[i.Execution] = true
|
||||
}
|
||||
return valueRow("rebuilt node states", "execgraph.Rebuild over journal + store",
|
||||
"<stem>.execution.jsonl + subagents/<ref>.meta.json",
|
||||
"recomputed from durable facts, never restored",
|
||||
statesExcept(graphOf(before), cut), statesExcept(rebuilt.Graph, cut), false)
|
||||
}
|
||||
|
||||
// rebuildTopologyRow compares every typed edge. Spawn and depends are stated by
|
||||
// the openings, adopt names its source, and context is derived — so a mismatch
|
||||
// here says which of those four is not yet recoverable.
|
||||
func rebuildTopologyRow(before, after Observation) row {
|
||||
return valueRow("rebuilt topology", "execgraph.Rebuild over journal + store",
|
||||
"<stem>.execution.jsonl", "spawn and depends stated, adopt named, context derived",
|
||||
edgeSummary(graphOf(before)), edgeSummary(rebuiltGraph(after).Graph), false)
|
||||
}
|
||||
|
||||
// rebuildEnvelopeRow is the authority and the wait provenance together: what
|
||||
// each item was allowed to touch, and why admission was first refused.
|
||||
func rebuildEnvelopeRow(before, after Observation) row {
|
||||
return valueRow("rebuilt grants and wait causes", "execgraph.Rebuild over journal + store",
|
||||
"<stem>.execution.jsonl", "recorded at the opening and at the first refusal",
|
||||
envelopeSummary(graphOf(before)), envelopeSummary(rebuiltGraph(after).Graph), false)
|
||||
}
|
||||
|
||||
// rebuildInterruptedRow accounts for the nodes the state row leaves out. They
|
||||
// are not dropped: the fold names them as interruptions, split by whether they
|
||||
// had reached a slot. A node the dying process showed live and the rebuild
|
||||
// mentions nowhere would be a loss the state row could not see.
|
||||
func rebuildInterruptedRow(before, after Observation) row {
|
||||
rebuilt := rebuiltGraph(after)
|
||||
var got []string
|
||||
for _, i := range rebuilt.Interrupted {
|
||||
kind := execjournal.InterruptedBeforeStart
|
||||
if i.Started {
|
||||
kind = execjournal.InterruptedDuringExecution
|
||||
}
|
||||
got = append(got, i.Execution+":"+kind)
|
||||
}
|
||||
sort.Strings(got)
|
||||
return valueRow("nodes the rebuild will not call live", "execgraph.Rebuild over journal + store",
|
||||
"<stem>.execution.jsonl", "an execution with no owner is named, never redrawn as running",
|
||||
liveAtDeath(before), join(got), false)
|
||||
}
|
||||
|
||||
// liveAtDeath is what the dying process still showed as someone's work.
|
||||
func liveAtDeath(o Observation) string {
|
||||
var out []string
|
||||
for _, n := range o.Graph.Nodes {
|
||||
if n.Kind == agentgraph.KindGroup {
|
||||
continue
|
||||
}
|
||||
switch n.State {
|
||||
case agentgraph.StateRunning:
|
||||
out = append(out, n.ID+":"+execjournal.InterruptedDuringExecution)
|
||||
case agentgraph.StateQueued, agentgraph.StatePending:
|
||||
out = append(out, n.ID+":"+execjournal.InterruptedBeforeStart)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
// rebuildIdentityRow is what a reader sees on a node beyond its state: the
|
||||
// label it was dispatched under, and which worker ran it. Only one of those is
|
||||
// recorded, so this row is where a remaining gap shows up rather than hiding
|
||||
// inside an exact verdict about states.
|
||||
func rebuildIdentityRow(before, after Observation) row {
|
||||
return valueRow("rebuilt node identity", "execgraph.Rebuild over journal + store",
|
||||
"<stem>.execution.jsonl", "label from the opening; the worker override is recorded nowhere",
|
||||
identitySummary(graphOf(before)), identitySummary(rebuiltGraph(after).Graph), false)
|
||||
}
|
||||
|
||||
func identitySummary(g agentgraph.Graph) string {
|
||||
var out []string
|
||||
for _, n := range g.Nodes {
|
||||
if n.Kind != agentgraph.KindWorker {
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s[label=%s profile=%s model=%s effort=%s]",
|
||||
n.ID, orNone(n.Label), orNone(n.Profile), orNone(n.Model), orNone(n.Effort)))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
// statesExcept renders worker states, skipping the executions whose owner is
|
||||
// gone. Those are not a state the rebuild may assert, and comparing them would
|
||||
// demand exactly the ghost this refuses to draw.
|
||||
func statesExcept(g agentgraph.Graph, skip map[string]bool) string {
|
||||
var out []string
|
||||
for _, n := range g.Nodes {
|
||||
if n.Kind == agentgraph.KindGroup || skip[n.ID] {
|
||||
continue
|
||||
}
|
||||
out = append(out, n.ID+":"+string(n.State))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
func edgeSummary(g agentgraph.Graph) string {
|
||||
out := make([]string, 0, len(g.Edges))
|
||||
for _, e := range g.Edges {
|
||||
out = append(out, fmt.Sprintf("%s-%s->%s", e.From, e.Kind, e.To))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
func envelopeSummary(g agentgraph.Graph) string {
|
||||
var out []string
|
||||
for _, n := range g.Nodes {
|
||||
if n.Grant != "" {
|
||||
out = append(out, n.ID+"="+string(n.Grant))
|
||||
}
|
||||
if n.Wait != "" {
|
||||
out = append(out, n.ID+"@"+string(n.Wait))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
// executionJournalRow is the record written before the work started, which is
|
||||
// the only artifact here that can outlive an unfinished turn. It is what a
|
||||
// restart reads to say a delegation existed at all.
|
||||
func executionJournalRow(before, after Observation) row {
|
||||
return valueRow("delegations the journal recorded", "execution journal, written before dispatch",
|
||||
"<stem>.execution.jsonl", "folded from openings and settlings",
|
||||
executionSummary(before), executionSummary(after), true)
|
||||
}
|
||||
|
||||
// executionTopologyRow is the ordering a restart inherits. An item that never
|
||||
// started is explained by what it was held behind, so the topology has to
|
||||
// survive the process that knew it — and survive unchanged, or the explanation
|
||||
// is a different graph's.
|
||||
func executionTopologyRow(before, after Observation) row {
|
||||
return valueRow("declared ordering between items", "the fan-out plan, recorded at the opening",
|
||||
"<stem>.execution.jsonl", "read back with the openings, never re-derived",
|
||||
topologySummary(before), topologySummary(after), true)
|
||||
}
|
||||
|
||||
func topologySummary(o Observation) string {
|
||||
var out []string
|
||||
for _, e := range o.Executions {
|
||||
if len(e.DependsOn) == 0 {
|
||||
continue
|
||||
}
|
||||
up := append([]string(nil), e.DependsOn...)
|
||||
sort.Strings(up)
|
||||
out = append(out, e.ID+"<-"+strings.Join(up, "+"))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return ""
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
// executionKindRow holds the two interruptions apart against what the dying
|
||||
// process actually showed. The item the graph had running must come back as one
|
||||
// that reached a slot; the one its dependency blocked must come back as one
|
||||
// that did not. A classification agreeing with neither is a label, not a fact.
|
||||
func executionKindRow(before, after Observation) row {
|
||||
running := runningWorkers(before)
|
||||
verdict := verdictHolds
|
||||
if len(after.InterruptedExecutions) == 0 {
|
||||
verdict = verdictNotMeasured
|
||||
}
|
||||
for _, e := range after.InterruptedExecutions {
|
||||
want := execjournal.InterruptedBeforeStart
|
||||
if slices.Contains(running, e.ID) {
|
||||
want = execjournal.InterruptedDuringExecution
|
||||
}
|
||||
if e.Interruption() != want {
|
||||
verdict = verdictViolated
|
||||
}
|
||||
}
|
||||
return row{
|
||||
Semantic: "how each interruption is classified", Authority: "the execution journal's start records",
|
||||
Artifact: "<stem>.execution.jsonl", Reconstruction: "started at death -> during-execution, never started -> before-start",
|
||||
Before: orNone(join(running)) + " running at death", After: interruptionKinds(after), Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func interruptionKinds(o Observation) string {
|
||||
if len(o.InterruptedExecutions) == 0 {
|
||||
return "none"
|
||||
}
|
||||
out := make([]string, 0, len(o.InterruptedExecutions))
|
||||
for _, e := range o.InterruptedExecutions {
|
||||
out = append(out, e.ID+":"+e.Interruption())
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
// executionContextRow is the other side of that record: what the next request
|
||||
// is told. It must state provenance and never offer a continuation, so the
|
||||
// count is checked against the interruptions rather than against a fixed
|
||||
// number — a block with nothing behind it is a ghost in prose.
|
||||
func executionContextRow(before, after Observation) row {
|
||||
verdict := verdictHolds
|
||||
if blocksOwed(before) != before.ModelSeesInterruptedExecution ||
|
||||
blocksOwed(after) != after.ModelSeesInterruptedExecution {
|
||||
verdict = verdictViolated
|
||||
}
|
||||
return row{
|
||||
Semantic: "what the next request says about them", Authority: "control, projected per request",
|
||||
Artifact: "none: derived, never stored", Reconstruction: "one block while an interrupted execution stands, none otherwise",
|
||||
Before: executionBlocks(before), After: executionBlocks(after), Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func blocksOwed(o Observation) int {
|
||||
if len(o.InterruptedExecutions) > 0 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func executionBlocks(o Observation) string {
|
||||
return fmt.Sprintf("%d block(s) for %d interrupted", o.ModelSeesInterruptedExecution, len(o.InterruptedExecutions))
|
||||
}
|
||||
|
||||
func executionSummary(o Observation) string {
|
||||
if len(o.Executions) == 0 {
|
||||
return ""
|
||||
}
|
||||
open, started := 0, 0
|
||||
for _, e := range o.Executions {
|
||||
if e.Open() {
|
||||
open++
|
||||
}
|
||||
if e.Started() {
|
||||
started++
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%d opened, %d started, %d still open, %d with no owner here",
|
||||
len(o.Executions), started, open, len(o.InterruptedExecutions))
|
||||
}
|
||||
|
||||
func childrenInState(o Observation, status string) []string {
|
||||
var out []string
|
||||
for _, f := range o.Children.Facts {
|
||||
if f.Status == status {
|
||||
out = append(out, f.Ref)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func childSummary(o Observation) string {
|
||||
if len(o.Children.Facts) == 0 {
|
||||
if o.Children.Err != "" {
|
||||
return "error: " + o.Children.Err
|
||||
}
|
||||
return ""
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for _, f := range o.Children.Facts {
|
||||
counts[f.Status]++
|
||||
}
|
||||
keys := make([]string, 0, len(counts))
|
||||
for k := range counts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
|
||||
}
|
||||
return fmt.Sprintf("%d child(ren): %s", len(o.Children.Facts), strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
// graphSummary is the fan-out as the dying process last saw it, which is the
|
||||
// "before" every graph row is compared against.
|
||||
func graphSummary(o Observation) string {
|
||||
if len(o.Graph.Nodes) == 0 {
|
||||
return "no fan-out"
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for _, n := range o.Graph.Nodes {
|
||||
if n.Kind == agentgraph.KindWorker {
|
||||
counts[string(n.State)]++
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(counts))
|
||||
for k := range counts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
|
||||
}
|
||||
return fmt.Sprintf("%d node(s), %d edge(s): %s",
|
||||
len(o.Graph.Nodes), len(o.Graph.Edges), strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
// graphArmInvalid refuses an arm that never reached the state it names. The
|
||||
// running arms need a child in flight at death; every fan-out arm needs a
|
||||
// fan-out at all. Reporting a smaller shape as the thing that was interrupted
|
||||
// is how a probe answers a question nobody asked.
|
||||
func graphArmInvalid(arm string, before Observation) string {
|
||||
if len(before.Graph.Nodes) == 0 {
|
||||
return "no fan-out was published before the process exited, so nothing was measured"
|
||||
}
|
||||
if arm == armGraphCompleted {
|
||||
if len(runningWorkers(before)) > 0 {
|
||||
return "the fan-out was still running at exit, so this is not the completed arm"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
if len(runningWorkers(before)) == 0 {
|
||||
return "no child was executing when the process died, so nothing was interrupted"
|
||||
}
|
||||
if arm == armGraphMixed {
|
||||
return mixedArmInvalid(before)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// mixedArmInvalid holds the density arm to its own premise. Its whole claim is
|
||||
// that one death carried every reachable state at once, and an arm that reached
|
||||
// three of them measures a different scenario than the one it reports.
|
||||
func mixedArmInvalid(before Observation) string {
|
||||
var missing []string
|
||||
for _, state := range wantedStates(armGraphMixed) {
|
||||
if !holdsAll(agentgraph.Graph{Nodes: before.Graph.Nodes}, []agentgraph.NodeState{state}) {
|
||||
missing = append(missing, string(state))
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return "the fan-out never reached " + join(missing) + ", so the arm is not the mixed shape it reports"
|
||||
}
|
||||
if !carriesGrant(before, agentgraph.GrantRead) || !carriesGrant(before, agentgraph.GrantWrite) {
|
||||
return "the fan-out carried only one kind of grant, so the envelope row measures one side of it"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func carriesGrant(o Observation, want agentgraph.Grant) bool {
|
||||
for _, got := range o.Graph.Grants {
|
||||
if got == string(want) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// graphEdges renders the typed relations in a stable order. Sorting is not
|
||||
// cosmetic: the fold keeps first-seen order, which two processes reach by
|
||||
// different routes, and an unsorted comparison would report a reordering as a
|
||||
// changed graph.
|
||||
func graphEdges(o Observation) string {
|
||||
if len(o.Graph.Edges) == 0 {
|
||||
return ""
|
||||
}
|
||||
out := make([]string, 0, len(o.Graph.Edges))
|
||||
for _, e := range o.Graph.Edges {
|
||||
out = append(out, fmt.Sprintf("%s-%s->%s", e.From, e.Kind, e.To))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
@@ -0,0 +1,602 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/agentgraph"
|
||||
"reasonix/internal/session/control"
|
||||
)
|
||||
|
||||
// The interactive host-initiated arms, named for the one surface they measure: a
|
||||
// person typing /<skill> into a live session. The headless counterpart is not
|
||||
// measured here — it is caller-owned and its cancellation need not match, and
|
||||
// reading one for the other is how a real number gets the wrong subject.
|
||||
const (
|
||||
armHostCompleted = "interactive-host-skill-completed"
|
||||
armHostQueued = "interactive-host-skill-queued-crash"
|
||||
armHostRunning = "interactive-host-skill-running-crash"
|
||||
armHostCancel = "interactive-host-skill-cancel"
|
||||
// The headless counterpart: the controller's own synchronous entry point,
|
||||
// whose caller owns both the context and the cancellation. It is measured
|
||||
// apart from the four above and compared to them, never read for them.
|
||||
armHeadlessCompleted = "headless-host-skill-completed"
|
||||
armHeadlessRunningCancel = "headless-host-skill-running-cancel"
|
||||
armHeadlessQueuedCancel = "headless-host-skill-queued-cancel"
|
||||
)
|
||||
|
||||
func headlessHostArm(name string) bool {
|
||||
switch name {
|
||||
case armHeadlessCompleted, armHeadlessRunningCancel, armHeadlessQueuedCancel:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// callerStopArm names the arms a caller stops through its own context. The
|
||||
// controller never admitted these turns, so its own cancel holds nothing for
|
||||
// them — reading one surface with the other's stop is a mistake this probe has
|
||||
// already made once.
|
||||
func callerStopArm(name string) bool {
|
||||
return name == armHeadlessRunningCancel || name == armHeadlessQueuedCancel
|
||||
}
|
||||
|
||||
func hostExecArm(name string) bool {
|
||||
switch name {
|
||||
case armHostCompleted, armHostQueued, armHostRunning, armHostCancel:
|
||||
return true
|
||||
}
|
||||
return headlessHostArm(name)
|
||||
}
|
||||
|
||||
// queuedHostArm names the arms whose ceiling is filled before the invocation.
|
||||
func queuedHostArm(name string) bool {
|
||||
return name == armHostQueued || name == armHeadlessQueuedCancel
|
||||
}
|
||||
|
||||
// The four identities one host-started run can be known by. They are printed
|
||||
// apart because this is the first surface where they may not coincide: for a
|
||||
// model-delegated run the execution and the parent call are one string, and
|
||||
// that has only ever been true because a tool call produced every durable
|
||||
// execution there was.
|
||||
const (
|
||||
probeHostDispatch = "probe:host-dispatch-id"
|
||||
probeHostChildRef = "probe:host-child-ref"
|
||||
probeHostParentID = "probe:host-parent-lineage"
|
||||
probeHostExecution = "probe:host-execution-id"
|
||||
// probeHostStanding is what the call itself was doing at the reading:
|
||||
// admitted or held back, its child in the provider or not.
|
||||
probeHostStanding = "probe:host-standing"
|
||||
// probeHostAuthority is the snapshot at the same instant: what a reader
|
||||
// starting from the authority would hold, on either surface.
|
||||
probeHostAuthority = "probe:host-authority"
|
||||
)
|
||||
|
||||
// hostSlashInput is what a person types. The task carries the child sentinel so
|
||||
// the scripted provider knows how this arm's child behaves.
|
||||
func hostSlashInput(arm string) string {
|
||||
return "/" + probeSkillName + " " + hostChildSentinel(arm) + " host-started work"
|
||||
}
|
||||
|
||||
// runHostExecConstruct drives one interactive slash invocation and dies where
|
||||
// this arm is about. The queued arm fills the ceiling first with a background
|
||||
// fleet, which is dispatched through Run rather than Submit: Run is never
|
||||
// admitted through the turn gate, so the slash the arm measures can still be.
|
||||
func runHostExecConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
|
||||
if queuedHostArm(arm) {
|
||||
go func() { _ = ctrl.Run(ctx, fanOutTurn(turn+1, fleetHolderSentinel)) }()
|
||||
if err := waitForCeiling(prov); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Each surface is entered and stopped by the owner it has: the controller's
|
||||
// gate for a turn it admitted, and the caller's own context for a
|
||||
// synchronous call it never did.
|
||||
callerCtx, stopCaller := context.WithCancel(ctx)
|
||||
defer stopCaller()
|
||||
var answered atomic.Bool
|
||||
if headlessHostArm(arm) {
|
||||
go func() {
|
||||
_, _ = ctrl.RunSubagentProfile(callerCtx, probeSkillName, hostHeadlessTask(arm), false)
|
||||
answered.Store(true)
|
||||
}()
|
||||
} else {
|
||||
go ctrl.Submit(hostSlashInput(arm))
|
||||
}
|
||||
if err := waitForHostExec(ctrl, prov, arm, &answered); err != nil {
|
||||
return err
|
||||
}
|
||||
stop := ""
|
||||
switch {
|
||||
case arm == armHostCancel:
|
||||
stop = stopHostExec(ctrl, prov)
|
||||
case callerStopArm(arm):
|
||||
stop = stopHostCaller(ctrl, prov, arm, stopCaller)
|
||||
}
|
||||
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
|
||||
recordHostIdentities(&obs, ctrl, sink, prov, arm)
|
||||
if stop != "" {
|
||||
obs.Progress[probeStopReached] = []string{stop}
|
||||
}
|
||||
if err := writeObservation(root, obs); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Exit(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// recordHostIdentities writes down all four names this run could be known by,
|
||||
// at one instant. A single "identity: none" would hide which of them exists and
|
||||
// which does not, and that difference is the whole question.
|
||||
func recordHostIdentities(obs *Observation, ctrl *control.Controller, sink *graphSink, prov *scripted, arm string) {
|
||||
obs.Progress[probeChildEntered] = []string{fmt.Sprint(prov.fleets.held(childEntered(hostChildSentinel(arm))))}
|
||||
obs.Progress[probeHostDispatch] = []string{orNone(join(hostDispatchIDs(sink)))}
|
||||
obs.Progress[probeHostChildRef] = []string{orNone(join(hostChildRefs(*obs)))}
|
||||
obs.Progress[probeHostParentID] = []string{orNone(join(hostParentLineage(*obs)))}
|
||||
obs.Progress[probeHostExecution] = []string{orNone(join(hostExecutions(*obs)))}
|
||||
obs.Progress[probeHostStanding] = []string{hostStanding(ctrl, prov, arm)}
|
||||
// The authority, read at the same instant. It is what a reader starts from
|
||||
// on either surface, so it is where the two are compared — the delta stream
|
||||
// exists only where a sink does, and a synchronous caller has none.
|
||||
obs.Progress[probeHostAuthority] = []string{hostAuthorityNodes(ctrl)}
|
||||
}
|
||||
|
||||
// hostHeadlessTask is what the controller's own entry point is asked to do.
|
||||
func hostHeadlessTask(arm string) string {
|
||||
return hostChildSentinel(arm) + " host-started work"
|
||||
}
|
||||
|
||||
func hostAuthorityNodes(ctrl *control.Controller) string {
|
||||
var out []string
|
||||
for _, n := range ctrl.ExecutionGraph().Graph.Nodes {
|
||||
if ofHostExecution(n.ID) {
|
||||
out = append(out, n.ID+":"+orDash(string(n.State)))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return orNone(join(out))
|
||||
}
|
||||
|
||||
// stopHostCaller stops work through the only owner a synchronous call has: the
|
||||
// context its caller passed in. The controller's gate never admitted this turn,
|
||||
// so asking it to stop one would be reading a surface with another's stop.
|
||||
func stopHostCaller(ctrl *control.Controller, prov *scripted, arm string, stop func()) string {
|
||||
stopped := time.Now()
|
||||
stop()
|
||||
child := hostChildSentinel(arm)
|
||||
deadline := time.Now().Add(20 * time.Second)
|
||||
for time.Now().Before(deadline) && !prov.fleets.held(childLeft(child)) {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
live := "gone"
|
||||
if prov.fleets.held(childEntered(child)) && !prov.fleets.held(childLeft(child)) {
|
||||
live = "still running"
|
||||
}
|
||||
return fmt.Sprintf("turn=%s work=%s ctx=%t after=%s", turnStanding(ctrl, arm), live,
|
||||
prov.fleets.held(childCtxDone(child)), time.Since(stopped).Round(time.Second))
|
||||
}
|
||||
|
||||
func hostChildSentinel(arm string) string {
|
||||
if arm == armHostCompleted || arm == armHeadlessCompleted {
|
||||
return childDone
|
||||
}
|
||||
return childHang
|
||||
}
|
||||
|
||||
// hostDispatchIDs are the synthetic event ids the host minted so the child's
|
||||
// activity could nest under something on screen. They are UI identity, not a
|
||||
// call the model made, which is why no persisted lineage may carry one.
|
||||
func hostDispatchIDs(sink *graphSink) []string {
|
||||
var out []string
|
||||
for id := range sink.toolIDs() {
|
||||
if strings.HasPrefix(id, "slash-skill-") {
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// hostChildRefs are the store's own artifact ids. They exist only once a child
|
||||
// transcript has been prepared, which is why they cannot serve as the identity
|
||||
// of work the scheduler is still holding back.
|
||||
func hostChildRefs(o Observation) []string {
|
||||
var out []string
|
||||
for _, f := range o.Children.Facts {
|
||||
if f.Kind == "skill" {
|
||||
out = append(out, f.Ref+"="+f.Status)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func hostParentLineage(o Observation) []string {
|
||||
var out []string
|
||||
for _, f := range o.Children.Facts {
|
||||
if f.Kind == "skill" {
|
||||
out = append(out, f.Ref+"→"+orDash(f.ParentToolCallID))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func hostExecutions(o Observation) []string {
|
||||
var out []string
|
||||
for _, e := range o.Executions {
|
||||
if ofHostExecution(e.ID) || strings.Contains(e.Name, probeSkillName) {
|
||||
out = append(out, e.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// ofHostExecution matches both names this work can go by: the execution the host
|
||||
// owns, and the synthetic event id it mints for the screen. Reading only the
|
||||
// second is how an arm reports absence when what changed is the identity.
|
||||
func ofHostExecution(id string) bool {
|
||||
return strings.HasPrefix(id, "host-") || strings.HasPrefix(id, "slash-skill-")
|
||||
}
|
||||
|
||||
func hostStanding(ctrl *control.Controller, prov *scripted, arm string) string {
|
||||
return fmt.Sprintf("turn=%v ceiling=%t child=%t", ctrl.Running(),
|
||||
prov.fleets.held(childEntered(childHold)),
|
||||
prov.fleets.held(childEntered(hostChildSentinel(arm))))
|
||||
}
|
||||
|
||||
func waitForCeiling(prov *scripted) error {
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if prov.fleets.held(childEntered(childHold)) {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return errUnexpected("a fan-out holding the ceiling", "none reached the provider")
|
||||
}
|
||||
|
||||
// waitForHostExec blocks until the invocation stands where this arm dies. The
|
||||
// queued arm asks a negative — the child never arrived while the ceiling was
|
||||
// full — so it settles and asks again rather than reading a race.
|
||||
func waitForHostExec(ctrl *control.Controller, prov *scripted, arm string, answered *atomic.Bool) error {
|
||||
child := hostChildSentinel(arm)
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
switch {
|
||||
case queuedHostArm(arm):
|
||||
// In flight, read through whichever owner this surface has: a turn
|
||||
// the controller admitted answers its gate, and a synchronous call
|
||||
// it never admitted answers only by returning.
|
||||
if hostInFlight(ctrl, arm, answered) && !prov.fleets.held(childEntered(child)) {
|
||||
time.Sleep(2 * time.Second)
|
||||
if prov.fleets.held(childEntered(child)) {
|
||||
return errUnexpected("a host invocation the scheduler was still holding back", "its child ran")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
case arm == armHostCompleted:
|
||||
if prov.fleets.held(childLeft(child)) && !ctrl.Running() {
|
||||
return nil
|
||||
}
|
||||
case arm == armHeadlessCompleted:
|
||||
if prov.fleets.held(childLeft(child)) {
|
||||
return nil
|
||||
}
|
||||
default:
|
||||
if prov.fleets.held(childEntered(child)) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return errUnexpected("a host invocation "+hostWanted(arm), hostStanding(ctrl, prov, arm))
|
||||
}
|
||||
|
||||
// hostInFlight reports that the invocation has not answered yet, asked of the
|
||||
// owner the surface actually has.
|
||||
func hostInFlight(ctrl *control.Controller, arm string, answered *atomic.Bool) bool {
|
||||
if headlessHostArm(arm) {
|
||||
return !answered.Load()
|
||||
}
|
||||
return ctrl.Running()
|
||||
}
|
||||
|
||||
func hostWanted(arm string) string {
|
||||
switch {
|
||||
case queuedHostArm(arm):
|
||||
return "held back by a full ceiling"
|
||||
case arm == armHostCompleted || arm == armHeadlessCompleted:
|
||||
return "that finished"
|
||||
}
|
||||
return "with its child inside the provider"
|
||||
}
|
||||
|
||||
// stopHostExec stops the work the way a person does: the turn's own cancel. A
|
||||
// slash invocation is admitted through the same gate a typed message is, so
|
||||
// that gate is the owner of its stop.
|
||||
func stopHostExec(ctrl *control.Controller, prov *scripted) string {
|
||||
stopped := time.Now()
|
||||
ctrl.Cancel()
|
||||
deadline := time.Now().Add(20 * time.Second)
|
||||
for time.Now().Before(deadline) && ctrl.Running() {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
live := "gone"
|
||||
if prov.fleets.held(childEntered(childHang)) && !prov.fleets.held(childLeft(childHang)) {
|
||||
live = "still running"
|
||||
}
|
||||
return fmt.Sprintf("turn=%s work=%s ctx=%t after=%s", turnStanding(ctrl, armHostCancel), live,
|
||||
prov.fleets.held(childCtxDone(childHang)), time.Since(stopped).Round(time.Second))
|
||||
}
|
||||
|
||||
// The verdicts this arm needs. They are separate words because the failure
|
||||
// modes are separate facts, and one label over all of them would say less than
|
||||
// any of them does.
|
||||
const (
|
||||
// verdictIdentityGap is work the host admitted that no identity outliving
|
||||
// the process names. It is not about a lost record: none of the candidate
|
||||
// identities can carry one.
|
||||
verdictIdentityGap = "IDENTITY-GAP"
|
||||
// verdictResultDurable is an invocation whose child record survives while
|
||||
// the execution that produced it does not. A reader can see that a child of
|
||||
// this name was cut; nothing says which invocation it belonged to.
|
||||
verdictResultDurable = "record-only"
|
||||
)
|
||||
|
||||
func hostExecRows(arm string, before, after Observation) []row {
|
||||
return []row{
|
||||
hostAdmissionRow(arm, before),
|
||||
hostIdentityRow(before),
|
||||
hostGraphRow(before),
|
||||
hostJournalRow(before, after),
|
||||
hostStoreRow(before, after),
|
||||
hostLineageOracleRow(before, after),
|
||||
hostPreStartIdentityRow(arm, before, after),
|
||||
hostCancelRow(arm, before),
|
||||
hostRestartRow(arm, before, after),
|
||||
}
|
||||
}
|
||||
|
||||
func hostAdmissionRow(arm string, before Observation) row {
|
||||
entered := firstProgress(before, probeChildEntered) == "true"
|
||||
answer, verdict := "admitted: the child reached the provider", verdictHolds
|
||||
switch {
|
||||
case queuedHostArm(arm) && !entered:
|
||||
answer = "refused: the ceiling was full and the child never arrived"
|
||||
case queuedHostArm(arm):
|
||||
answer, verdict = "not held back: the child ran with the ceiling full", verdictViolated
|
||||
case !entered:
|
||||
answer, verdict = "the child never arrived", verdictNotMeasured
|
||||
}
|
||||
return row{
|
||||
Semantic: "does it ask the session scheduler", Authority: skillAuthority,
|
||||
Artifact: "none observed", Reconstruction: "in-process only",
|
||||
Before: answer + " [" + firstProgress(before, probeHostStanding) + "]",
|
||||
After: "—", Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
// hostIdentityRow prints all four names at once. This is the row the arm exists
|
||||
// for: for a model-delegated run the execution and the parent call are the same
|
||||
// string, and that has only ever held because a tool call produced every durable
|
||||
// execution there was.
|
||||
func hostIdentityRow(before Observation) row {
|
||||
return row{
|
||||
Semantic: "the four names this run could be known by", Authority: crossAuthority,
|
||||
Artifact: "the event stream, subagents/<ref>.meta.json, <stem>.execution.jsonl",
|
||||
Reconstruction: "read apart, because they may not coincide here",
|
||||
Before: fmt.Sprintf("dispatch=%s child=%s lineage=%s execution=%s",
|
||||
firstProgress(before, probeHostDispatch), firstProgress(before, probeHostChildRef),
|
||||
firstProgress(before, probeHostParentID), firstProgress(before, probeHostExecution)),
|
||||
After: "—", Verdict: verdictHolds,
|
||||
}
|
||||
}
|
||||
|
||||
func hostGraphRow(before Observation) row {
|
||||
var drawn []string
|
||||
for _, n := range before.Graph.Nodes {
|
||||
if ofHostExecution(n.ID) {
|
||||
drawn = append(drawn, n.ID+":"+string(n.State))
|
||||
}
|
||||
}
|
||||
sort.Strings(drawn)
|
||||
// The stream and the authority are reported together because only one of
|
||||
// them is surface-independent: a synchronous caller has no sink to draw on,
|
||||
// and a reader on either surface starts from the snapshot regardless.
|
||||
authority := firstProgress(before, probeHostAuthority)
|
||||
return row{
|
||||
Semantic: "the graph, drawn and reconstructable",
|
||||
Authority: liveAuthority + " and " + authorityName,
|
||||
Artifact: "<stem>.execution.jsonl", Reconstruction: "the stream where a sink exists; the authority always",
|
||||
Before: fmt.Sprintf("stream=%s authority=%s", orNone(join(drawn)), authority),
|
||||
After: "—", Verdict: presenceVerdict(authority != "none"),
|
||||
}
|
||||
}
|
||||
|
||||
func hostJournalRow(before, after Observation) row {
|
||||
return presenceRow("does the journal record it", journalAuthority, "<stem>.execution.jsonl",
|
||||
"ExecutionHistory over the whole session", hostExecutions(before), hostExecutions(after))
|
||||
}
|
||||
|
||||
func hostStoreRow(before, after Observation) row {
|
||||
return presenceRow("what the store holds for it", storeAuthority, "subagents/<ref>.meta.json",
|
||||
"ListSubagentsByParent over the transcript stem", hostChildRefs(before), hostChildRefs(after))
|
||||
}
|
||||
|
||||
// hostLineageOracleRow is the contract Step 8 respected and this must not
|
||||
// weaken. A synthetic host event id is UI identity; persisting one as a parent
|
||||
// would say the model issued a call it never made.
|
||||
func hostLineageOracleRow(before, after Observation) row {
|
||||
leaked := hostSyntheticLeaks(before)
|
||||
leaked = append(leaked, hostSyntheticLeaks(after)...)
|
||||
empty := hostLineageEmpty(before) && hostLineageEmpty(after)
|
||||
answer, verdict := "no persisted lineage names a synthetic host id, and every parent is empty", verdictHolds
|
||||
switch {
|
||||
case len(leaked) > 0:
|
||||
answer, verdict = "a synthetic host id was persisted as lineage: "+join(leaked), verdictViolated
|
||||
case !empty:
|
||||
answer, verdict = "a host-started run was filed under a parent call: "+
|
||||
join(hostParentLineage(after)), verdictViolated
|
||||
case len(hostChildRefs(before))+len(hostChildRefs(after)) == 0:
|
||||
verdict = verdictNotMeasured
|
||||
answer = "no child record exists to carry a lineage"
|
||||
}
|
||||
return row{
|
||||
Semantic: "synthetic host id persisted as lineage", Authority: storeAuthority,
|
||||
Artifact: "subagents/<ref>.meta.json", Reconstruction: "every persisted parent, read for host event ids",
|
||||
Before: answer, After: "—", Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func hostSyntheticLeaks(o Observation) []string {
|
||||
var out []string
|
||||
for _, f := range o.Children.Facts {
|
||||
if strings.HasPrefix(f.ParentToolCallID, "slash-skill-") {
|
||||
out = append(out, f.Ref+"→"+f.ParentToolCallID)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func hostLineageEmpty(o Observation) bool {
|
||||
for _, f := range o.Children.Facts {
|
||||
if f.Kind == "skill" && strings.TrimSpace(f.ParentToolCallID) != "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// hostPreStartIdentityRow is what decides where an identity would have to be
|
||||
// created. Work the scheduler is holding back has no child artifact yet, so a
|
||||
// store ref cannot name it; if nothing else survives the process either, no
|
||||
// candidate identity exists at the moment the work first became real.
|
||||
func hostPreStartIdentityRow(arm string, before, after Observation) row {
|
||||
survives := len(hostExecutions(after)) > 0
|
||||
answer := "nothing that outlives the process names this work"
|
||||
verdict := verdictIdentityGap
|
||||
if survives {
|
||||
answer, verdict = "an execution the journal opened: "+join(hostExecutions(after)), verdictHolds
|
||||
}
|
||||
if !queuedHostArm(arm) && firstProgress(before, probeChildEntered) != "true" {
|
||||
verdict = verdictNotMeasured
|
||||
}
|
||||
return row{
|
||||
Semantic: "an identity that exists before the child does", Authority: crossAuthority,
|
||||
Artifact: "<stem>.execution.jsonl",
|
||||
Reconstruction: "the candidates, read against what the work needs",
|
||||
Before: hostAtDeath(arm), After: answer, Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func hostAtDeath(arm string) string {
|
||||
switch {
|
||||
case queuedHostArm(arm):
|
||||
return "admitted-domain work, held back, no child artifact yet"
|
||||
case arm == armHostCompleted || arm == armHeadlessCompleted:
|
||||
return "ran and returned"
|
||||
}
|
||||
return "executing inside the provider"
|
||||
}
|
||||
|
||||
func hostCancelRow(arm string, before Observation) row {
|
||||
if arm != armHostCancel && !callerStopArm(arm) {
|
||||
return row{
|
||||
Semantic: "what a stop reaches", Authority: "the turn's own cancel",
|
||||
Artifact: "none observed", Reconstruction: "in-process only",
|
||||
Before: "not stopped in this arm", After: "—", Verdict: verdictNotMeasured,
|
||||
}
|
||||
}
|
||||
return row{
|
||||
Semantic: "what a stop reaches", Authority: "the turn's own cancel",
|
||||
Artifact: "subagents/<ref>.meta.json",
|
||||
Reconstruction: "the live stop and the record it left, kept apart",
|
||||
Before: firstProgress(before, probeStopReached) + " store=" + orNone(join(hostChildRefs(before))),
|
||||
After: "—", Verdict: verdictHolds,
|
||||
}
|
||||
}
|
||||
|
||||
// hostRestartRow classifies what the next process inherits. An answer that
|
||||
// survived while its execution did not is not a loss to the user, so it is not
|
||||
// called one; work that was executing and left nothing is.
|
||||
func hostRestartRow(arm string, before, after Observation) row {
|
||||
exec, children := hostExecutions(after), hostChildRefs(after)
|
||||
rebuilt := hostRebuilt(after)
|
||||
answer := fmt.Sprintf("execution=%s store=%s rebuilt=%s",
|
||||
orNone(join(exec)), orNone(join(children)), orNone(join(rebuilt)))
|
||||
return row{
|
||||
Semantic: "what a restart inherits", Authority: rebuildAuthority,
|
||||
Artifact: "<stem>.execution.jsonl + subagents/<ref>.meta.json",
|
||||
Reconstruction: "execgraph.Rebuild, then the store beside it",
|
||||
Before: hostAtDeath(arm), After: answer,
|
||||
Verdict: hostRestartVerdict(arm, before, exec, children),
|
||||
}
|
||||
}
|
||||
|
||||
// hostRestartVerdict is driven by what a next process can actually hold, not by
|
||||
// what the arm expected. A child record with nothing to place it is not silence:
|
||||
// a reader can still see that a child of this name was cut. What it cannot do is
|
||||
// say which invocation that was, and the identity row answers that separately —
|
||||
// folding the two would report a surviving record as a total loss.
|
||||
func hostRestartVerdict(arm string, before Observation, exec, children []string) string {
|
||||
switch {
|
||||
case len(exec) > 0:
|
||||
return verdictHolds
|
||||
case len(children) > 0:
|
||||
return verdictResultDurable
|
||||
case firstProgress(before, probeChildEntered) == "true" || queuedHostArm(arm):
|
||||
return graphLostSilent
|
||||
default:
|
||||
return verdictNotMeasured
|
||||
}
|
||||
}
|
||||
|
||||
// hostRebuilt is what a restart draws for this invocation. The whole rebuilt
|
||||
// graph is wider than the arm — the queued arm fills its ceiling with a fan-out,
|
||||
// and a fan-out records everything — so reading its nodes as the invocation's is
|
||||
// how an arm reports a gap as closed by evidence belonging to something else.
|
||||
func hostRebuilt(o Observation) []string {
|
||||
var out []string
|
||||
for _, n := range rebuiltGraph(o).Graph.Nodes {
|
||||
if n.Kind == agentgraph.KindGroup {
|
||||
continue
|
||||
}
|
||||
if ofHostExecution(n.ID) || strings.Contains(n.Label, probeSkillName) {
|
||||
out = append(out, n.ID+":"+string(n.State))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// hostExecArmInvalid reports the premise this arm could not establish.
|
||||
func hostExecArmInvalid(arm string, before Observation) string {
|
||||
entered := firstProgress(before, probeChildEntered) == "true"
|
||||
switch {
|
||||
case queuedHostArm(arm) && entered:
|
||||
return "the invocation ran while the ceiling was meant to be full"
|
||||
case !queuedHostArm(arm) && !entered:
|
||||
return "the invocation's child never reached the provider"
|
||||
case headlessHostArm(arm) && len(hostDispatchIDs2(before)) > 0:
|
||||
return "a synchronous call minted a dispatch id, so this was not the headless surface"
|
||||
case !headlessHostArm(arm) && len(hostDispatchIDs2(before)) == 0:
|
||||
return "the host minted no dispatch id, so this was not the slash surface"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// hostDispatchIDs2 reads the dispatch id back off the observation, where the
|
||||
// construct phase recorded what only a live sink could see.
|
||||
func hostDispatchIDs2(o Observation) []string {
|
||||
if got := firstProgress(o, probeHostDispatch); got != "none" && got != "unrecorded" {
|
||||
return strings.Split(got, ", ")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/session/control"
|
||||
)
|
||||
|
||||
// Two trees, read at once. A stop aimed at a turn must close the turn's own
|
||||
// tree and leave the job's alone; an explicit job stop does the reverse, and
|
||||
// reading only one of them is how a probe calls a violated handoff correct. The
|
||||
// manager's own root needs no separate question: a job still running with its
|
||||
// child's context open is the root answering directly.
|
||||
const (
|
||||
// armLineageForeground is the positive control: it proves the reading can
|
||||
// see an ordinary cancellation arrive.
|
||||
armLineageForeground = "lineage-foreground-cancel"
|
||||
// armLineageBackground is the one under suspicion.
|
||||
armLineageBackground = "lineage-background-cancel"
|
||||
// armLineageJobKill is the other positive control: without it, a reading
|
||||
// that could never observe a job's own cancellation would make the arm
|
||||
// above pass for the wrong reason.
|
||||
armLineageJobKill = "lineage-job-kill"
|
||||
)
|
||||
|
||||
func lineageArm(name string) bool {
|
||||
switch name {
|
||||
case armLineageForeground, armLineageBackground, armLineageJobKill:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// runLineageConstruct drives one delegation, stops it the way this arm's owner
|
||||
// stops it, and reads both trees afterwards. It records six facts, kept apart
|
||||
// on purpose: a context that closed and a call that returned are different
|
||||
// events, and only their combination says whether a cancellation propagated or
|
||||
// something else ended the work.
|
||||
func runLineageConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
|
||||
ctrl.Send(fanOutTurn(turn+1, taskSentinel))
|
||||
if err := waitFor(60*time.Second, func() bool { return prov.fleets.held(childEntered(childHang)) }); err != nil {
|
||||
return errUnexpected("a delegated child inside the provider", "none arrived")
|
||||
}
|
||||
before := lineageReading(ctrl, prov, "before")
|
||||
|
||||
switch arm {
|
||||
case armLineageJobKill:
|
||||
jobs := ctrl.Jobs()
|
||||
if len(jobs) == 0 {
|
||||
return errUnexpected("a background job to stop", "none running")
|
||||
}
|
||||
for _, job := range jobs {
|
||||
ctrl.CancelJob(job.ID)
|
||||
}
|
||||
default:
|
||||
ctrl.Cancel()
|
||||
}
|
||||
// Long enough for a cancellation that was going to arrive to have arrived.
|
||||
time.Sleep(3 * time.Second)
|
||||
after := lineageReading(ctrl, prov, "after")
|
||||
|
||||
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
|
||||
obs.Progress[probeLineage] = []string{before, after}
|
||||
if err := writeObservation(root, obs); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Exit(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
const probeLineage = "probe:lineage"
|
||||
|
||||
// lineageReading is the state of both trees at one instant, in the terms the
|
||||
// rows judge: the turn's gate, the job's own liveness, the child's context, the
|
||||
// child's call, and the durable record the orchestration keeps.
|
||||
func lineageReading(ctrl *control.Controller, prov *scripted, at string) string {
|
||||
return fmt.Sprintf("%s turn=%s job=%s childCtx=%s childCall=%s record=%s",
|
||||
at, turnGateStanding(ctrl), jobStanding(ctrl),
|
||||
yesNo(prov.fleets.held(childCtxDone(childHang)), "cancelled", "open"),
|
||||
yesNo(prov.fleets.held(childLeft(childHang)), "returned", "inside"),
|
||||
loneRecordStanding(ctrl))
|
||||
}
|
||||
|
||||
func turnGateStanding(ctrl *control.Controller) string {
|
||||
if ctrl.Running() {
|
||||
return "active"
|
||||
}
|
||||
return "ended"
|
||||
}
|
||||
|
||||
func jobStanding(ctrl *control.Controller) string {
|
||||
if len(ctrl.Jobs()) > 0 {
|
||||
return "running"
|
||||
}
|
||||
return "gone"
|
||||
}
|
||||
|
||||
func loneRecordStanding(ctrl *control.Controller) string {
|
||||
for _, e := range control.ExecutionHistory(ctrl.SessionPath()) {
|
||||
if e.ID == parallelLoneNodeID {
|
||||
return lifecycleOf(e)[1:]
|
||||
}
|
||||
}
|
||||
return "absent"
|
||||
}
|
||||
|
||||
func yesNo(ok bool, yes, no string) string {
|
||||
if ok {
|
||||
return yes
|
||||
}
|
||||
return no
|
||||
}
|
||||
|
||||
func waitFor(within time.Duration, ready func() bool) error {
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
if ready() {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("timed out")
|
||||
}
|
||||
|
||||
// lineageRows read the two trees against the ownership each arm is about. The
|
||||
// three sets of expectations are deliberately different: a fix that closes
|
||||
// everything on a stop satisfies the foreground arm and violates both others.
|
||||
func lineageRows(arm string, before Observation) []row {
|
||||
at := reading(before.Progress[probeLineage], 1)
|
||||
turnEnded := strings.Contains(at, "turn=ended")
|
||||
jobLive := strings.Contains(at, "job=running")
|
||||
childCancelled := strings.Contains(at, "childCtx=cancelled")
|
||||
recordOpen := !strings.Contains(at, "settled") && !strings.Contains(at, "record=absent")
|
||||
|
||||
rows := []row{lineageRow("the turn the stop was aimed at", at, "ends", turnEnded)}
|
||||
if arm == armLineageForeground {
|
||||
return append(rows,
|
||||
lineageRow("the child's own context", at, "closes with the turn that owned it", childCancelled),
|
||||
lineageRow("the delegation record", at, "is let go of", !recordOpen))
|
||||
}
|
||||
if arm == armLineageJobKill {
|
||||
return append(rows[:0],
|
||||
lineageRow("the job the stop was aimed at", at, "ends", !jobLive),
|
||||
lineageRow("the child's own context", at, "closes with the job that owned it", childCancelled),
|
||||
lineageRow("the delegation record", at, "is let go of", !recordOpen))
|
||||
}
|
||||
return append(rows,
|
||||
lineageRow("the job the turn no longer owns", at, "keeps running", jobLive),
|
||||
lineageRow("the child's own context", at, "is not closed by another owner's stop", !childCancelled),
|
||||
lineageRow("the delegation record", at, "stays open while its child runs", recordOpen))
|
||||
}
|
||||
|
||||
func lineageRow(semantic, got, want string, ok bool) row {
|
||||
return row{
|
||||
Semantic: semantic, Authority: "both trees, read at the same instant",
|
||||
Artifact: "none: in-process state", Reconstruction: want,
|
||||
Before: want, After: orNone(got), Verdict: held(ok),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/agentgraph"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/session/control"
|
||||
)
|
||||
|
||||
// The four single-delegation arms. A lone task is not another executor: it
|
||||
// reaches the same RunProfileSpec a fleet item does and asks the same scheduler
|
||||
// for a slot. What these ask is whether that shared chain is wired to the
|
||||
// durable authority the fan-out arms established — and where, exactly, it stops
|
||||
// being wired, which is a different question from "does it have a graph".
|
||||
const (
|
||||
// armTaskCompleted lets the delegation finish and the turn close. Nothing is
|
||||
// interrupted, so a loss here is not about interruption: it says the run's
|
||||
// provenance was never durable at all.
|
||||
armTaskCompleted = "task-completed"
|
||||
// armTaskRunning kills the process with the child mid-execution, the same
|
||||
// death the fan-out arms take.
|
||||
armTaskRunning = "task-running"
|
||||
// armTaskFgQueued is the live-truth arm: the ceiling is full when the
|
||||
// delegation asks, so the scheduler is holding it back. What the parent and
|
||||
// the graph say while that is true is the whole question — a picture that
|
||||
// says running while nothing has started is wrong before any restart.
|
||||
armTaskFgQueued = "task-foreground-queued"
|
||||
// armTaskBgQueued is the one the foreground arms cannot stand in for: a
|
||||
// background run hands its execution to a job and asks for a slot inside it,
|
||||
// so the parent tool call has already returned when the refusal happens.
|
||||
armTaskBgQueued = "task-background-queued"
|
||||
// armTaskBgRunning is the same handoff with the slot granted, which is where
|
||||
// ownership of the terminal moves from the caller to the job.
|
||||
armTaskBgRunning = "task-background-running"
|
||||
// The cancellation arms. They ask the one question the five before them
|
||||
// cannot: who owns the ending when work is stopped, on each side of the
|
||||
// boundary that decides whether it ever ran. A child the scheduler never
|
||||
// admitted produced nothing, so the store owes it no terminal at all.
|
||||
armTaskFgQueuedCancel = "task-foreground-queued-cancel"
|
||||
armTaskBgQueuedCancel = "task-background-queued-cancel"
|
||||
armTaskFgRunningCancel = "task-foreground-running-cancel"
|
||||
armTaskBgRunningCancel = "task-background-running-cancel"
|
||||
)
|
||||
|
||||
func cancelTaskArm(name string) bool {
|
||||
switch name {
|
||||
case armTaskFgQueuedCancel, armTaskBgQueuedCancel, armTaskFgRunningCancel, armTaskBgRunningCancel:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// beforeStartArm names the arms cancelled while the scheduler still held the
|
||||
// delegation, which is the half of the boundary the store must stay out of.
|
||||
func beforeStartArm(name string) bool {
|
||||
return name == armTaskFgQueuedCancel || name == armTaskBgQueuedCancel
|
||||
}
|
||||
|
||||
func loneTaskArm(name string) bool {
|
||||
switch name {
|
||||
case armTaskCompleted, armTaskRunning, armTaskFgQueued, armTaskBgQueued, armTaskBgRunning:
|
||||
return true
|
||||
}
|
||||
return cancelTaskArm(name)
|
||||
}
|
||||
|
||||
// queuedTaskArm names the arms that fill the ceiling before delegating, so the
|
||||
// scheduler has to refuse the delegation admission.
|
||||
func queuedTaskArm(name string) bool {
|
||||
return name == armTaskFgQueued || name == armTaskBgQueued || beforeStartArm(name)
|
||||
}
|
||||
|
||||
// childEntered names the record a child leaves when it reaches the provider.
|
||||
// The queued arms ask a negative — that one never did — which nothing else in
|
||||
// the probe can answer.
|
||||
func childEntered(sentinel string) string { return "child-entered:" + sentinel }
|
||||
|
||||
// childLeft names the record a child leaves when it returns. Entered without
|
||||
// left is a child still inside the provider — the state a cancellation that
|
||||
// never arrived leaves behind.
|
||||
func childLeft(sentinel string) string { return "child-left:" + sentinel }
|
||||
|
||||
// childCtxDone names the record a child leaves when its own context closes,
|
||||
// which is a different fact from having returned: a run whose caller stopped
|
||||
// waiting returns to nobody with its context still open.
|
||||
func childCtxDone(sentinel string) string { return "child-ctxdone:" + sentinel }
|
||||
|
||||
// probeChildEntered is where that answer rides into the observation. It is not
|
||||
// a status the kernel emits: the key names the probe so no row reads it as one.
|
||||
const probeChildEntered = "probe:delegated-child-entered"
|
||||
|
||||
func backgroundTaskArm(name string) bool {
|
||||
switch name {
|
||||
case armTaskBgQueued, armTaskBgRunning, armTaskBgQueuedCancel, armTaskBgRunningCancel,
|
||||
armLineageBackground, armLineageJobKill, armCancelBackground, armFleetActiveStore:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The sentinels one turn carries. The queued arm names the holder as well: its
|
||||
// ceiling has to be occupied before the delegation asks for a slot, or the
|
||||
// refusal it is built to record never happens.
|
||||
const (
|
||||
taskSentinel = "PROBE-TASK"
|
||||
taskCallID = "probe_task"
|
||||
taskHoldsFirst = fleetHolderSentinel + " " + taskSentinel
|
||||
)
|
||||
|
||||
// loneTaskCall is the delegation itself, dispatched through the tool a model
|
||||
// calls. Nothing here reaches past the tool's own arguments: the probe measures
|
||||
// the production path, so it must enter it the way a model does.
|
||||
func loneTaskCall(arm string) []provider.Chunk {
|
||||
prompt, description := childDone+" lone task", "completes"
|
||||
switch {
|
||||
case arm == armFleetActiveStore:
|
||||
// This delegation is the other occupant of the ceiling its arm shares
|
||||
// with a fan-out, and its own child is the only thing that can say the
|
||||
// slot is taken. Holding is how it hangs and says so at once.
|
||||
prompt, description = childHold+" lone task", "holds a slot to the death"
|
||||
case arm != armTaskCompleted:
|
||||
prompt, description = childHang+" lone task", "still executing at death"
|
||||
}
|
||||
args, _ := json.Marshal(map[string]any{
|
||||
"prompt": prompt, "description": description,
|
||||
"run_in_background": backgroundTaskArm(arm),
|
||||
})
|
||||
return []provider.Chunk{{
|
||||
Type: provider.ChunkToolCall,
|
||||
ToolCall: &provider.ToolCall{ID: taskCallID, Name: "task", Arguments: string(args)},
|
||||
}}
|
||||
}
|
||||
|
||||
// probeStepIDs are the host's open steps, in the order they have to be signed.
|
||||
// The arm whose turn ends has to close them: a delivery is refused while the
|
||||
// list is open, and a todo_write that marks an item completed on its own is
|
||||
// refused too — the sign-off is the only door.
|
||||
func probeStepIDs() []string { return []string{"probe_step_01", "probe_step_02", "probe_step_03"} }
|
||||
|
||||
// signStep signs off one step per round, which is what the host accepts: the
|
||||
// sign-off promotes the next item, so two in one round sign against a list that
|
||||
// no longer stands.
|
||||
func signStep(id string) []provider.Chunk {
|
||||
args, _ := json.Marshal(map[string]any{
|
||||
"step_id": id,
|
||||
"result": "the probe drove this step to its end",
|
||||
"evidence": []map[string]any{
|
||||
{"kind": "manual", "summary": "the probe established this state directly"},
|
||||
},
|
||||
})
|
||||
return []provider.Chunk{{
|
||||
Type: provider.ChunkToolCall,
|
||||
ToolCall: &provider.ToolCall{ID: "probe_sign_" + id, Name: "complete_step", Arguments: string(args)},
|
||||
}}
|
||||
}
|
||||
|
||||
// closesItsTurn names the arms whose turn has to reach its end. A turn that
|
||||
// called a tool cannot deliver while the host's step list is open, so these
|
||||
// sign it off one step per round — the sign-off promotes the next item, and two
|
||||
// in one round would sign against a list that no longer stands.
|
||||
func closesItsTurn(arm string) bool {
|
||||
return arm == armTaskCompleted || arm == armSkillCompleted || arm == armReadOnlySkillCompleted
|
||||
}
|
||||
|
||||
// loneTaskSentinels is what this arm's one prompt carries.
|
||||
func loneTaskSentinels(arm string) string {
|
||||
if queuedTaskArm(arm) {
|
||||
return taskHoldsFirst
|
||||
}
|
||||
return taskSentinel
|
||||
}
|
||||
|
||||
// runLoneTaskConstruct drives one delegation and dies at the moment this arm is
|
||||
// about. The completed arm is the exception that proves the rest: it lets the
|
||||
// turn close, so what it loses was never about dying inside one.
|
||||
func runLoneTaskConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
|
||||
// One turn, whatever the arm needs in it: a session admits no second one. The
|
||||
// queued arm's ceiling is filled by a background fleet named in the same
|
||||
// prompt, whose own child says when the slot is taken.
|
||||
prompt := fanOutTurn(turn+1, loneTaskSentinels(arm))
|
||||
switch {
|
||||
case arm == armTaskCompleted:
|
||||
if err := ctrl.Run(ctx, prompt); err != nil {
|
||||
return fmt.Errorf("delegating turn: %w", err)
|
||||
}
|
||||
case guardedArm(arm):
|
||||
// A stop belongs to whoever admitted the turn, so an arm that is going
|
||||
// to be stopped is admitted the way a person's input is.
|
||||
ctrl.Send(prompt)
|
||||
if err := waitForLoneTask(sink, prov, arm); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
go func() { _ = ctrl.Run(ctx, prompt) }()
|
||||
if err := waitForLoneTask(sink, prov, arm); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
stop := ""
|
||||
if cancelTaskArm(arm) {
|
||||
stop = stopDelegation(ctrl, prov, arm)
|
||||
}
|
||||
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
|
||||
if stop != "" {
|
||||
obs.Progress[probeStopReached] = []string{stop}
|
||||
}
|
||||
if queuedTaskArm(arm) {
|
||||
// Whether the delegation's own child ever ran is the fact the live rows
|
||||
// are read against, and only the scripted provider knows it.
|
||||
obs.Progress[probeChildEntered] = []string{fmt.Sprint(prov.fleets.held(childEntered(childHang)))}
|
||||
}
|
||||
if err := writeObservation(root, obs); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Exit(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// stopDelegation stops the work the way a person does — the turn's own cancel
|
||||
// for a foreground run, the job list and a kill for a backgrounded one — and
|
||||
// waits for the orchestration to let go of it. What each layer holds then is
|
||||
// this arm's whole question, so nothing is read until the record closes.
|
||||
func stopDelegation(ctrl *control.Controller, prov *scripted, arm string) string {
|
||||
stopped := time.Now()
|
||||
if backgroundTaskArm(arm) {
|
||||
for _, job := range ctrl.Jobs() {
|
||||
ctrl.CancelJob(job.ID)
|
||||
}
|
||||
} else {
|
||||
ctrl.Cancel()
|
||||
}
|
||||
// Wait for the record to close, but do not demand it: whether a stop reaches
|
||||
// the work it was aimed at is what this arm measures, and an arm that failed
|
||||
// when it did not would report the finding as its own premise breaking.
|
||||
deadline := time.Now().Add(20 * time.Second)
|
||||
for time.Now().Before(deadline) && !settledLone(ctrl) {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
live := "gone"
|
||||
if prov.fleets.held(childEntered(childHang)) && !prov.fleets.held(childLeft(childHang)) {
|
||||
live = "still running"
|
||||
}
|
||||
return fmt.Sprintf("turn=%s work=%s after=%s", turnStanding(ctrl, arm), live,
|
||||
time.Since(stopped).Round(time.Second))
|
||||
}
|
||||
|
||||
func settledLone(ctrl *control.Controller) bool {
|
||||
for _, e := range control.ExecutionHistory(ctrl.SessionPath()) {
|
||||
if e.ID == parallelLoneNodeID && !e.SettledAt.IsZero() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// waitForLoneTask blocks until the delegation stands where this arm dies. The
|
||||
// foreground arms read the live graph; the background ones cannot — a handed-off
|
||||
// run publishes no node — so they read the progress the parent was told.
|
||||
func waitForLoneTask(sink *graphSink, prov *scripted, arm string) error {
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if loneTaskReady(sink, prov, arm) {
|
||||
// A refusal is a state nothing announces, so the arm holds still and
|
||||
// asks again: a child that was merely slow to start would have
|
||||
// arrived by now, and the death would be measuring a race.
|
||||
if beforeStartArm(arm) || arm == armTaskFgQueued {
|
||||
time.Sleep(2 * time.Second)
|
||||
return refusalHolding(prov, arm)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
graph, _ := sink.snapshot()
|
||||
return errUnexpected("a lone delegation "+loneTaskWanted(arm),
|
||||
nodeStates(graph)+" progress="+fmt.Sprint(sink.phaseSeries()))
|
||||
}
|
||||
|
||||
// refusalHolding reports the arm's premise once more, after the settle: the
|
||||
// delegation's child must still not have run.
|
||||
func refusalHolding(prov *scripted, arm string) error {
|
||||
if prov.fleets.held(childEntered(childHang)) {
|
||||
return errUnexpected("a delegation the scheduler was still holding back", "its child ran")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func loneTaskReady(sink *graphSink, prov *scripted, arm string) bool {
|
||||
graph, _ := sink.snapshot()
|
||||
switch arm {
|
||||
case armTaskFgQueued, armTaskFgQueuedCancel:
|
||||
// The node is drawn — the delegation reached the graph — and its child
|
||||
// has not started, which with the ceiling full is the refusal.
|
||||
return loneNodeDrawn(graph) && !prov.fleets.held(childEntered(childHang))
|
||||
case armTaskRunning, armTaskFgRunningCancel, armTaskBgRunningCancel:
|
||||
return holdsAll(graph, []agentgraph.NodeState{agentgraph.StateRunning})
|
||||
case armTaskBgQueued, armTaskBgQueuedCancel:
|
||||
phases := sink.phaseSeries()[taskCallID]
|
||||
return len(phases) > 0 && phases[len(phases)-1] == string(subagentQueued)
|
||||
case armTaskBgRunning:
|
||||
return slices.Contains(sink.phaseSeries()[taskCallID], subagentRunning)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// loneNodeDrawn reports whether the delegation's own node is on the graph.
|
||||
func loneNodeDrawn(g agentgraph.Graph) bool {
|
||||
for _, n := range g.Nodes {
|
||||
if n.Kind == agentgraph.KindWorker && n.ID == parallelLoneNodeID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// parallelLoneNodeID is what a lone delegation's node is called: the parent
|
||||
// call, and the first slot of the fan-out numbering it shares.
|
||||
const parallelLoneNodeID = taskCallID + "/sub-1"
|
||||
|
||||
func loneTaskWanted(arm string) string {
|
||||
switch arm {
|
||||
case armTaskRunning, armTaskFgRunningCancel, armTaskBgRunningCancel:
|
||||
return "running"
|
||||
case armTaskFgQueued, armTaskFgQueuedCancel, armTaskBgQueuedCancel:
|
||||
return "drawn while the scheduler holds it back"
|
||||
case armTaskBgQueued:
|
||||
return "queued for a slot and not started"
|
||||
}
|
||||
return "running in its job"
|
||||
}
|
||||
|
||||
// The two phases the probe waits on. They are the parent-facing statuses a
|
||||
// background delegation emits, and for a handed-off run they are the only live
|
||||
// evidence there is.
|
||||
const (
|
||||
subagentQueued = "queued"
|
||||
subagentRunning = "running"
|
||||
)
|
||||
|
||||
// progressPhase reads one subagent status event. The tool id is the parent call
|
||||
// the delegation was made from, which is also what the graph node hangs under
|
||||
// and what the sub-agent store records as its parent — the join this arm exists
|
||||
// to check rather than assume.
|
||||
func progressPhase(e event.Event) (id, phase string, ok bool) {
|
||||
if e.Kind != event.ToolProgress || e.Tool.Name != event.SubagentProgressStatusName {
|
||||
return "", "", false
|
||||
}
|
||||
return e.Tool.ID, e.Tool.Output, e.Tool.ID != ""
|
||||
}
|
||||
@@ -0,0 +1,536 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/agentgraph"
|
||||
"reasonix/internal/state/execgraph"
|
||||
"reasonix/internal/state/execjournal"
|
||||
)
|
||||
|
||||
// What a lone delegation leaves behind, layer by layer. The columns are the
|
||||
// ordinary ones: what the dying process held, and what the next one can still
|
||||
// say. Nothing here is a fix — the arm exists to find which transition stops
|
||||
// being recorded, because "single task has no graph" was the roadmap's guess
|
||||
// and the live chain has since been unified.
|
||||
func loneTaskRows(arm string, before, after Observation) []row {
|
||||
return []row{
|
||||
liveDelegationRow(before),
|
||||
parentStatusRow(before),
|
||||
durableOpeningRow(before, after),
|
||||
durableQueueRow(before, after),
|
||||
durableStartRow(before, after),
|
||||
durableSettleRow(before, after),
|
||||
delegationStoreRow(before, after),
|
||||
rebuiltDelegationRow(before, after),
|
||||
rebuiltDelegationIdentityRow(after),
|
||||
delegationInterruptionRow(before, after),
|
||||
identityJoinRow(before),
|
||||
heldBackRow(arm, before),
|
||||
stoppedOwnershipRow(arm, before),
|
||||
startedTerminalRow(before, after),
|
||||
fanOutControlRow(before),
|
||||
}
|
||||
}
|
||||
|
||||
// ofLoneTask reports whether an id belongs to the delegation under measurement.
|
||||
// The queued arm fills its ceiling with a fan-out, and a fan-out records
|
||||
// everything — so without this the rows would read that record as the lone
|
||||
// task's own and report a gap as closed.
|
||||
func ofLoneTask(id string) bool {
|
||||
return id == taskCallID || strings.HasPrefix(id, taskCallID+"/")
|
||||
}
|
||||
|
||||
const (
|
||||
crossAuthority = "the journal and the store, read against each other"
|
||||
liveAuthority = "event sink (GraphDelta), in-process"
|
||||
journalAuthority = "execjournal, written by the orchestration"
|
||||
storeAuthority = "sub-agent store"
|
||||
rebuildAuthority = "execgraph.Rebuild over journal + store"
|
||||
)
|
||||
|
||||
// liveDelegationRow is what the dying process actually drew. A backgrounded run
|
||||
// draws nothing at all — openDelegationGraph returns early for it — so this row
|
||||
// separates "the graph was lost" from "there was never a node".
|
||||
func liveDelegationRow(before Observation) row {
|
||||
var drawn []string
|
||||
for _, n := range before.Graph.Nodes {
|
||||
if n.Kind == agentgraph.KindWorker && ofLoneTask(n.ID) {
|
||||
drawn = append(drawn, n.ID+":"+string(n.State))
|
||||
}
|
||||
}
|
||||
sort.Strings(drawn)
|
||||
return row{
|
||||
Semantic: "the delegation the live graph drew", Authority: liveAuthority,
|
||||
Artifact: "none observed", Reconstruction: "in-process only",
|
||||
Before: orNone(join(drawn)), After: "—", Verdict: liveVerdict(len(drawn) > 0),
|
||||
}
|
||||
}
|
||||
|
||||
func liveVerdict(drawn bool) string {
|
||||
if drawn {
|
||||
return verdictStable
|
||||
}
|
||||
return verdictNotMeasured
|
||||
}
|
||||
|
||||
// parentStatusRow is the other live surface, and the only one a handed-off run
|
||||
// has: the statuses the parent tool call was told.
|
||||
func parentStatusRow(before Observation) row {
|
||||
var told []string
|
||||
for id, phases := range before.Progress {
|
||||
if ofLoneTask(id) {
|
||||
told = append(told, id+":"+strings.Join(phases, "→"))
|
||||
}
|
||||
}
|
||||
sort.Strings(told)
|
||||
return row{
|
||||
Semantic: "what the parent call was told", Authority: "subagent progress (ToolProgress)",
|
||||
Artifact: "none observed", Reconstruction: "in-process only",
|
||||
Before: orNone(join(told)), After: "—", Verdict: liveVerdict(len(told) > 0),
|
||||
}
|
||||
}
|
||||
|
||||
func durableOpeningRow(before, after Observation) row {
|
||||
return valueRow("durable opening", journalAuthority, "<stem>.execution.jsonl",
|
||||
"read back by ExecutionHistory", executionIDs(before), executionIDs(after), true)
|
||||
}
|
||||
|
||||
func durableQueueRow(before, after Observation) row {
|
||||
return valueRow("durable queue refusal", journalAuthority, "<stem>.execution.jsonl",
|
||||
"the scheduler's first refusal, with its cause",
|
||||
queuedCauses(before), queuedCauses(after), true)
|
||||
}
|
||||
|
||||
func durableStartRow(before, after Observation) row {
|
||||
return valueRow("durable slot grant", journalAuthority, "<stem>.execution.jsonl",
|
||||
"written before the child can act", startedIDs(before), startedIDs(after), true)
|
||||
}
|
||||
|
||||
func durableSettleRow(before, after Observation) row {
|
||||
return valueRow("durable settlement", journalAuthority, "<stem>.execution.jsonl",
|
||||
"the orchestration letting go of the execution",
|
||||
settledIDs(before), settledIDs(after), true)
|
||||
}
|
||||
|
||||
// delegationStoreRow is the layer that already survives for a fan-out item. It
|
||||
// is the reason this arm cannot be read off the graph alone: a store record with
|
||||
// no opening leaves the run's grant, worker and scheduling unaccounted for.
|
||||
func delegationStoreRow(before, after Observation) row {
|
||||
return valueRow("child the store kept", storeAuthority, "subagents/<ref>.meta.json",
|
||||
"ListSubagentsByParent over the transcript stem",
|
||||
childFacts(before), childFacts(after), true)
|
||||
}
|
||||
|
||||
func rebuiltDelegationRow(before, after Observation) row {
|
||||
rebuilt := rebuiltGraph(after)
|
||||
var nodes []string
|
||||
for _, n := range rebuilt.Graph.Nodes {
|
||||
if n.Kind != agentgraph.KindGroup && ofLoneTask(n.ID) {
|
||||
nodes = append(nodes, n.ID+":"+string(n.State))
|
||||
}
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
return valueRow("the node a restart rebuilds", rebuildAuthority, "<stem>.execution.jsonl",
|
||||
"execgraph.Rebuild", loneDelegationAtDeath(before), join(nodes), false)
|
||||
}
|
||||
|
||||
// rebuiltDelegationIdentityRow is what a reader sees on the rebuilt node beyond
|
||||
// its state. Grant and worker are opening facts: the store cannot answer them,
|
||||
// so an arm that only compared states would call a hollow node exact.
|
||||
func rebuiltDelegationIdentityRow(after Observation) row {
|
||||
var got []string
|
||||
for _, n := range rebuiltGraph(after).Graph.Nodes {
|
||||
if n.Kind == agentgraph.KindGroup || !ofLoneTask(n.ID) {
|
||||
continue
|
||||
}
|
||||
got = append(got, fmt.Sprintf("%s grant=%s model=%s effort=%s wait=%s",
|
||||
n.ID, orDash(string(n.Grant)), orDash(n.Model), orDash(n.Effort), orDash(string(n.Wait))))
|
||||
}
|
||||
sort.Strings(got)
|
||||
return row{
|
||||
Semantic: "grant, worker and wait on the rebuilt node", Authority: rebuildAuthority,
|
||||
Artifact: "<stem>.execution.jsonl", Reconstruction: "opening facts, which no store record carries",
|
||||
Before: "declared at the opening", After: orNone(join(got)),
|
||||
Verdict: rebuiltIdentityVerdict(got),
|
||||
}
|
||||
}
|
||||
|
||||
func rebuiltIdentityVerdict(got []string) string {
|
||||
if len(got) == 0 {
|
||||
return verdictLost
|
||||
}
|
||||
return verdictExact
|
||||
}
|
||||
|
||||
func delegationInterruptionRow(before, after Observation) row {
|
||||
var got []string
|
||||
for _, i := range rebuiltGraph(after).Interrupted {
|
||||
if ofLoneTask(i.Execution) {
|
||||
got = append(got, i.Execution+":"+interruptionKind(i))
|
||||
}
|
||||
}
|
||||
sort.Strings(got)
|
||||
return valueRow("the interruption a restart names", rebuildAuthority, "<stem>.execution.jsonl",
|
||||
"an execution with no owner is named, never redrawn as running",
|
||||
loneDelegationAtDeath(before), join(got), false)
|
||||
}
|
||||
|
||||
// loneDelegationAtDeath is the delegation as the dying process last knew it,
|
||||
// from whichever surface knew: a foreground run has a node, a handed-off one
|
||||
// has only the store's record, and an arm whose rows read the wrong surface
|
||||
// would report a loss the other one never had.
|
||||
func loneDelegationAtDeath(o Observation) string {
|
||||
if live := liveWorkers(o); live != "" {
|
||||
return live
|
||||
}
|
||||
return childFacts(o)
|
||||
}
|
||||
|
||||
func interruptionKind(i execgraph.Interruption) string {
|
||||
if i.Started {
|
||||
return execjournal.InterruptedDuringExecution
|
||||
}
|
||||
return execjournal.InterruptedBeforeStart
|
||||
}
|
||||
|
||||
// identityJoinRow is the question that decides whether Step 8 needs an identity
|
||||
// at all: the live node, the call it hung under, the parent the store recorded,
|
||||
// and the execution the journal opened. Four spellings, or one identity.
|
||||
func identityJoinRow(before Observation) row {
|
||||
live, call, stored, opened := identitiesAtDeath(before)
|
||||
verdict := verdictViolated
|
||||
switch {
|
||||
case stored != "" && (stored == opened || stored == call || strings.HasPrefix(live, stored)):
|
||||
verdict = verdictHolds
|
||||
case stored == "" && !startedAtDeath(before):
|
||||
// Nothing executed, so the store owes no record and there is no join to
|
||||
// make. A missing artifact here is the arm's own premise, not a defect.
|
||||
verdict = verdictNotMeasured
|
||||
}
|
||||
return row{
|
||||
Semantic: "one identity or four", Authority: "the graph, the call, the store and the journal",
|
||||
Artifact: "subagents/<ref>.meta.json, <stem>.execution.jsonl",
|
||||
Reconstruction: "the store's parent call is what a rebuild joins on",
|
||||
Before: fmt.Sprintf("node=%s call=%s", orDash(live), orDash(call)),
|
||||
After: fmt.Sprintf("store parent=%s journal=%s", orDash(stored), orDash(opened)),
|
||||
Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
// identitiesAtDeath reads the four spellings out of the dying process's own
|
||||
// observation, so a join that only holds after a restart cannot be claimed.
|
||||
func identitiesAtDeath(o Observation) (live, call, stored, opened string) {
|
||||
for _, n := range o.Graph.Nodes {
|
||||
if n.Kind == agentgraph.KindWorker && ofLoneTask(n.ID) {
|
||||
live, call = n.ID, n.ParentID
|
||||
break
|
||||
}
|
||||
}
|
||||
for id := range o.Progress {
|
||||
if call == "" && ofLoneTask(id) {
|
||||
call = id
|
||||
}
|
||||
}
|
||||
for _, f := range o.Children.Facts {
|
||||
if ofLoneTask(f.ParentToolCallID) {
|
||||
stored = f.ParentToolCallID
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, e := range o.Executions {
|
||||
if ofLoneTask(e.ID) {
|
||||
opened = e.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
return live, call, stored, opened
|
||||
}
|
||||
|
||||
// startedAtDeath reports whether the delegation was ever granted a slot.
|
||||
func startedAtDeath(o Observation) bool {
|
||||
for _, e := range o.Executions {
|
||||
if ofLoneTask(e.ID) && e.Started() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func executionIDs(o Observation) string {
|
||||
var out []string
|
||||
for _, e := range o.Executions {
|
||||
if ofLoneTask(e.ID) {
|
||||
out = append(out, e.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
func queuedCauses(o Observation) string {
|
||||
var out []string
|
||||
for _, e := range o.Executions {
|
||||
if e.Queued() && ofLoneTask(e.ID) {
|
||||
out = append(out, e.ID+":"+orDash(e.Cause))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
func startedIDs(o Observation) string {
|
||||
var out []string
|
||||
for _, e := range o.Executions {
|
||||
if e.Started() && ofLoneTask(e.ID) {
|
||||
out = append(out, e.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
func settledIDs(o Observation) string {
|
||||
var out []string
|
||||
for _, e := range o.Executions {
|
||||
if !e.SettledAt.IsZero() && ofLoneTask(e.ID) {
|
||||
out = append(out, e.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
func childFacts(o Observation) string {
|
||||
var out []string
|
||||
for _, f := range o.Children.Facts {
|
||||
if ofLoneTask(f.ParentToolCallID) {
|
||||
out = append(out, f.ParentToolCallID+":"+f.Status)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
func liveWorkers(o Observation) string {
|
||||
var out []string
|
||||
for _, n := range o.Graph.Nodes {
|
||||
if n.Kind == agentgraph.KindWorker && ofLoneTask(n.ID) {
|
||||
out = append(out, n.ID+":"+string(n.State))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
func orDash(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "-"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// heldBackRow is the live-truth row: with the ceiling full and the delegation's
|
||||
// own child never reaching the provider, the scheduler is holding it back — and
|
||||
// what the picture says while that is true is a claim about now, which no
|
||||
// restart can correct. Running is the answer that is wrong.
|
||||
func heldBackRow(arm string, before Observation) row {
|
||||
if !queuedTaskArm(arm) {
|
||||
return row{
|
||||
Semantic: "what the picture says while the scheduler holds it back", Authority: liveAuthority,
|
||||
Artifact: "none observed", Reconstruction: "this arm never fills the ceiling",
|
||||
Before: "—", After: "—", Verdict: verdictNotMeasured,
|
||||
}
|
||||
}
|
||||
drawn := orNone(liveWorkers(before))
|
||||
told := orNone(loneStatus(before))
|
||||
verdict := verdictHolds
|
||||
if strings.Contains(drawn, ":running") || told == subagentRunning {
|
||||
verdict = verdictViolated
|
||||
}
|
||||
return row{
|
||||
Semantic: "what the picture says while the scheduler holds it back", Authority: liveAuthority,
|
||||
Artifact: "none observed",
|
||||
Reconstruction: "queued with its cause, or at least not started",
|
||||
Before: "held back: ceiling full, child never entered",
|
||||
After: "graph=" + drawn + " parent=" + told, Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func loneStatus(o Observation) string {
|
||||
for id, phases := range o.Progress {
|
||||
if ofLoneTask(id) && len(phases) > 0 {
|
||||
return phases[len(phases)-1]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// stoppedOwnershipRow is the invariant a stop has to leave behind: once the
|
||||
// turn is over, nothing it owned may still be open with a live owner. A record
|
||||
// left open is what a restart reads as an interruption, so a turn that merely
|
||||
// stopped waiting looks, afterwards, exactly like a process that died.
|
||||
func stoppedOwnershipRow(arm string, before Observation) row {
|
||||
if !cancelTaskArm(arm) {
|
||||
return row{
|
||||
Semantic: "work the stopped turn still owns", Authority: crossAuthority,
|
||||
Artifact: "<stem>.execution.jsonl", Reconstruction: "this arm stops nothing",
|
||||
Before: "—", After: "—", Verdict: verdictNotMeasured,
|
||||
}
|
||||
}
|
||||
reading := ""
|
||||
if seen := before.Progress[probeStopReached]; len(seen) > 0 {
|
||||
reading = seen[0]
|
||||
}
|
||||
entry, _ := loneEntry(before)
|
||||
open := entry.SettledAt.IsZero()
|
||||
turnOver := strings.Contains(reading, "turn=ended")
|
||||
// An open record implies a live owner: nothing else closes one. The child's
|
||||
// liveness rides in the reading as evidence, not as the test — a delegation
|
||||
// still waiting for a slot has no child and is just as much still owned.
|
||||
return row{
|
||||
Semantic: "work the stopped turn still owns", Authority: crossAuthority,
|
||||
Artifact: "<stem>.execution.jsonl",
|
||||
Reconstruction: "a turn that has ended owns nothing still open",
|
||||
Before: "the turn owned it when the stop was issued",
|
||||
After: orNone(reading) + " record=" + entryStanding(entry), Verdict: held(!(turnOver && open)),
|
||||
}
|
||||
}
|
||||
|
||||
func entryStanding(e execjournal.Entry) string {
|
||||
if e.SettledAt.IsZero() {
|
||||
return "still open"
|
||||
}
|
||||
return "settled"
|
||||
}
|
||||
|
||||
// startedTerminalRow refuses the combination neither layer may produce: a child
|
||||
// record for work the journal proves was never admitted. A state comparison
|
||||
// shows that only as "cancelled came back failed"; this names the cause, which
|
||||
// is a store asked to speak for an execution that never happened.
|
||||
func startedTerminalRow(before, after Observation) row {
|
||||
if startedAtDeath(before) {
|
||||
return row{
|
||||
Semantic: "a child record for work that never started", Authority: crossAuthority,
|
||||
Artifact: "subagents/<ref>.meta.json, <stem>.execution.jsonl",
|
||||
Reconstruction: "this delegation was granted a slot, so the store owes a record",
|
||||
Before: "—", After: "—", Verdict: verdictNotMeasured,
|
||||
}
|
||||
}
|
||||
kept := orNone(join(compact(childFacts(before), childFacts(after))))
|
||||
return row{
|
||||
Semantic: "a child record for work that never started", Authority: crossAuthority,
|
||||
Artifact: "subagents/<ref>.meta.json, <stem>.execution.jsonl",
|
||||
Reconstruction: "no slot was ever granted, so nothing ran and nothing owes a terminal",
|
||||
Before: "none", After: kept, Verdict: held(kept == "none"),
|
||||
}
|
||||
}
|
||||
|
||||
func compact(parts ...string) []string {
|
||||
var out []string
|
||||
for _, p := range parts {
|
||||
if strings.TrimSpace(p) != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fanOutControlRow is the positive control the queued arm carries with it: the
|
||||
// fleet that fills its ceiling runs through the same scheduler in the same
|
||||
// session, and records everything. What this arm reports missing is missing at
|
||||
// the entry point, not in the machinery.
|
||||
func fanOutControlRow(before Observation) row {
|
||||
var recorded []string
|
||||
for _, e := range before.Executions {
|
||||
if !ofLoneTask(e.ID) {
|
||||
recorded = append(recorded, e.ID+lifecycleOf(e))
|
||||
}
|
||||
}
|
||||
sort.Strings(recorded)
|
||||
verdict := verdictNotMeasured
|
||||
if len(recorded) > 0 {
|
||||
verdict = verdictPersisted
|
||||
}
|
||||
return row{
|
||||
Semantic: "a fan-out in the same session, same scheduler", Authority: journalAuthority,
|
||||
Artifact: "<stem>.execution.jsonl", Reconstruction: "control: the machinery this arm reports missing",
|
||||
Before: orNone(join(recorded)), After: "—", Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func lifecycleOf(e execjournal.Entry) string {
|
||||
marks := ""
|
||||
if e.Queued() {
|
||||
marks += " queued=" + orDash(e.Cause)
|
||||
}
|
||||
if e.Started() {
|
||||
marks += " started"
|
||||
}
|
||||
if !e.SettledAt.IsZero() {
|
||||
marks += " settled"
|
||||
}
|
||||
return marks
|
||||
}
|
||||
|
||||
// loneTaskArmInvalid guards each arm's premise: the state it is built to die in
|
||||
// has to have been reached, or its rows describe a different death.
|
||||
func loneTaskArmInvalid(arm string, before Observation) string {
|
||||
phases := strings.Join(before.Progress[taskCallID], "→")
|
||||
switch arm {
|
||||
case armTaskCompleted:
|
||||
if !strings.Contains(phases, "completed") && len(before.Children.Facts) == 0 {
|
||||
return "the delegation never completed, so this arm measures a different death"
|
||||
}
|
||||
case armTaskRunning:
|
||||
if !holdsState(before, string(agentgraph.StateRunning)) {
|
||||
return "nothing was executing when the process died"
|
||||
}
|
||||
case armTaskFgQueued:
|
||||
if entered := before.Progress[probeChildEntered]; len(entered) == 0 || entered[0] != "false" {
|
||||
return "the delegation's child ran, so the scheduler was not holding it back"
|
||||
}
|
||||
if !strings.Contains(fanOutControlRow(before).Before, "queued=slots") {
|
||||
return "no fan-out filled the ceiling, so nothing refused the delegation"
|
||||
}
|
||||
case armTaskBgQueued:
|
||||
if !strings.HasSuffix(phases, subagentQueued) {
|
||||
return "the delegation was not left queued for a slot: " + orNone(phases)
|
||||
}
|
||||
case armTaskBgRunning:
|
||||
if !strings.Contains(phases, subagentRunning) {
|
||||
return "the delegation never entered its job: " + orNone(phases)
|
||||
}
|
||||
}
|
||||
if cancelTaskArm(arm) {
|
||||
return stoppedArmInvalid(arm, before)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// stoppedArmInvalid holds a cancellation arm to the side of the boundary it was
|
||||
// built for. An arm that stopped the work somewhere else measures a different
|
||||
// ownership question and must say so rather than reporting the answer.
|
||||
func stoppedArmInvalid(arm string, before Observation) string {
|
||||
entry, ok := loneEntry(before)
|
||||
switch {
|
||||
case !ok:
|
||||
return "the delegation was never recorded, so there is no ownership to read"
|
||||
case beforeStartArm(arm) && entry.Started():
|
||||
return "the delegation was granted a slot before the stop, so this is not a pre-start cancellation"
|
||||
case !beforeStartArm(arm) && !entry.Started():
|
||||
return "the delegation never held a slot, so this is not a cancellation of work that ran"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func loneEntry(o Observation) (execjournal.Entry, bool) {
|
||||
for _, e := range o.Executions {
|
||||
if ofLoneTask(e.ID) {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return execjournal.Entry{}, false
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
// Process-boundary resurrection probe. It answers one question and modifies
|
||||
// nothing: after the process that built a session exits, what can a new process
|
||||
// still prove about that session's runtime semantics without asking a model?
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
phase := flag.String("phase", "", "internal: construct|successor|resume, spawned by the orchestrator")
|
||||
root := flag.String("root", "", "internal: this arm's root directory")
|
||||
armName := flag.String("arm", "", "internal: this arm's name")
|
||||
work := flag.String("work", "", "directory to hold arm roots; default is a temp dir")
|
||||
only := flag.String("only", "", "run a single arm by name")
|
||||
jsonOut := flag.String("json", "", "also write the machine-readable matrix here")
|
||||
keep := flag.Bool("keep", false, "keep arm roots on disk after the run")
|
||||
sabotage := flag.String("sabotage", "", "run under a deliberate defect the arm must fail on (ui-graph-mixed: publish|trajectory)")
|
||||
flag.Parse()
|
||||
if *sabotage != "" {
|
||||
// Carried to the children through the environment they already inherit,
|
||||
// so no phase has to learn about a flag it does not read.
|
||||
if err := os.Setenv(envProbeSabotage, *sabotage); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "runtime-resume:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
switch *phase {
|
||||
case "construct":
|
||||
err = runConstruct(*root, *armName)
|
||||
case "successor":
|
||||
err = runSuccessor(*root, *armName)
|
||||
case "resume":
|
||||
err = runResume(*root, *armName)
|
||||
case "":
|
||||
err = orchestrate(*work, *only, *jsonOut, *keep)
|
||||
default:
|
||||
err = fmt.Errorf("unknown phase %q", *phase)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "runtime-resume:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,584 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"reasonix/internal/contract/agentgraph"
|
||||
"reasonix/internal/contract/provider"
|
||||
"reasonix/internal/runtime/delegation"
|
||||
"reasonix/internal/safety/evidence"
|
||||
"reasonix/internal/session/control"
|
||||
"reasonix/internal/state/execjournal"
|
||||
"reasonix/internal/state/store"
|
||||
)
|
||||
|
||||
// Observation is everything one phase can read about a session without asking a
|
||||
// model. Both phases capture the same shape so the matrix is a comparison and
|
||||
// not two differently-shaped reports.
|
||||
type Observation struct {
|
||||
Phase string `json:"phase"`
|
||||
Arm string `json:"arm"`
|
||||
SessionPath string `json:"session_path"`
|
||||
SystemHash string `json:"system_hash"`
|
||||
SystemBytes int `json:"system_bytes"`
|
||||
// BootSystemHash is what this process's assembly composed, read before any
|
||||
// resume; SystemHash is the session's own leading row. Different questions.
|
||||
BootSystemHash string `json:"boot_system_hash"`
|
||||
BootSystemBytes int `json:"boot_system_bytes"`
|
||||
Transcript TranscriptObs `json:"transcript"`
|
||||
Goal GoalObs `json:"goal"`
|
||||
Todos []evidence.TodoItem `json:"todos"`
|
||||
Decisions []control.Decision `json:"decisions"`
|
||||
Context ContextObs `json:"context"`
|
||||
Sidecar SidecarObs `json:"sidecar"`
|
||||
View ViewObs `json:"view"`
|
||||
TodoNotes TodoNoteObs `json:"todo_notes"`
|
||||
Deferred DeferredObs `json:"deferred"`
|
||||
Obligation ObligationObs `json:"obligation"`
|
||||
// Interrupted is what the host derives from a barrier it found open and did
|
||||
// not open itself: evidence the wait happened, not a question to answer.
|
||||
Interrupted []control.InterruptedAdjudication `json:"interrupted"`
|
||||
// ModelSeesInterruption counts the interruption blocks the request carries,
|
||||
// which is a different surface from the derived fact behind them.
|
||||
ModelSeesInterruption int `json:"model_sees_interruption"`
|
||||
// Journal is how each barrier ended. A superseded edge is the durable
|
||||
// evidence that a turn received the interruption: nothing else writes one.
|
||||
Journal []control.AdjudicationEntry `json:"journal,omitempty"`
|
||||
Graph GraphObs `json:"graph"`
|
||||
// Children is the durable side of the same fan-out the graph draws. The
|
||||
// graph has no read surface after a restart, so without this a lost graph
|
||||
// and a lost delegation would be one indistinguishable row.
|
||||
Children ChildrenObs `json:"children"`
|
||||
// Listing is the same children through the read list_subagents performs.
|
||||
// It filters by workspace and walks lineage, so it can answer differently
|
||||
// from the one above — and it is the answer a model asking would be shown.
|
||||
Listing ListingObs `json:"listing"`
|
||||
// FanOutTurn is whether the dispatching turn is in this phase's transcript
|
||||
// at all. A turn is appended when it ends, so a process that dies inside one
|
||||
// leaves no record of the request, and a lost graph is the smaller half.
|
||||
FanOutTurn bool `json:"fan_out_turn"`
|
||||
// Executions is the durable record of delegations this turn opened, and
|
||||
// InterruptedExecutions the host's judgement over it: open, with no owner
|
||||
// here. Written before the work starts, so it outlives an unfinished turn.
|
||||
Executions []execjournal.Entry `json:"executions,omitempty"`
|
||||
InterruptedExecutions []execjournal.Entry `json:"interrupted_executions,omitempty"`
|
||||
// ModelSeesInterruptedExecution counts the blocks the next request carries
|
||||
// about them, which is a different surface from the fact behind them.
|
||||
ModelSeesInterruptedExecution int `json:"model_sees_interrupted_execution"`
|
||||
// WaitCauses counts the refusals the graph carries, and WaitSeries keeps
|
||||
// each node's causes in publication order. The fold keeps only the latest,
|
||||
// which cannot say whether a cause was reported once or replaced.
|
||||
WaitCauses map[string]int `json:"wait_causes,omitempty"`
|
||||
WaitSeries map[string][]string `json:"wait_series,omitempty"`
|
||||
// Progress is every parent-facing subagent status, per call. A background
|
||||
// delegation draws no node, so this is where its live evidence is.
|
||||
Progress map[string][]string `json:"progress,omitempty"`
|
||||
// SettledWorkers and QueuedWorkers are the transition arm's evidence that
|
||||
// capacity was actually freed while a refusal still stood.
|
||||
SettledWorkers int `json:"settled_workers"`
|
||||
QueuedWorkers []string `json:"queued_workers,omitempty"`
|
||||
Artifacts []ArtifactObs `json:"artifacts"`
|
||||
// UI is what a frontend attached to this process saw, in order. Only the one
|
||||
// arm that runs a client fills it; a value comparison cannot answer what it
|
||||
// asks, which is through which door each fact arrived.
|
||||
UI *UIObs `json:"ui,omitempty"`
|
||||
}
|
||||
|
||||
type TranscriptObs struct {
|
||||
Messages int `json:"messages"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
type GoalObs struct {
|
||||
Goal string `json:"goal"`
|
||||
Status string `json:"status"`
|
||||
TurnsUsed int `json:"turns_used"`
|
||||
}
|
||||
|
||||
// ContextObs is the live judgement: CheckpointState is "none" whenever the
|
||||
// projection did not survive validation, which is what the compaction arms are
|
||||
// actually asking about.
|
||||
type ContextObs struct {
|
||||
ProjectionVersion uint64 `json:"projection_version"`
|
||||
CheckpointState string `json:"checkpoint_state"`
|
||||
CanonicalTokens int `json:"canonical_tokens"`
|
||||
ProjectedTokens int `json:"projected_tokens"`
|
||||
Blocked bool `json:"blocked"`
|
||||
}
|
||||
|
||||
// SidecarObs is the stored claim, read off disk rather than through the agent,
|
||||
// so a projection the host refuses to use is still visible as present.
|
||||
type SidecarObs struct {
|
||||
Present bool `json:"present"`
|
||||
Messages int `json:"messages"`
|
||||
CoveredCount int `json:"covered_count"`
|
||||
CoveredPrefixHash string `json:"covered_prefix_hash"`
|
||||
ProjectionVersion uint64 `json:"projection_version"`
|
||||
TranscriptVersion uint64 `json:"transcript_version"`
|
||||
Generation uint64 `json:"generation"`
|
||||
Err string `json:"err,omitempty"`
|
||||
}
|
||||
|
||||
// ViewObs is the model-visible context, spliced the way ContextProjection
|
||||
// declares it: the stored body followed by canonical[CoveredCount:]. Markers
|
||||
// are what a fold can be held to — the scripted digest carries none, so a
|
||||
// marker that is gone was dropped, not summarised.
|
||||
type ViewObs struct {
|
||||
Messages int `json:"messages"`
|
||||
// Markers are the user turns still shown, Echoes the assistant replies. A
|
||||
// fold takes the reply first, so the two answer different questions.
|
||||
Markers []string `json:"markers"`
|
||||
Echoes []string `json:"echoes"`
|
||||
BodyMarkers []string `json:"body_markers"`
|
||||
BodyEchoes []string `json:"body_echoes"`
|
||||
SplicedFromTail int `json:"spliced_from_tail"`
|
||||
}
|
||||
|
||||
// TodoNoteObs locates the host's step-identity note inside the model-visible
|
||||
// view. BodyLen is where the frozen body ends, so an index below it says the
|
||||
// note was frozen ahead of the live tail rather than riding it.
|
||||
type TodoNoteObs struct {
|
||||
Count int `json:"count"`
|
||||
Indexes []int `json:"indexes"`
|
||||
IDs []string `json:"ids"`
|
||||
BodyLen int `json:"body_len"`
|
||||
ViewLen int `json:"view_len"`
|
||||
HostIDs []string `json:"host_ids"`
|
||||
// ReadableInHistory says the conversation still shows every current id, so
|
||||
// no note is owed. Without it a count of zero cannot be told apart from a
|
||||
// lost identity, and the gate would demand a note nobody needs.
|
||||
ReadableInHistory bool `json:"readable_in_history"`
|
||||
}
|
||||
|
||||
// todoNoteMarker is the opening of todoIdentityNote. The probe reads the view
|
||||
// the way a model would, so it matches the text rather than a type.
|
||||
const todoNoteMarker = "Host task state."
|
||||
|
||||
var stepIDPattern = regexp.MustCompile(`probe_step_\d+`)
|
||||
|
||||
func todoNoteObs(view, history []provider.Message, todos []evidence.TodoItem) TodoNoteObs {
|
||||
out := TodoNoteObs{BodyLen: len(history), ViewLen: len(view)}
|
||||
for _, t := range todos {
|
||||
if t.StepID != "" {
|
||||
out.HostIDs = append(out.HostIDs, t.StepID)
|
||||
}
|
||||
}
|
||||
out.ReadableInHistory = len(out.HostIDs) > 0
|
||||
for _, id := range out.HostIDs {
|
||||
if !mentionsID(history, id) {
|
||||
out.ReadableInHistory = false
|
||||
}
|
||||
}
|
||||
for i, m := range view {
|
||||
if !strings.Contains(m.Content, todoNoteMarker) {
|
||||
continue
|
||||
}
|
||||
out.Count++
|
||||
out.Indexes = append(out.Indexes, i)
|
||||
out.IDs = append(out.IDs, stepIDPattern.FindAllString(m.Content, -1)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// DeferredObs is the effect a pending decision is holding back. Whether the
|
||||
// decision itself survives is one question; whether what it blocked stayed
|
||||
// blocked is the one that cannot be answered by a design choice.
|
||||
type DeferredObs struct {
|
||||
MarkerPath string `json:"marker_path"`
|
||||
Executed bool `json:"executed"`
|
||||
}
|
||||
|
||||
func deferredObs(workspace string) DeferredObs {
|
||||
path := filepath.Join(workspace, deferredEffect)
|
||||
_, err := os.Stat(path)
|
||||
return DeferredObs{MarkerPath: deferredEffect, Executed: err == nil}
|
||||
}
|
||||
|
||||
// ObligationObs is what the transcript itself still says about a turn that
|
||||
// never finished: a tool call with no result is a claim the host made and did
|
||||
// not settle, and an interruption record is the host saying so outright. Either
|
||||
// tells a lost decision apart from a silently dropped one.
|
||||
type ObligationObs struct {
|
||||
UnansweredCalls []string `json:"unanswered_calls"`
|
||||
InterruptionMarked int `json:"interruption_marked"`
|
||||
}
|
||||
|
||||
func obligationObs(msgs []provider.Message) ObligationObs {
|
||||
answered := map[string]bool{}
|
||||
var out ObligationObs
|
||||
for _, m := range msgs {
|
||||
if m.ToolCallID != "" {
|
||||
answered[m.ToolCallID] = true
|
||||
}
|
||||
if m.InterruptedTurn != nil {
|
||||
out.InterruptionMarked++
|
||||
}
|
||||
}
|
||||
for _, m := range msgs {
|
||||
for _, call := range m.ToolCalls {
|
||||
if call.ID != "" && call.ID != provider.LocalOnlyToolID && !answered[call.ID] {
|
||||
out.UnansweredCalls = append(out.UnansweredCalls, call.Name+":"+call.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type GraphObs struct {
|
||||
Deltas int `json:"deltas"`
|
||||
Nodes []agentgraph.Node `json:"nodes,omitempty"`
|
||||
Edges []agentgraph.Edge `json:"edges,omitempty"`
|
||||
Grants map[string]string `json:"grants,omitempty"`
|
||||
Waits map[string]string `json:"waits,omitempty"`
|
||||
}
|
||||
|
||||
// ChildFact is one delegated run as the durable store records it, which is a
|
||||
// different set of facts from the node the graph drew for it. Whatever the
|
||||
// graph carried and this does not is a semantic no reconstruction can recover.
|
||||
type ChildFact struct {
|
||||
Ref string `json:"ref"`
|
||||
Status string `json:"status"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
// ExecutionID is which execution the record belongs to; ParentToolCallID is
|
||||
// the provider-visible call it descends from. They are separate answers, and
|
||||
// a host-started run has only the first.
|
||||
ExecutionID string `json:"executionId,omitempty"`
|
||||
ParentToolCallID string `json:"parentToolCallId,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Effort string `json:"effort,omitempty"`
|
||||
}
|
||||
|
||||
// ChildrenObs is every child the store still owns for this parent. It is the
|
||||
// only durable execution evidence a restart has been shown to read, so a graph
|
||||
// classification that ignores it would report a loss the host can in fact
|
||||
// still speak to.
|
||||
type ChildrenObs struct {
|
||||
Facts []ChildFact `json:"facts,omitempty"`
|
||||
Err string `json:"err,omitempty"`
|
||||
}
|
||||
|
||||
// childrenObs reads the store the way a restart does: by parent session id,
|
||||
// which is the transcript's stem. A session with no path owns nothing.
|
||||
func childrenObs(root armRoot, sessionPath string) ChildrenObs {
|
||||
if strings.TrimSpace(sessionPath) == "" {
|
||||
return ChildrenObs{}
|
||||
}
|
||||
parent := strings.TrimSuffix(filepath.Base(sessionPath), ".jsonl")
|
||||
artifacts, err := sessionstore.ListSubagentsByParent(root.Sessions, parent)
|
||||
if err != nil {
|
||||
return ChildrenObs{Err: err.Error()}
|
||||
}
|
||||
out := ChildrenObs{}
|
||||
for _, a := range artifacts {
|
||||
out.Facts = append(out.Facts, ChildFact{
|
||||
Ref: a.Ref, Status: string(a.Meta.Status), Kind: a.Meta.Kind, Name: a.Meta.Name,
|
||||
ExecutionID: a.Meta.ExecutionID,
|
||||
ParentToolCallID: a.Meta.ParentToolCallID, Model: a.Meta.Model, Effort: a.Meta.Effort,
|
||||
})
|
||||
}
|
||||
sort.Slice(out.Facts, func(i, j int) bool { return out.Facts[i].Ref < out.Facts[j].Ref })
|
||||
return out
|
||||
}
|
||||
|
||||
// ListingRow is one row list_subagents would print. The tool call itself needs
|
||||
// a registry this probe has no way to build, so the read behind it is what is
|
||||
// exercised; the formatting adds no fact the row does not already carry.
|
||||
type ListingRow struct {
|
||||
Ref string `json:"ref"`
|
||||
Status string `json:"status"`
|
||||
From string `json:"from,omitempty"`
|
||||
}
|
||||
|
||||
// ListingObs is what the recovery surface would show for this conversation.
|
||||
type ListingObs struct {
|
||||
Rows []ListingRow `json:"rows,omitempty"`
|
||||
Err string `json:"err,omitempty"`
|
||||
}
|
||||
|
||||
// listingObs reads the store the way the tool does: by the parent session the
|
||||
// child recorded, filtered to this workspace. The workspace filter is why this
|
||||
// cannot be folded into childrenObs — a root that does not match drops every
|
||||
// row, and a reader must be able to see that happen rather than read it as an
|
||||
// empty store.
|
||||
func listingObs(root armRoot, sessionPath string) ListingObs {
|
||||
if strings.TrimSpace(sessionPath) == "" {
|
||||
return ListingObs{}
|
||||
}
|
||||
store := delegation.NewSubagentStore(filepath.Join(root.Sessions, "subagents"))
|
||||
parent := strings.TrimSuffix(filepath.Base(sessionPath), ".jsonl")
|
||||
artifacts, err := store.ListForParent(parent, root.Workspace)
|
||||
if err != nil {
|
||||
return ListingObs{Err: err.Error()}
|
||||
}
|
||||
out := ListingObs{}
|
||||
for _, a := range artifacts {
|
||||
out.Rows = append(out.Rows, ListingRow{
|
||||
Ref: a.Ref, Status: string(a.Meta.Status), From: a.Meta.ParentToolCallID,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type ArtifactObs struct {
|
||||
Name string `json:"name"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
}
|
||||
|
||||
func capture(phase, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, root armRoot) Observation {
|
||||
path := ctrl.SessionPath()
|
||||
history := ctrl.History()
|
||||
system := systemText(history)
|
||||
snap := ctrl.ContextMaintenanceSnapshot()
|
||||
st, loaded, loadErr := sessionstore.LoadCompactionState(path)
|
||||
view, viewMsgs := viewObs(st, loaded, history)
|
||||
graph, deltas := sink.snapshot()
|
||||
return Observation{
|
||||
Phase: phase,
|
||||
Arm: arm,
|
||||
SessionPath: path,
|
||||
SystemHash: shortSum(system),
|
||||
SystemBytes: len(system),
|
||||
BootSystemHash: shortSum(bootSystem),
|
||||
BootSystemBytes: len(bootSystem),
|
||||
Transcript: TranscriptObs{Messages: len(history), Digest: transcriptDigest(history)},
|
||||
Goal: GoalObs{Goal: ctrl.Goal(), Status: ctrl.GoalStatus(), TurnsUsed: ctrl.GoalRuntime().TurnsUsed},
|
||||
Todos: ctrl.Todos(),
|
||||
Decisions: ctrl.Decisions(),
|
||||
Context: ContextObs{
|
||||
ProjectionVersion: snap.ProjectionVersion,
|
||||
CheckpointState: snap.CheckpointState,
|
||||
CanonicalTokens: snap.CanonicalTokens,
|
||||
ProjectedTokens: snap.ProjectedTokens,
|
||||
Blocked: snap.Blocked,
|
||||
},
|
||||
Sidecar: sidecarObs(st, loaded, loadErr),
|
||||
View: view,
|
||||
// The note is derived for a request and stored nowhere, so the contracts
|
||||
// about it are read off the request rather than off the stored view.
|
||||
TodoNotes: todoNoteObs(ctrl.ModelVisibleMessages(), viewMsgs, ctrl.Todos()),
|
||||
Graph: graphObs(graph, deltas),
|
||||
WaitCauses: waitCauseCounts(graph),
|
||||
WaitSeries: sink.waitSeries(),
|
||||
Progress: sink.phaseSeries(),
|
||||
SettledWorkers: settledWorkers(graph),
|
||||
QueuedWorkers: queuedWorkers(graph),
|
||||
Children: childrenObs(root, path),
|
||||
Listing: listingObs(root, path),
|
||||
FanOutTurn: fanOutTurnRecorded(history),
|
||||
Executions: control.ExecutionHistory(path),
|
||||
InterruptedExecutions: ctrl.InterruptedExecutions(),
|
||||
ModelSeesInterruptedExecution: countBlocks(
|
||||
ctrl.ModelVisibleMessages(), "<interrupted-execution>"),
|
||||
Deferred: deferredObs(root.Workspace),
|
||||
Obligation: obligationObs(history),
|
||||
Interrupted: ctrl.InterruptedAdjudications(),
|
||||
Journal: control.AdjudicationHistory(path),
|
||||
ModelSeesInterruption: countBlocks(ctrl.ModelVisibleMessages(), "<interrupted-adjudication>"),
|
||||
Artifacts: readArtifacts(path),
|
||||
}
|
||||
}
|
||||
|
||||
// bootSystemText is what the freshly built assembly composed, read before a
|
||||
// resume replaces the session under it.
|
||||
func bootSystemText(ctrl *control.Controller) string { return systemText(ctrl.History()) }
|
||||
|
||||
func systemText(msgs []provider.Message) string {
|
||||
var b strings.Builder
|
||||
for _, m := range msgs {
|
||||
if m.Role == provider.RoleSystem {
|
||||
b.WriteString(m.Content)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// transcriptDigest covers only what reaches a provider, so a local-only marker
|
||||
// appended by either process cannot read as a changed conversation.
|
||||
func transcriptDigest(msgs []provider.Message) string {
|
||||
h := sha256.New()
|
||||
for _, m := range provider.ModelMessages(msgs) {
|
||||
b, _ := json.Marshal(struct {
|
||||
R, C, N string
|
||||
}{string(m.Role), m.Content, m.Name})
|
||||
h.Write(b)
|
||||
h.Write([]byte{'\n'})
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))[:16]
|
||||
}
|
||||
|
||||
func shortSum(s string) string {
|
||||
sum := sha256.Sum256([]byte(s))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
func sidecarObs(st sessionstore.CompactionState, ok bool, err error) SidecarObs {
|
||||
out := SidecarObs{Present: ok}
|
||||
if err != nil {
|
||||
out.Err = err.Error()
|
||||
return out
|
||||
}
|
||||
if !ok {
|
||||
return out
|
||||
}
|
||||
out.Messages = len(st.Projection.Messages)
|
||||
out.CoveredCount = st.Projection.CoveredCount
|
||||
out.CoveredPrefixHash = st.Projection.CoveredPrefixHash
|
||||
out.ProjectionVersion = st.Projection.ProjectionVersion
|
||||
out.TranscriptVersion = st.TranscriptVersion
|
||||
out.Generation = st.Generation
|
||||
return out
|
||||
}
|
||||
|
||||
// viewObs splices the model-visible context. With no usable projection the
|
||||
// view is the canonical transcript, which is what the host would send.
|
||||
func viewObs(st sessionstore.CompactionState, ok bool, canonical []provider.Message) (ViewObs, []provider.Message) {
|
||||
var out ViewObs
|
||||
if !ok || len(st.Projection.Messages) == 0 {
|
||||
out.Messages = len(canonical)
|
||||
out.Markers = markersIn(contents(canonical), markerPattern)
|
||||
out.Echoes = markersIn(contents(canonical), echoPattern)
|
||||
return out, canonical
|
||||
}
|
||||
body := st.Projection.Messages
|
||||
out.BodyMarkers = markersIn(contents(body), markerPattern)
|
||||
out.BodyEchoes = markersIn(contents(body), echoPattern)
|
||||
view := append([]provider.Message(nil), body...)
|
||||
if n := st.Projection.CoveredCount; n >= 0 && n < len(canonical) {
|
||||
view = append(view, canonical[n:]...)
|
||||
out.SplicedFromTail = len(canonical) - n
|
||||
}
|
||||
out.Messages = len(view)
|
||||
out.Markers = markersIn(contents(view), markerPattern)
|
||||
out.Echoes = markersIn(contents(view), echoPattern)
|
||||
return out, view
|
||||
}
|
||||
|
||||
// mentionsID reads an id the way the host's own visibility check does: text or
|
||||
// tool arguments, anywhere in the messages the request carries.
|
||||
func mentionsID(msgs []provider.Message, id string) bool {
|
||||
for _, m := range msgs {
|
||||
if strings.Contains(m.Content, id) {
|
||||
return true
|
||||
}
|
||||
for _, call := range m.ToolCalls {
|
||||
if strings.Contains(call.Arguments, id) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func countBlocks(msgs []provider.Message, marker string) int {
|
||||
n := 0
|
||||
for _, m := range msgs {
|
||||
if strings.Contains(m.Content, marker) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func contents(msgs []provider.Message) []string {
|
||||
out := make([]string, 0, len(msgs))
|
||||
for _, m := range msgs {
|
||||
out = append(out, m.Content)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// waitCauseCounts is how many nodes the graph shows under each refusal. It
|
||||
// counts nodes rather than reports: a node keeps its cause after it starts, so
|
||||
// this answers what was refused, not what is refused now.
|
||||
func waitCauseCounts(g agentgraph.Graph) map[string]int {
|
||||
out := map[string]int{}
|
||||
for _, n := range g.Nodes {
|
||||
if n.Wait != "" {
|
||||
out[string(n.Wait)]++
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// queuedWorkers are the items still refused admission, in a stable order.
|
||||
func queuedWorkers(g agentgraph.Graph) []string {
|
||||
var out []string
|
||||
for _, n := range g.Nodes {
|
||||
if n.Kind != agentgraph.KindGroup && n.State == agentgraph.StateQueued {
|
||||
out = append(out, n.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func graphObs(g agentgraph.Graph, deltas int) GraphObs {
|
||||
out := GraphObs{Deltas: deltas}
|
||||
if len(g.Nodes) == 0 {
|
||||
return out
|
||||
}
|
||||
out.Nodes = g.Nodes
|
||||
out.Edges = g.Edges
|
||||
out.Grants = map[string]string{}
|
||||
out.Waits = map[string]string{}
|
||||
for _, n := range g.Nodes {
|
||||
if n.Grant != "" {
|
||||
out.Grants[n.ID] = string(n.Grant)
|
||||
}
|
||||
if n.Wait != "" {
|
||||
out.Waits[n.ID] = string(n.Wait)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// readArtifacts lists the session sidecars that exist, which is the evidence
|
||||
// behind every "persisted" claim in the matrix.
|
||||
func readArtifacts(sessionPath string) []ArtifactObs {
|
||||
if sessionPath == "" {
|
||||
return nil
|
||||
}
|
||||
paths := append([]string{sessionPath}, store.SessionSidecarFiles(sessionPath)...)
|
||||
var out []ArtifactObs
|
||||
for _, p := range paths {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, ArtifactObs{Name: filepath.Base(p), Bytes: info.Size()})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func writeObservation(root armRoot, obs Observation) error {
|
||||
b, err := json.MarshalIndent(obs, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(filepath.Join(root.Dir, "obs-"+obs.Phase+".json"), b, 0o644)
|
||||
}
|
||||
|
||||
func readObservation(root armRoot, phase string) (Observation, error) {
|
||||
var obs Observation
|
||||
b, err := os.ReadFile(filepath.Join(root.Dir, "obs-"+phase+".json"))
|
||||
if err != nil {
|
||||
return obs, err
|
||||
}
|
||||
return obs, json.Unmarshal(b, &obs)
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"reasonix/internal/state/sessionstore"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/runtime/agent"
|
||||
"reasonix/internal/session/control"
|
||||
"reasonix/internal/state/checkpoint"
|
||||
)
|
||||
|
||||
// probeTurns is enough conversation for a fold to have a region to work on:
|
||||
// the system row and the first user turn are pinned, and the recent tail is
|
||||
// kept verbatim, so a short session has nothing foldable between them.
|
||||
const probeTurns = 8
|
||||
|
||||
// refoldTurns grows the transcript past the first fold by less than the fold
|
||||
// keeps verbatim, so the second fold has to reach into the stored body.
|
||||
const refoldTurns = 3
|
||||
|
||||
// runTurns drives count turns, each carrying its own marker, and returns the
|
||||
// next turn number.
|
||||
func runTurns(ctx context.Context, ctrl *control.Controller, from, count int) (int, error) {
|
||||
for i := range count {
|
||||
n := from + i + 1
|
||||
prompt := fmt.Sprintf("%s Probe turn %d: record the task list and keep working.", marker(n), n)
|
||||
if err := ctrl.Run(ctx, prompt); err != nil {
|
||||
return n, fmt.Errorf("turn %d: %w", n, err)
|
||||
}
|
||||
}
|
||||
return from + count, nil
|
||||
}
|
||||
|
||||
// openDecisionAndDie leaves a question open and kills the process while the
|
||||
// turn is still parked on it. It exits rather than returning: a clean shutdown
|
||||
// would resolve or cancel the question, which is the one thing a process that
|
||||
// dies mid-decision does not do.
|
||||
func openDecisionAndDie(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink) error {
|
||||
// The gate a person answers through, wired the way chat and desktop wire
|
||||
// it. Without it the ask tool decides for itself and nothing ever blocks.
|
||||
ctrl.EnableInteractiveApproval()
|
||||
go func() { _ = ctrl.Run(ctx, marker(99)+" "+askSentinel+": ask before writing.") }()
|
||||
if err := waitForAsk(ctrl); err != nil {
|
||||
return err
|
||||
}
|
||||
obs := capture("construct", arm, bootSystem, ctrl, sink, root)
|
||||
if obs.Deferred.Executed {
|
||||
return errUnexpected("a write still held back by the open question", obs.Deferred.MarkerPath)
|
||||
}
|
||||
if err := writeObservation(root, obs); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Exit(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForAsk blocks until the host reports a question waiting on a person.
|
||||
func waitForAsk(ctrl *control.Controller) error {
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
for _, d := range ctrl.Decisions() {
|
||||
if d.Kind == control.DecisionAsk {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return errUnexpected("a question waiting on a person", ctrl.Decisions())
|
||||
}
|
||||
|
||||
// rewindTurn drives the rewind a person drives, recording the projection
|
||||
// before it: the tail arm takes back the turn appended after the fold, the
|
||||
// covered arm lands below the fold boundary and must lose the projection
|
||||
// whatever coverage says.
|
||||
func rewindTurn(root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink) error {
|
||||
points := ctrl.Checkpoints()
|
||||
if len(points) == 0 {
|
||||
return errUnexpected("a checkpoint to rewind to", 0)
|
||||
}
|
||||
target := points[len(points)-1].Turn
|
||||
if arm == armCoveredRewind {
|
||||
var err error
|
||||
if target, err = turnBelowFold(ctrl, points); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := writeObservation(root, capture(extraPhase(arm), arm, bootSystem, ctrl, sink, root)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctrl.Rewind(target, control.RewindConversation); err != nil {
|
||||
return fmt.Errorf("rewind: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// turnBelowFold picks the checkpoint whose boundary sits just below the fold,
|
||||
// read from the sidecar rather than assumed: a control aiming at a fixed turn
|
||||
// stops removing folded history the moment coverage moves, and says nothing.
|
||||
// A boundary at the first message is skipped — a session left holding only a
|
||||
// system row is never persisted, so the restart would read the arm's work back.
|
||||
func turnBelowFold(ctrl *control.Controller, points []checkpoint.Meta) (int, error) {
|
||||
covered := 0
|
||||
if st, ok, _ := sessionstore.LoadCompactionState(ctrl.SessionPath()); ok {
|
||||
covered = st.Projection.CoveredCount
|
||||
}
|
||||
target := -1
|
||||
for _, p := range points {
|
||||
plan, err := ctrl.PrepareRewind(p.Turn, control.RewindConversation)
|
||||
if err != nil || !plan.HasBoundary {
|
||||
continue
|
||||
}
|
||||
if plan.BoundaryIndex > 1 && plan.BoundaryIndex < covered {
|
||||
target = p.Turn
|
||||
}
|
||||
}
|
||||
if target < 0 {
|
||||
return 0, errUnexpected("a checkpoint below the fold boundary", covered)
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
|
||||
// extraPhase names the in-process observation an arm records before its
|
||||
// process exits, empty when it records none.
|
||||
func extraPhase(arm string) string {
|
||||
switch arm {
|
||||
case armRefoldIntoBody:
|
||||
return "prefold"
|
||||
case armTailRewind, armCoveredRewind:
|
||||
return "prerewind"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
const probeGoal = "Prove which runtime semantics survive an OS process boundary."
|
||||
|
||||
// armConstruct hands the arm to the construct path it needs, reporting whether
|
||||
// one took over. Each of them records its own observation and exits inside the
|
||||
// turn, so a caller that sees taken has nothing left to do but report the error.
|
||||
func armConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) (bool, error) {
|
||||
switch {
|
||||
case arm == armOpenDecision || successorArm(arm):
|
||||
return true, openDecisionAndDie(ctx, root, arm, bootSystem, ctrl, sink)
|
||||
case deriveArm(arm):
|
||||
return true, runDeriveConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
|
||||
case terminalArm(arm):
|
||||
return true, runTerminalConstruct(ctx, root, arm, bootSystem, ctrl, sink, turn)
|
||||
case schedulerWaitArm(arm):
|
||||
return true, runSchedulerWaitConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
|
||||
case lineageArm(arm):
|
||||
return true, runLineageConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
|
||||
case cancelRoutingArm(arm):
|
||||
return true, runCancelRoutingConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
|
||||
case loneTaskArm(arm):
|
||||
return true, runLoneTaskConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
|
||||
case activeStoreArm(arm):
|
||||
return true, runActiveStoreConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
|
||||
case skillArm(arm):
|
||||
return true, runSkillClassConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
|
||||
case ephemeralArm(arm):
|
||||
return true, runEphemeralConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
|
||||
case hostExecArm(arm):
|
||||
return true, runHostExecConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn)
|
||||
case graphArm(arm) || uiArm(arm):
|
||||
return true, runFanOutConstruct(ctx, root, arm, bootSystem, ctrl, sink, turn)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// runConstruct establishes host state through the same calls a frontend makes,
|
||||
// records what this process can see, and returns so the process can exit. It
|
||||
// never asserts: an arm that failed to establish something is reported as not
|
||||
// measured, which is the one reading a missing "before" supports.
|
||||
func runConstruct(dir, arm string) error {
|
||||
root := rootFor(dir)
|
||||
if err := root.create(); err != nil {
|
||||
return err
|
||||
}
|
||||
if skillArm(arm) || arm == armEphemeralSkill || hostExecArm(arm) {
|
||||
if err := writeProbeSkill(root.Workspace); err != nil {
|
||||
return fmt.Errorf("probe skill: %w", err)
|
||||
}
|
||||
}
|
||||
if total, writers := schedulerLimits(arm); total > 0 {
|
||||
if err := root.writeProjectConfig(total, writers); err != nil {
|
||||
return fmt.Errorf("project config: %w", err)
|
||||
}
|
||||
}
|
||||
ctx := context.Background()
|
||||
sink := &graphSink{}
|
||||
bus, into := uiBus(arm, sink)
|
||||
ctrl, prov, err := buildRuntime(ctx, root, arm, into)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build: %w", err)
|
||||
}
|
||||
defer ctrl.Close()
|
||||
// The process that dies is a window in production, and a window records what
|
||||
// it served. Without that the trajectory this session leaves behind holds
|
||||
// none of its own frames.
|
||||
recordLikeAWindow(ctrl, bus)
|
||||
|
||||
bootSystem := bootSystemText(ctrl)
|
||||
ctrl.EnsureSessionPath()
|
||||
if ctrl.SessionPath() == "" {
|
||||
return errUnexpected("a session path", "")
|
||||
}
|
||||
turn, err := runTurns(ctx, ctrl, 0, probeTurns)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctrl.SetGoalDurable(probeGoal); err != nil {
|
||||
return fmt.Errorf("goal: %w", err)
|
||||
}
|
||||
if _, err := ctrl.Compact(ctx, agent.CompactRequest{Instructions: "Fold the probe turn."}); err != nil {
|
||||
return fmt.Errorf("compact: %w", err)
|
||||
}
|
||||
if taken, err := armConstruct(ctx, root, arm, bootSystem, ctrl, sink, prov, turn); taken {
|
||||
return err
|
||||
}
|
||||
if arm == armTodoIdentity {
|
||||
// Identity A is already written and folded away. Grow, replace the list
|
||||
// with identity B, then fold again: the second fold is what decides
|
||||
// whether the model is left holding one current identity or two.
|
||||
if turn, err = runTurns(ctx, ctrl, turn, refoldTurns); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctrl.Run(ctx, fmt.Sprintf("%s %s: replace the task list.", marker(turn+1), retodoSentinel)); err != nil {
|
||||
return fmt.Errorf("retodo turn: %w", err)
|
||||
}
|
||||
// Grow well past the rewrite so the fold takes the new ids out of view.
|
||||
// A scenario that leaves them readable asks nothing of the note.
|
||||
if _, err = runTurns(ctx, ctrl, turn+1, probeTurns); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ctrl.Compact(ctx, agent.CompactRequest{Instructions: "Fold again, after the identity moved."}); err != nil {
|
||||
return fmt.Errorf("refold: %w", err)
|
||||
}
|
||||
} else if arm == armRefoldIntoBody {
|
||||
// Grow past the fold, record the view a second fold will operate on,
|
||||
// then fold again: its region reaches back into the stored body, which
|
||||
// has no canonical counterpart to map a new boundary onto.
|
||||
if _, err = runTurns(ctx, ctrl, turn, refoldTurns); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeObservation(root, capture(extraPhase(arm), arm, bootSystem, ctrl, sink, root)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ctrl.Compact(ctx, agent.CompactRequest{Instructions: "Fold again, into the stored body."}); err != nil {
|
||||
return fmt.Errorf("refold: %w", err)
|
||||
}
|
||||
} else if appendsAfterFold(arm) {
|
||||
if _, err = runTurns(ctx, ctrl, turn, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if arm == armTailRewind || arm == armCoveredRewind {
|
||||
if err := rewindTurn(root, arm, bootSystem, ctrl, sink); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return writeObservation(root, capture("construct", arm, bootSystem, ctrl, sink, root))
|
||||
}
|
||||
|
||||
// successorArm reports whether this arm starts from an interrupted barrier.
|
||||
func successorArm(arm string) bool {
|
||||
_, ok := successorTurn(arm)
|
||||
return ok
|
||||
}
|
||||
|
||||
// runSuccessor is the middle process: it inherits the interruption and does
|
||||
// what a person would do next — nothing, unrelated work, or something that
|
||||
// reads like an answer to the dead question. It runs a model on purpose; the
|
||||
// phase that judges what the host knows still does not.
|
||||
func runSuccessor(dir, arm string) error {
|
||||
root := rootFor(dir)
|
||||
before, err := readObservation(root, "construct")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx := context.Background()
|
||||
sink := &graphSink{}
|
||||
ctrl, _, err := buildRuntime(ctx, root, arm, sink)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build: %w", err)
|
||||
}
|
||||
defer ctrl.Close()
|
||||
bootSystem := bootSystemText(ctrl)
|
||||
session, err := sessionstore.LoadSession(before.SessionPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load session: %w", err)
|
||||
}
|
||||
if err := ctrl.Resume(session, before.SessionPath); err != nil {
|
||||
return fmt.Errorf("resume: %w", err)
|
||||
}
|
||||
if turn, _ := successorTurn(arm); turn != "" {
|
||||
if err := ctrl.Run(ctx, turn); err != nil {
|
||||
return fmt.Errorf("successor turn: %w", err)
|
||||
}
|
||||
}
|
||||
return writeObservation(root, capture("successor", arm, bootSystem, ctrl, sink, root))
|
||||
}
|
||||
|
||||
// runResume boots a second time against the same roots and reads host state.
|
||||
// It deliberately never calls Run: the question is what the host can prove
|
||||
// before a provider call, and a model that reconstructs a lost identity from
|
||||
// the transcript would turn a real loss into a passing row.
|
||||
func runResume(dir, arm string) error {
|
||||
root := rootFor(dir)
|
||||
before, err := readObservation(root, "construct")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read construct observation: %w", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
sink := &graphSink{}
|
||||
bus, into := uiBus(arm, sink)
|
||||
ctrl, _, err := buildRuntime(ctx, root, arm, into)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build: %w", err)
|
||||
}
|
||||
defer ctrl.Close()
|
||||
|
||||
bootSystem := bootSystemText(ctrl)
|
||||
session, err := sessionstore.LoadSession(before.SessionPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load session: %w", err)
|
||||
}
|
||||
if err := ctrl.Resume(session, before.SessionPath); err != nil {
|
||||
return fmt.Errorf("resume: %w", err)
|
||||
}
|
||||
obs := capture("resume", arm, bootSystem, ctrl, sink, root)
|
||||
if uiArm(arm) {
|
||||
ui := observeUI(root, ctrl, bus, obs)
|
||||
obs.UI = &ui
|
||||
}
|
||||
return writeObservation(root, obs)
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A fold may only remove a run of markers from the middle: the pinned head
|
||||
// stays, the retained tail stays, and everything between them becomes the
|
||||
// digest. Any other shape means the view lost something no fold folded.
|
||||
const (
|
||||
verdictContiguous = "contiguous"
|
||||
verdictHole = "HOLE"
|
||||
markPrefix = "PROBE-MARK-"
|
||||
echoPrefix = "PROBE-ECHO-"
|
||||
)
|
||||
|
||||
// refoldRows report what the second fold did. They are their own set because
|
||||
// the comparison is inside one process — the view before the refold against
|
||||
// the view after it — not across the process boundary.
|
||||
func refoldRows(prefold, postfold, resumed Observation, total int) []row {
|
||||
return []row{
|
||||
coverageRow("fold 1: stored body and coverage", prefold),
|
||||
coverageRow("fold 2: stored body and coverage", postfold),
|
||||
continuityRow("assistant work visible after fold 1", prefold, total),
|
||||
continuityRow("assistant work visible after fold 2", postfold, total),
|
||||
continuityRow("assistant work visible after restart", resumed, total),
|
||||
turnsRow("user turns visible after fold 2", postfold),
|
||||
}
|
||||
}
|
||||
|
||||
func coverageRow(semantic string, o Observation) row {
|
||||
return row{
|
||||
Semantic: semantic, Authority: "compaction sidecar", Artifact: "<stem>.context.json",
|
||||
Reconstruction: "projection.Messages + canonical[CoveredCount:]",
|
||||
Before: fmt.Sprintf("canonical=%d", o.Transcript.Messages),
|
||||
After: fmt.Sprintf("body=%d covered=%d spliced=+%d view=%d",
|
||||
o.Sidecar.Messages, o.Sidecar.CoveredCount, o.View.SplicedFromTail, o.View.Messages),
|
||||
Verdict: verdictStable,
|
||||
}
|
||||
}
|
||||
|
||||
// continuityRow judges the assistant side: a fold replaces one run of replies
|
||||
// with a digest, so anything but a single surviving run is a loss no fold made.
|
||||
func continuityRow(semantic string, o Observation, total int) row {
|
||||
shape := markerShape(o.View.Echoes, echoPrefix)
|
||||
verdict := verdictContiguous
|
||||
if reason := markerHole(o.View.Echoes, echoPrefix, total); reason != "" {
|
||||
verdict, shape = verdictHole, shape+" — "+reason
|
||||
}
|
||||
return row{
|
||||
Semantic: semantic, Authority: "the spliced model-visible view",
|
||||
Artifact: "none: derived", Reconstruction: "assistant replies surviving in the view",
|
||||
Before: fmt.Sprintf("emitted 1-%d", total), After: shape, Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
// turnsRow reports the user side beside it: the retention budget keeps those
|
||||
// verbatim, which is why a one-sided tag could not see a dropped reply.
|
||||
func turnsRow(semantic string, o Observation) row {
|
||||
return row{
|
||||
Semantic: semantic, Authority: "the spliced model-visible view",
|
||||
Artifact: "none: derived", Reconstruction: "user turns surviving in the view",
|
||||
Before: "kept by the retention budget",
|
||||
After: markerShape(o.View.Markers, markPrefix), Verdict: verdictStable,
|
||||
}
|
||||
}
|
||||
|
||||
// pad renders a tag number the way the scripted turns emit it.
|
||||
func pad(n int) string { return fmt.Sprintf("%03d", n) }
|
||||
|
||||
func markerNumbers(markers []string, prefix string) []int {
|
||||
seen := map[int]bool{}
|
||||
var out []int
|
||||
for _, m := range markers {
|
||||
n, err := strconv.Atoi(strings.TrimPrefix(m, prefix))
|
||||
if err != nil || seen[n] {
|
||||
continue
|
||||
}
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Ints(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// markerShape renders the surviving markers as runs, which is the form the
|
||||
// judgement is about: "1, 9-12" says a fold took 2-8.
|
||||
func markerShape(markers []string, prefix string) string {
|
||||
nums := markerNumbers(markers, prefix)
|
||||
if len(nums) == 0 {
|
||||
return "none"
|
||||
}
|
||||
var runs []string
|
||||
start := nums[0]
|
||||
for i := 1; i <= len(nums); i++ {
|
||||
if i < len(nums) && nums[i] == nums[i-1]+1 {
|
||||
continue
|
||||
}
|
||||
if end := nums[i-1]; end == start {
|
||||
runs = append(runs, strconv.Itoa(start))
|
||||
} else {
|
||||
runs = append(runs, fmt.Sprintf("%d-%d", start, end))
|
||||
}
|
||||
if i < len(nums) {
|
||||
start = nums[i]
|
||||
}
|
||||
}
|
||||
return strings.Join(runs, ", ")
|
||||
}
|
||||
|
||||
// markerHole names the violation, or returns "" when the shape is one a fold
|
||||
// can produce: an optional pinned head starting at 1, then one run reaching the
|
||||
// last turn. Two gaps, or a tail that stops short of it, are losses.
|
||||
func markerHole(markers []string, prefix string, total int) string {
|
||||
nums := markerNumbers(markers, prefix)
|
||||
if len(nums) == 0 {
|
||||
return "the view shows no turn at all"
|
||||
}
|
||||
runs := strings.Split(markerShape(markers, prefix), ", ")
|
||||
if last := nums[len(nums)-1]; last != total {
|
||||
return fmt.Sprintf("the newest turn %d is missing", total)
|
||||
}
|
||||
if len(runs) > 2 {
|
||||
return fmt.Sprintf("%d separate runs; a fold removes one", len(runs))
|
||||
}
|
||||
if len(runs) == 2 && nums[0] != 1 {
|
||||
return fmt.Sprintf("the kept head starts at %d, not the pinned first turn", nums[0])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// The oracle has to be able to fail, or "no hole" says nothing. These are the
|
||||
// shapes a fold can produce and the shapes only a loss can.
|
||||
func TestMarkerHoleDiscriminates(t *testing.T) {
|
||||
echo := func(ns ...int) []string {
|
||||
out := make([]string, 0, len(ns))
|
||||
for _, n := range ns {
|
||||
out = append(out, echoPrefix+pad(n))
|
||||
}
|
||||
return out
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
in []string
|
||||
total int
|
||||
holed bool
|
||||
}{
|
||||
{"nothing folded", echo(1, 2, 3, 4, 5), 5, false},
|
||||
{"middle folded away", echo(4, 5), 5, false},
|
||||
{"pinned head kept beside the tail", echo(1, 4, 5), 5, false},
|
||||
{"a reply dropped inside the kept tail", echo(3, 5), 5, true},
|
||||
{"two gaps", echo(1, 3, 5), 5, true},
|
||||
{"newest turn missing", echo(3, 4), 5, true},
|
||||
{"head kept but not the first turn", echo(2, 4, 5), 5, true},
|
||||
{"empty view", nil, 5, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
reason := markerHole(tc.in, echoPrefix, tc.total)
|
||||
if holed := reason != ""; holed != tc.holed {
|
||||
t.Fatalf("markerHole(%v) = %q, want holed=%v", tc.in, reason, tc.holed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkerShapeRendersRuns(t *testing.T) {
|
||||
in := []string{echoPrefix + pad(1), echoPrefix + pad(9), echoPrefix + pad(10), echoPrefix + pad(11)}
|
||||
if got := markerShape(in, echoPrefix); got != "1, 9-11" {
|
||||
t.Fatalf("markerShape = %q, want %q", got, "1, 9-11")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// renderMatrix prints one table per arm. It reports what each row's value was
|
||||
// on both sides of the process boundary, because a verdict without the two
|
||||
// readings behind it is an assertion.
|
||||
func renderMatrix(results []armResult, work string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Runtime resurrection matrix\n\n")
|
||||
b.WriteString("Measurement boundary: OS process. Phase B loads and inspects; it never calls a model.\n\n")
|
||||
fmt.Fprintf(&b, "Arm roots: %s\n\n", work)
|
||||
for _, res := range results {
|
||||
fmt.Fprintf(&b, "## arm: %s\n\n%s\n\n", res.Arm, res.Asks)
|
||||
if res.Invalid != "" {
|
||||
fmt.Fprintf(&b, "**ARM INVALID — %s.** Its rows below establish nothing.\n\n", res.Invalid)
|
||||
}
|
||||
b.WriteString("| Semantic | Before death | After new process | Verdict | Authority | Artifact |\n")
|
||||
b.WriteString("| --- | --- | --- | --- | --- | --- |\n")
|
||||
for _, r := range res.Rows {
|
||||
fmt.Fprintf(&b, "| %s | %s | %s | %s | %s | %s |\n",
|
||||
r.Semantic, cell(r.Before), cell(r.After), r.Verdict, r.Authority, r.Artifact)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
b.WriteString(summarize(res))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func summarize(res armResult) string {
|
||||
counts := map[string]int{}
|
||||
for _, r := range res.Rows {
|
||||
counts[r.Verdict]++
|
||||
}
|
||||
var parts []string
|
||||
// Every verdict the matrix can produce. A summary that silently omits one
|
||||
// is how a VIOLATED row reads as a clean arm.
|
||||
for _, v := range []string{verdictPersisted, verdictExact, verdictLossy, verdictLost, verdictNotMeasured,
|
||||
verdictChanged, verdictStable, verdictContiguous, verdictHole, verdictHolds, verdictViolated} {
|
||||
if counts[v] > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", v, counts[v]))
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " ") + "\n"
|
||||
}
|
||||
|
||||
func cell(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "—"
|
||||
}
|
||||
s = strings.ReplaceAll(s, "|", "\\|")
|
||||
if len(s) > 64 {
|
||||
return s[:61] + "..."
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Verdicts. They are deliberately not booleans: "the host holds it verbatim"
|
||||
// and "a fold over canonical artifacts happens to agree" are different claims
|
||||
// about durability, and a row that was never established supports neither.
|
||||
const (
|
||||
verdictPersisted = "persisted-direct"
|
||||
verdictExact = "reconstructed-exact"
|
||||
verdictLossy = "reconstructed-lossy"
|
||||
verdictLost = "lost"
|
||||
verdictNotMeasured = "not-measured"
|
||||
verdictChanged = "changed"
|
||||
verdictStable = "stable"
|
||||
)
|
||||
|
||||
type row struct {
|
||||
Semantic string `json:"semantic"`
|
||||
Authority string `json:"authority"`
|
||||
Artifact string `json:"artifact"`
|
||||
Reconstruction string `json:"reconstruction"`
|
||||
Before string `json:"before"`
|
||||
After string `json:"after"`
|
||||
Verdict string `json:"verdict"`
|
||||
}
|
||||
|
||||
type armResult struct {
|
||||
Arm string `json:"arm"`
|
||||
Asks string `json:"asks"`
|
||||
Invalid string `json:"invalid,omitempty"`
|
||||
Rows []row `json:"rows"`
|
||||
Extra *Observation `json:"extra,omitempty"`
|
||||
Before Observation `json:"before"`
|
||||
After Observation `json:"after"`
|
||||
}
|
||||
|
||||
func classify(a arm, extra *Observation, before, after Observation) armResult {
|
||||
res := armResult{Arm: a.name, Asks: a.asks, Extra: extra, Before: before, After: after}
|
||||
prefix := prefixRow(before, after)
|
||||
res.Rows = append(res.Rows, prefix)
|
||||
if a.name == "system-swap" && prefix.Verdict != verdictChanged {
|
||||
res.Invalid = "the lever did not move the stable prefix, so this arm measures nothing"
|
||||
}
|
||||
if a.name == "exact" && prefix.Verdict != verdictStable {
|
||||
res.Invalid = "the stable prefix moved without a lever, so this arm is not the control it claims"
|
||||
}
|
||||
res.Rows = append(res.Rows,
|
||||
valueRow("system row in the session", "the transcript's own leading system message",
|
||||
"<stem>.jsonl", "loaded with the transcript",
|
||||
before.SystemHash, after.SystemHash, true),
|
||||
valueRow("canonical transcript", "session transcript + event log",
|
||||
"<stem>.jsonl, <stem>.events.jsonl", "LoadSession replays the append log",
|
||||
before.Transcript.Digest, after.Transcript.Digest, true),
|
||||
valueRow("active goal identity", "goalMachine (control)",
|
||||
"<stem>.goal-state.json", "restoreFromState on Resume",
|
||||
before.Goal.Goal, after.Goal.Goal, true),
|
||||
valueRow("goal status", "goalMachine (control)",
|
||||
"<stem>.goal-state.json", "restoreFromState on Resume",
|
||||
before.Goal.Status, after.Goal.Status, true),
|
||||
valueRow("todo host identity", "agent todoState (host-owned, never in the prompt)",
|
||||
"none verbatim; goal-state.json snapshots a terminal goal's list",
|
||||
"rebuilt from the transcript's todo_write / complete_step receipts",
|
||||
todoIDs(before), todoIDs(after), false),
|
||||
valueRow("open decisions", "approvalManager (control, in-memory)",
|
||||
"none", "none: derived from live pending approvals/asks",
|
||||
decisionIDs(before), decisionIDs(after), false),
|
||||
valueRow("compaction projection (stored)", "compaction sidecar",
|
||||
"<stem>.context.json", "LoadProjectionSidecar on bind",
|
||||
sidecarClaim(before), sidecarClaim(after), true),
|
||||
useRow("compaction projection (in use)", before, after),
|
||||
valueRow("run graph nodes", "event sink (GraphDelta)",
|
||||
"none observed", "fold GraphDelta events, if any survive",
|
||||
graphNodes(before), graphNodes(after), false),
|
||||
valueRow("run graph grants", "event sink (GraphDelta)",
|
||||
"none observed", "fold GraphDelta events, if any survive",
|
||||
graphMap(before.Graph.Grants), graphMap(after.Graph.Grants), false),
|
||||
valueRow("run graph wait causes", "event sink (GraphDelta)",
|
||||
"none observed", "fold GraphDelta events, if any survive",
|
||||
graphMap(before.Graph.Waits), graphMap(after.Graph.Waits), false),
|
||||
)
|
||||
switch {
|
||||
case successorArm(a.name) && extra != nil:
|
||||
res.Rows = append(res.Rows, successorRows(a.name, *extra, after)...)
|
||||
// The premise is a barrier open at death. The construct process is its
|
||||
// own waiter, so it reports no interruption of its own.
|
||||
if len(before.Decisions) == 0 {
|
||||
res.Invalid = "no decision was open when the first process died"
|
||||
}
|
||||
case a.name == armOpenDecision:
|
||||
res.Rows = append(res.Rows, decisionRows(before, after)...)
|
||||
if len(before.Decisions) == 0 {
|
||||
res.Invalid = "no decision was open when the process died, so nothing was interrupted"
|
||||
}
|
||||
case deriveArm(a.name):
|
||||
res.Rows = append(res.Rows, deriveRows(a.name, before, after)...)
|
||||
res.Invalid = deriveArmInvalid(a.name, before)
|
||||
case terminalArm(a.name):
|
||||
res.Rows = append(res.Rows, terminalRows(a.name, before, after)...)
|
||||
res.Invalid = terminalArmInvalid(a.name, before)
|
||||
case schedulerWaitArm(a.name):
|
||||
res.Rows = append(res.Rows, schedulerWaitRows(a.name, before, after)...)
|
||||
res.Invalid = schedulerArmInvalid(a.name, before)
|
||||
case graphArm(a.name):
|
||||
res.Rows = append(res.Rows, graphRows(before, after)...)
|
||||
res.Invalid = graphArmInvalid(a.name, before)
|
||||
case uiArm(a.name):
|
||||
res.Rows = append(res.Rows, uiRows(before, after)...)
|
||||
res.Invalid = uiArmInvalid(before, after)
|
||||
case lineageArm(a.name):
|
||||
res.Rows = append(res.Rows, lineageRows(a.name, before)...)
|
||||
case cancelRoutingArm(a.name):
|
||||
res.Rows = append(res.Rows, cancelRoutingRows(a.name, before)...)
|
||||
case loneTaskArm(a.name):
|
||||
res.Rows = append(res.Rows, loneTaskRows(a.name, before, after)...)
|
||||
res.Invalid = loneTaskArmInvalid(a.name, before)
|
||||
case activeStoreArm(a.name):
|
||||
res.Rows = append(res.Rows, activeStoreRows(before, after)...)
|
||||
res.Invalid = activeStoreInvalid(before)
|
||||
case skillArm(a.name):
|
||||
res.Rows = append(res.Rows, skillClassRows(a.name, before, after)...)
|
||||
res.Invalid = skillArmInvalid(a.name, before)
|
||||
case ephemeralArm(a.name):
|
||||
res.Rows = append(res.Rows, ephemeralRows(a.name, before, after)...)
|
||||
res.Invalid = ephemeralArmInvalid(before)
|
||||
case hostExecArm(a.name):
|
||||
res.Rows = append(res.Rows, hostExecRows(a.name, before, after)...)
|
||||
res.Invalid = hostExecArmInvalid(a.name, before)
|
||||
case a.name == armTodoIdentity:
|
||||
res.Rows = append(res.Rows, todoRows(before, after)...)
|
||||
case a.name == armRefoldIntoBody && extra != nil:
|
||||
res.Rows = append(res.Rows, refoldRows(*extra, before, after, probeTurns+refoldTurns)...)
|
||||
if extra.Sidecar.Messages == 0 {
|
||||
res.Invalid = "the first fold stored no body, so the second had none to reach into"
|
||||
}
|
||||
case (a.name == armTailRewind || a.name == armCoveredRewind) && extra != nil:
|
||||
res.Rows = append(res.Rows, useRow("projection across the rewind, same process", *extra, before))
|
||||
res.Rows = append(res.Rows, rewindLandingRow(a.name, *extra, before))
|
||||
res.Invalid = rewindLandedWrong(a.name, *extra, before)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// prefixRow is the arm's own control rather than a durability claim: it reports
|
||||
// whether the stable prefix moved, which is the thing each arm sets up.
|
||||
func prefixRow(before, after Observation) row {
|
||||
verdict := verdictStable
|
||||
if before.BootSystemHash != after.BootSystemHash {
|
||||
verdict = verdictChanged
|
||||
}
|
||||
return row{
|
||||
Semantic: "stable prefix identity (composed)", Authority: "boot assembly (recomposed per process)",
|
||||
Artifact: "none (composed from config, memory and instructions)", Reconstruction: "recomposed at boot",
|
||||
Before: before.BootSystemHash, After: after.BootSystemHash, Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
func valueRow(semantic, authority, artifact, reconstruction, before, after string, persisted bool) row {
|
||||
r := row{Semantic: semantic, Authority: authority, Artifact: artifact,
|
||||
Reconstruction: reconstruction, Before: before, After: after}
|
||||
switch {
|
||||
case before == "" || before == "-":
|
||||
r.Verdict = verdictNotMeasured
|
||||
case after == "" || after == "-":
|
||||
r.Verdict = verdictLost
|
||||
case before != after:
|
||||
r.Verdict = verdictLossy
|
||||
case persisted:
|
||||
r.Verdict = verdictPersisted
|
||||
default:
|
||||
r.Verdict = verdictExact
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func todoIDs(o Observation) string {
|
||||
var ids []string
|
||||
for _, t := range o.Todos {
|
||||
id := t.StepID
|
||||
if id == "" {
|
||||
id = "(no step_id)"
|
||||
}
|
||||
ids = append(ids, id+":"+t.Status)
|
||||
}
|
||||
return join(ids)
|
||||
}
|
||||
|
||||
func decisionIDs(o Observation) string {
|
||||
var ids []string
|
||||
for _, d := range o.Decisions {
|
||||
ids = append(ids, string(d.Kind)+":"+d.ID)
|
||||
}
|
||||
return join(ids)
|
||||
}
|
||||
|
||||
func sidecarClaim(o Observation) string {
|
||||
if !o.Sidecar.Present {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("v%d msgs=%d covered=%d hash=%s",
|
||||
o.Sidecar.ProjectionVersion, o.Sidecar.Messages, o.Sidecar.CoveredCount, short(o.Sidecar.CoveredPrefixHash))
|
||||
}
|
||||
|
||||
// rewindLandingRow says where the rewind actually landed relative to the fold.
|
||||
// An arm that names a side and does not reach it is not measuring that side.
|
||||
func rewindLandingRow(arm string, prerewind, after Observation) row {
|
||||
side := "above the fold boundary"
|
||||
if after.Transcript.Messages < prerewind.Sidecar.CoveredCount {
|
||||
side = "below the fold boundary"
|
||||
}
|
||||
return row{
|
||||
Semantic: "where the rewind landed", Authority: "checkpoint boundary against the sidecar",
|
||||
Artifact: "none: derived", Reconstruction: "canonical length after the rewind against CoveredCount before it",
|
||||
Before: fmt.Sprintf("covered=%d", prerewind.Sidecar.CoveredCount),
|
||||
After: fmt.Sprintf("canonical=%d, %s", after.Transcript.Messages, side), Verdict: verdictStable,
|
||||
}
|
||||
}
|
||||
|
||||
// rewindLandedWrong invalidates an arm that aimed at one side of the fold and
|
||||
// reached the other. covered-rewind stopped removing folded history the moment
|
||||
// coverage became a real boundary, and nothing in the matrix said so.
|
||||
func rewindLandedWrong(arm string, prerewind, after Observation) string {
|
||||
below := after.Transcript.Messages < prerewind.Sidecar.CoveredCount
|
||||
switch {
|
||||
case arm == armCoveredRewind && !below:
|
||||
return "the rewind landed above the fold boundary, so it removed no folded history"
|
||||
case arm == armTailRewind && below:
|
||||
return "the rewind landed below the fold boundary, so it is not a tail-only truncation"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// useRow turns on whether the next request would ride the projection, not on
|
||||
// the token counts beside it: a tail edit legitimately moves those, and letting
|
||||
// them decide the verdict reported a surviving projection as a loss.
|
||||
func useRow(semantic string, before, after Observation) row {
|
||||
r := valueRow(semantic, "projectionValid over the covered prefix",
|
||||
"none: a judgement, not a stored value",
|
||||
"revalidated against the canonical prefix on every read",
|
||||
projectionState(before), projectionState(after), false)
|
||||
r.Before, r.After = projectionUse(before), projectionUse(after)
|
||||
return r
|
||||
}
|
||||
|
||||
// projectionState is the compared value: in use, not in use, or never measured.
|
||||
func projectionState(o Observation) string {
|
||||
switch o.Context.CheckpointState {
|
||||
case "":
|
||||
return ""
|
||||
case "none":
|
||||
return "NOT-in-use"
|
||||
}
|
||||
return "in-use"
|
||||
}
|
||||
|
||||
// projectionUse reads the host's own judgement rather than the stored claim.
|
||||
// CheckpointState is "none" whenever validation failed, which is the difference
|
||||
// between a sidecar that loaded and one the next request will actually ride.
|
||||
// The label itself is not compared — "applied" and "restored" are the same
|
||||
// answer reached from different sides — so the row turns on use and size.
|
||||
func projectionUse(o Observation) string {
|
||||
if o.Context.CheckpointState == "" {
|
||||
return ""
|
||||
}
|
||||
use := "in-use"
|
||||
if o.Context.CheckpointState == "none" {
|
||||
use = "NOT-in-use"
|
||||
}
|
||||
return fmt.Sprintf("%s projected=%d canonical=%d", use,
|
||||
o.Context.ProjectedTokens, o.Context.CanonicalTokens)
|
||||
}
|
||||
|
||||
// graphNodes renders the fold's nodes in a stable order. The fold keeps
|
||||
// first-seen order, which two processes reach by different routes, so an
|
||||
// unsorted rendering would report a reordering as a changed graph.
|
||||
func graphNodes(o Observation) string {
|
||||
if len(o.Graph.Nodes) == 0 {
|
||||
return ""
|
||||
}
|
||||
ids := make([]string, 0, len(o.Graph.Nodes))
|
||||
for _, n := range o.Graph.Nodes {
|
||||
ids = append(ids, n.ID+":"+string(n.State))
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return join(ids)
|
||||
}
|
||||
|
||||
// graphMap renders one per-node attribute set. Map iteration is unordered, so
|
||||
// without the sort two identical envelopes would compare as different.
|
||||
func graphMap(m map[string]string) string {
|
||||
if len(m) == 0 {
|
||||
return ""
|
||||
}
|
||||
out := make([]string, 0, len(m))
|
||||
for k, v := range m {
|
||||
out = append(out, k+"="+v)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
func join(in []string) string {
|
||||
if len(in) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(in, ", ")
|
||||
}
|
||||
|
||||
func short(s string) string {
|
||||
if len(s) <= 12 {
|
||||
return s
|
||||
}
|
||||
return s[:12]
|
||||
}
|
||||
|
||||
func marshalResults(results []armResult) ([]byte, error) {
|
||||
return json.MarshalIndent(results, "", " ")
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/assembly/boot"
|
||||
"reasonix/internal/contract/agentgraph"
|
||||
"reasonix/internal/contract/event"
|
||||
"reasonix/internal/session/control"
|
||||
)
|
||||
|
||||
// armRoot names the four directories one arm owns. Home and Workspace are
|
||||
// separate so a lever can change what the prefix folds without touching the
|
||||
// transcript, and Sessions is explicit so the probe never writes to the
|
||||
// developer's own session directory.
|
||||
type armRoot struct{ Home, Workspace, Sessions, Dir string }
|
||||
|
||||
func rootFor(dir string) armRoot {
|
||||
return armRoot{
|
||||
Dir: dir,
|
||||
Home: filepath.Join(dir, "home"),
|
||||
Workspace: filepath.Join(dir, "workspace"),
|
||||
Sessions: filepath.Join(dir, "sessions"),
|
||||
}
|
||||
}
|
||||
|
||||
func (r armRoot) create() error {
|
||||
for _, d := range []string{r.Home, r.Workspace, r.Sessions} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeProjectConfig puts a project config beside the workspace, which is where
|
||||
// a person configures these ceilings. The scheduler arms need them small enough
|
||||
// to reach a specific refusal, and a poked field would prove nothing about the
|
||||
// judgement production makes.
|
||||
func (r armRoot) writeProjectConfig(total, writers int) error {
|
||||
body := fmt.Sprintf("[agent]\nmax_subagent_concurrency = %d\nmax_parallel_writers = %d\n", total, writers)
|
||||
return os.WriteFile(filepath.Join(r.Workspace, "reasonix.toml"), []byte(body), 0o644)
|
||||
}
|
||||
|
||||
// requireFresh refuses a root a previous run left behind. A reused root is how
|
||||
// a lever's effect ends up already present in the construct phase, which reads
|
||||
// as "the lever changed nothing" — a false negative that looks like a result.
|
||||
func (r armRoot) requireFresh() error {
|
||||
entries, err := os.ReadDir(r.Dir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if len(entries) > 0 {
|
||||
return fmt.Errorf("arm root %s is not empty; each run needs a clean root", r.Dir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// graphSink records what the run graph published while this process was alive.
|
||||
// It is the only "before" the graph row has: nothing else in a session's
|
||||
// artifacts is known to carry it, and a row with no before is not measured.
|
||||
type graphSink struct {
|
||||
mu sync.Mutex
|
||||
graph agentgraph.Graph
|
||||
deltas int
|
||||
// waits is every wait cause published per node, in order. The fold keeps
|
||||
// only the latest, which cannot say whether a cause was reported once or
|
||||
// replaced — and that is exactly what the transition arm asks.
|
||||
waits map[string][]string
|
||||
// phases is every subagent status the parent was told, per call. A run
|
||||
// handed to a job publishes no graph node at all, so for those this is the
|
||||
// only live evidence that anything happened.
|
||||
phases map[string][]string
|
||||
// tools is when each call was dispatched and when it answered. A stop that
|
||||
// reached a running tool is visible as the second arriving early.
|
||||
tools map[string][2]time.Time
|
||||
}
|
||||
|
||||
func (s *graphSink) Emit(e event.Event) {
|
||||
if e.Kind == event.ToolDispatch || e.Kind == event.ToolResult {
|
||||
s.noteTool(e)
|
||||
return
|
||||
}
|
||||
if id, phase, ok := progressPhase(e); ok {
|
||||
s.mu.Lock()
|
||||
if s.phases == nil {
|
||||
s.phases = map[string][]string{}
|
||||
}
|
||||
s.phases[id] = append(s.phases[id], phase)
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if e.Kind != event.GraphDelta || e.Graph == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.deltas++
|
||||
for _, n := range e.Graph.Nodes {
|
||||
if n.Wait == "" {
|
||||
continue
|
||||
}
|
||||
if s.waits == nil {
|
||||
s.waits = map[string][]string{}
|
||||
}
|
||||
s.waits[n.ID] = append(s.waits[n.ID], string(n.Wait))
|
||||
}
|
||||
s.graph.Apply(*e.Graph)
|
||||
}
|
||||
|
||||
func (s *graphSink) snapshot() (agentgraph.Graph, int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.graph, s.deltas
|
||||
}
|
||||
|
||||
// noteTool records the two edges of a tool call this probe may be asked about.
|
||||
func (s *graphSink) noteTool(e event.Event) {
|
||||
if e.Tool.ID == "" {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.tools == nil {
|
||||
s.tools = map[string][2]time.Time{}
|
||||
}
|
||||
at := s.tools[e.Tool.ID]
|
||||
if e.Kind == event.ToolDispatch {
|
||||
at[0] = time.Now()
|
||||
} else {
|
||||
at[1] = time.Now()
|
||||
}
|
||||
s.tools[e.Tool.ID] = at
|
||||
}
|
||||
|
||||
func (s *graphSink) toolDispatched(id string) (time.Time, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
at, ok := s.tools[id]
|
||||
return at[0], ok && !at[0].IsZero()
|
||||
}
|
||||
|
||||
func (s *graphSink) toolResult(id string) (time.Time, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
at, ok := s.tools[id]
|
||||
return at[1], ok && !at[1].IsZero()
|
||||
}
|
||||
|
||||
// phaseSeries is every status published for one call, in publication order.
|
||||
func (s *graphSink) phaseSeries() map[string][]string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := map[string][]string{}
|
||||
for id, seq := range s.phases {
|
||||
out[id] = append([]string(nil), seq...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// waitSeries is every cause published for one node, in publication order.
|
||||
func (s *graphSink) waitSeries() map[string][]string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := map[string][]string{}
|
||||
for id, seq := range s.waits {
|
||||
out[id] = append([]string(nil), seq...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildRuntime assembles a real runtime bound to this arm's roots. Everything a
|
||||
// frontend would supply is supplied; only the provider is scripted, so the
|
||||
// probe never depends on a network or a key.
|
||||
func buildRuntime(ctx context.Context, root armRoot, arm string, sink event.Sink) (*control.Controller, *scripted, error) {
|
||||
prov := newScripted(arm)
|
||||
ctrl, err := boot.Build(ctx, boot.Options{
|
||||
Version: "runtime-resume-probe",
|
||||
Model: probeModelRef,
|
||||
Sink: sink,
|
||||
WorkspaceRoot: root.Workspace,
|
||||
Home: root.Home,
|
||||
SessionDir: root.Sessions,
|
||||
ProviderResolver: resolverFor(prov),
|
||||
Stderr: io.Discard,
|
||||
HeadlessApprovalMode: approvalMode(arm),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ctrl.ApplyHeadlessApprovalMode(approvalMode(arm))
|
||||
return ctrl, prov, nil
|
||||
}
|
||||
|
||||
// approvalMode is the gate this arm needs. Everything else runs denied, the
|
||||
// strictest a headless frontend can be. A fleet is not read-only, so under a
|
||||
// denied gate the dispatch is refused before any child starts and the arm
|
||||
// measures a permission decision instead of a process boundary.
|
||||
func approvalMode(arm string) string {
|
||||
if graphArm(arm) || uiArm(arm) || loneTaskArm(arm) || cancelRoutingArm(arm) || lineageArm(arm) ||
|
||||
schedulerWaitArm(arm) || terminalArm(arm) || deriveArm(arm) || activeStoreArm(arm) ||
|
||||
skillArm(arm) || ephemeralArm(arm) || hostExecArm(arm) {
|
||||
return control.ToolApprovalAuto
|
||||
}
|
||||
return "deny"
|
||||
}
|
||||
|
||||
// childEnv binds the child process to this arm's home through the environment
|
||||
// as well as through Options.Home: a Controller's later re-reads and the shared
|
||||
// history index still read the process, not the assembly.
|
||||
func childEnv(root armRoot) []string {
|
||||
env := append(os.Environ(),
|
||||
"REASONIX_HOME="+root.Home,
|
||||
"REASONIX_STATE_HOME="+root.Home,
|
||||
)
|
||||
// A child runs in its own workspace, so the one arm that has to build the
|
||||
// frontend is told where the checkout is rather than searching for it.
|
||||
if repo := repoRoot(); repo != "" {
|
||||
env = append(env, envProbeRepo+"="+repo)
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
// repoRoot walks up from the orchestrator's working directory for the module
|
||||
// this probe lives in. Empty when it is not run from a checkout, which the arm
|
||||
// reports as not measured rather than as a failure.
|
||||
func repoRoot() string {
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for {
|
||||
if b, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil &&
|
||||
strings.HasPrefix(string(b), "module reasonix") {
|
||||
return dir
|
||||
}
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
return ""
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
}
|
||||
|
||||
// toolIDs is every call this sink saw dispatched, which is where a host-minted
|
||||
// event id first appears: it is not in the graph and not in the transcript.
|
||||
func (s *graphSink) toolIDs() map[string]bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make(map[string]bool, len(s.tools))
|
||||
for id := range s.tools {
|
||||
out[id] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"reasonix/internal/contract/agentgraph"
|
||||
"reasonix/internal/session/control"
|
||||
)
|
||||
|
||||
// The scheduler-wait arms. Each dies with one item ready and refused admission,
|
||||
// and they are separate because the scheduler decides in a fixed order —
|
||||
// slots, then writers, then claim. An arm that let an earlier cause stand would
|
||||
// report it as the later one, so each has to clear every check above the one it
|
||||
// is measuring.
|
||||
const (
|
||||
// armWaitSlots fills the session's total ceiling with runs that never end.
|
||||
// The refused item is read-only: it clears every writer check by not being
|
||||
// a writer at all.
|
||||
armWaitSlots = "wait-slots"
|
||||
// armWaitWriters leaves total capacity free and fills the writer ceiling.
|
||||
// Its two writers declare disjoint paths, so a claim conflict cannot be
|
||||
// what refuses the second one.
|
||||
armWaitWriters = "wait-writers"
|
||||
// armWaitClaim leaves both ceilings free and overlaps two write paths. A
|
||||
// single fan-out cannot reach this: preflight refuses concurrent writers
|
||||
// whose claims overlap, so the holder is dispatched as a background fleet
|
||||
// and the refused writer comes from a later one.
|
||||
armWaitClaim = "wait-claim"
|
||||
// armWaitTransition asks what a wait cause means once the thing it named
|
||||
// stops being what holds the item back. The scheduler reports the cause
|
||||
// once, when the request queues; the blocker can change afterwards.
|
||||
armWaitTransition = "wait-transition"
|
||||
)
|
||||
|
||||
func schedulerWaitArm(name string) bool {
|
||||
switch name {
|
||||
case armWaitSlots, armWaitWriters, armWaitClaim, armWaitTransition:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// schedulerLimits are the ceilings an arm needs to reach its refusal. They are
|
||||
// written to the project config, so the run makes the same judgement a person
|
||||
// configuring the session would get.
|
||||
func schedulerLimits(arm string) (total, writers int) {
|
||||
switch arm {
|
||||
case armTerminalCancelled:
|
||||
// One slot, so the group has an item running and items that never got
|
||||
// one — the split a cancellation is reported against.
|
||||
return 1, 1
|
||||
case armLineageBackground, armLineageJobKill:
|
||||
// Room for a second run: the alive check asks whether the manager can
|
||||
// still start anything, and a ceiling of one would answer for the
|
||||
// hanging child instead.
|
||||
return 4, 4
|
||||
case armTaskFgQueued, armTaskBgQueued, armTaskFgQueuedCancel, armTaskBgQueuedCancel:
|
||||
// One slot, held by the fleet dispatched ahead of the delegation, so the
|
||||
// refusal the arm records is the scheduler's and not a race.
|
||||
return 1, 1
|
||||
case armUIGraphMixed:
|
||||
// One slot, so the item after the one that hangs is refused: the death
|
||||
// then holds a node that was queued and never admitted.
|
||||
return 1, 1
|
||||
case armHostQueued, armHeadlessQueuedCancel:
|
||||
// One slot, held by the fan-out dispatched ahead of the invocation.
|
||||
return 1, 1
|
||||
case armSkillQueued, armReadOnlySkillQueued:
|
||||
// One slot, held by the fleet dispatched ahead of the skill, so the
|
||||
// refusal the arm reads is the scheduler's and not a race.
|
||||
return 1, 1
|
||||
case armFleetActiveStore:
|
||||
// Two slots: the delegation holds one to the death, and the fan-out's
|
||||
// first item frees the other for exactly one of the two behind it.
|
||||
return 2, 2
|
||||
case armWaitSlots:
|
||||
return 2, 1
|
||||
case armWaitWriters:
|
||||
return 6, 1
|
||||
case armWaitClaim:
|
||||
return 6, 3
|
||||
case armWaitTransition:
|
||||
return 3, 1
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
// wantedCause is the refusal this arm exists to produce. The transition arm
|
||||
// starts at slots and asks what happens to that answer afterwards.
|
||||
func wantedCause(arm string) agentgraph.WaitCause {
|
||||
switch arm {
|
||||
case armWaitWriters:
|
||||
return agentgraph.WaitWriters
|
||||
case armWaitClaim:
|
||||
return agentgraph.WaitClaim
|
||||
}
|
||||
return agentgraph.WaitSlots
|
||||
}
|
||||
|
||||
// waitingWorkers are the nodes the scheduler refused and has not admitted: the
|
||||
// arm's premise. State is checked as well as the cause, because a node that
|
||||
// went on to run still carries the cause it waited under.
|
||||
func waitingWorkers(g agentgraph.Graph, cause agentgraph.WaitCause) []string {
|
||||
var out []string
|
||||
for _, n := range g.Nodes {
|
||||
if n.Kind == agentgraph.KindGroup || n.State != agentgraph.StateQueued {
|
||||
continue
|
||||
}
|
||||
if n.Wait == cause {
|
||||
out = append(out, n.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// waitForRefusal blocks until one item is queued under this arm's cause.
|
||||
func waitForRefusal(sink *graphSink, arm string) error {
|
||||
cause := wantedCause(arm)
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
graph, _ := sink.snapshot()
|
||||
if len(waitingWorkers(graph, cause)) > 0 {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
graph, _ := sink.snapshot()
|
||||
return errUnexpected("an item queued under "+string(cause), waitStates(graph))
|
||||
}
|
||||
|
||||
func waitStates(g agentgraph.Graph) string {
|
||||
out := make([]string, 0, len(g.Nodes))
|
||||
for _, n := range g.Nodes {
|
||||
out = append(out, fmt.Sprintf("%s:%s/%s", n.ID, n.State, orNone(string(n.Wait))))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return orNone(join(out))
|
||||
}
|
||||
|
||||
// schedTurn is the prompt that opens this arm's fan-out. The slots arm needs
|
||||
// only one fleet; the others name both, and the latch order dispatches the
|
||||
// holder first while the refused fleet waits on the holder's own signal.
|
||||
func schedTurn(n int, arm string) string {
|
||||
if arm == armWaitSlots {
|
||||
return fmt.Sprintf("%s %s: dispatch the fan-out.", marker(n), fleetPairSentinel)
|
||||
}
|
||||
return fmt.Sprintf("%s %s %s: dispatch the fan-out.", marker(n), fleetHolderSentinel, fleetRefusedSentinel)
|
||||
}
|
||||
|
||||
// runSchedulerWaitConstruct drives the arm to its refusal and exits while the
|
||||
// item is still queued. The transition arm goes further: it frees the capacity
|
||||
// the refusal named and asks what the reported cause does about it.
|
||||
func runSchedulerWaitConstruct(ctx context.Context, root armRoot, arm, bootSystem string, ctrl *control.Controller, sink *graphSink, prov *scripted, turn int) error {
|
||||
go func() { _ = ctrl.Run(ctx, schedTurn(turn+1, arm)) }()
|
||||
if err := waitForRefusal(sink, arm); err != nil {
|
||||
return err
|
||||
}
|
||||
if arm == armWaitTransition {
|
||||
if err := releaseAndSettle(sink, prov); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := writeObservation(root, capture("construct", arm, bootSystem, ctrl, sink, root)); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Exit(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// releaseAndSettle frees the run that filled total capacity, then waits for the
|
||||
// graph to show it gone. What the refused item's cause does after that is the
|
||||
// arm's question: the ceiling it named has room, and the writer ceiling that
|
||||
// now holds it back was never reported.
|
||||
func releaseAndSettle(sink *graphSink, prov *scripted) error {
|
||||
prov.releaseOnce()
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
graph, _ := sink.snapshot()
|
||||
if settledWorkers(graph) > 0 && len(waitingWorkers(graph, agentgraph.WaitSlots)) > 0 {
|
||||
// Give any second report a chance to arrive before reading.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
return nil
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
graph, _ := sink.snapshot()
|
||||
return errUnexpected("capacity freed while a refusal still stands", waitStates(graph))
|
||||
}
|
||||
|
||||
func settledWorkers(g agentgraph.Graph) int {
|
||||
n := 0
|
||||
for _, node := range g.Nodes {
|
||||
if node.Kind != agentgraph.KindGroup && node.State != "" && node.State.Terminal() {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// schedulerWaitRows report what a restart inherits about an item the scheduler
|
||||
// refused. The classification is already right — STARTED settled that — so the
|
||||
// question here is narrower: can the host still say why it never started?
|
||||
func schedulerWaitRows(arm string, before, after Observation) []row {
|
||||
rows := []row{refusalRow(arm, before, after)}
|
||||
if arm == armWaitTransition {
|
||||
rows = append(rows, capacityFreedRow(before))
|
||||
}
|
||||
return append(rows,
|
||||
waitSeriesRow(arm, before),
|
||||
executionKindRow(before, after),
|
||||
waitProvenanceRow(before, after),
|
||||
schedulerGateRow(before, after),
|
||||
)
|
||||
}
|
||||
|
||||
// capacityFreedRow is the transition arm's evidence. It freed the ceiling the
|
||||
// refusal named and the item stayed queued, held by a different one — so what
|
||||
// the reported cause does next is a question about the contract, not a race.
|
||||
func capacityFreedRow(before Observation) row {
|
||||
return row{
|
||||
Semantic: "the ceiling the refusal named, once freed", Authority: "the arm, through a child it releases",
|
||||
Artifact: "none", Reconstruction: "an item settles while the refused one stays queued",
|
||||
Before: fmt.Sprintf("%d settled, still queued: %s", before.SettledWorkers, orNone(join(before.QueuedWorkers))),
|
||||
After: "not applicable: the scheduler is gone", Verdict: verdictStable,
|
||||
}
|
||||
}
|
||||
|
||||
// refusalRow is the arm's own control: it reached the refusal it names, and no
|
||||
// earlier one. Reaching a different cause means an upper check intercepted it,
|
||||
// and the arm measured someone else's question.
|
||||
func refusalRow(arm string, before, after Observation) row {
|
||||
want := string(wantedCause(arm))
|
||||
verdict := verdictHolds
|
||||
if before.WaitCauses[want] == 0 {
|
||||
verdict = verdictNotMeasured
|
||||
}
|
||||
return row{
|
||||
Semantic: "the refusal this arm reached", Authority: "SubagentScheduler.canStartLocked",
|
||||
Artifact: "none: reported once through OnQueued", Reconstruction: "slots, then writers, then claim",
|
||||
Before: causeSummary(before), After: causeSummary(after), Verdict: verdict,
|
||||
}
|
||||
}
|
||||
|
||||
// waitSeriesRow is what the transition arm exists for: how many times a cause
|
||||
// was published for the item that never started. Once means the answer is the
|
||||
// cause that queued it, not the one that held it last.
|
||||
func waitSeriesRow(arm string, before Observation) row {
|
||||
return row{
|
||||
Semantic: "how often a wait cause was reported", Authority: "OnQueued, which fires once per request",
|
||||
Artifact: "none", Reconstruction: causeContract(before),
|
||||
Before: seriesSummary(before), After: "not applicable: the scheduler is gone",
|
||||
Verdict: verdictStable,
|
||||
}
|
||||
}
|
||||
|
||||
// causeContract states what the observed reporting says the cause means. It is
|
||||
// read off the series rather than assumed: one report can only be the cause
|
||||
// that queued the request, and more than one would track the current blocker.
|
||||
func causeContract(o Observation) string {
|
||||
for _, seq := range o.WaitSeries {
|
||||
if len(seq) > 1 {
|
||||
return "reported again as the blocker changes: the cause is the current one"
|
||||
}
|
||||
}
|
||||
return "reported once: the cause is the one that queued the request, not the one holding it now"
|
||||
}
|
||||
|
||||
// waitProvenanceRow is what the batch was built to expose and then close. The
|
||||
// refusal exists only at the moment the scheduler makes it, so it is recorded
|
||||
// there; this compares what the graph showed against what a later process can
|
||||
// still read back.
|
||||
func waitProvenanceRow(before, after Observation) row {
|
||||
return valueRow("why the item never started", "the execution journal's queue records",
|
||||
"<stem>.execution.jsonl", "the cause the scheduler gave when it first refused admission",
|
||||
causeSummary(before), journalCauses(after), true)
|
||||
}
|
||||
|
||||
// journalCauses is the refusal a restart reads back, counted the way the graph
|
||||
// counts it so the two sides of the row are the same measurement.
|
||||
func journalCauses(o Observation) string {
|
||||
counts := map[string]int{}
|
||||
for _, e := range o.Executions {
|
||||
if e.Queued() {
|
||||
counts[e.Cause]++
|
||||
}
|
||||
}
|
||||
if len(counts) == 0 {
|
||||
return ""
|
||||
}
|
||||
keys := make([]string, 0, len(counts))
|
||||
for k := range counts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", k, counts[k]))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// schedulerGateRow is the other half of what a queue record proves. An item
|
||||
// only reaches the scheduler once its dependencies are answered, so a queued
|
||||
// entry is durable evidence it crossed that gate — the one thing a dependency
|
||||
// edge alone could never say.
|
||||
func schedulerGateRow(before, after Observation) row {
|
||||
return valueRow("items proven to have reached the scheduler", "the execution journal's queue records",
|
||||
"<stem>.execution.jsonl", "a queue record implies the dependency gate was crossed",
|
||||
queuedIDs(before), queuedIDs(after), true)
|
||||
}
|
||||
|
||||
func queuedIDs(o Observation) string {
|
||||
var out []string
|
||||
for _, e := range o.Executions {
|
||||
if e.Queued() {
|
||||
out = append(out, e.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return join(out)
|
||||
}
|
||||
|
||||
func causeSummary(o Observation) string {
|
||||
if len(o.WaitCauses) == 0 {
|
||||
return ""
|
||||
}
|
||||
keys := make([]string, 0, len(o.WaitCauses))
|
||||
for k := range o.WaitCauses {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", k, o.WaitCauses[k]))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func seriesSummary(o Observation) string {
|
||||
if len(o.WaitSeries) == 0 {
|
||||
return "none"
|
||||
}
|
||||
ids := make([]string, 0, len(o.WaitSeries))
|
||||
for id := range o.WaitSeries {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
out := make([]string, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
out = append(out, id+":["+strings.Join(o.WaitSeries[id], "->")+"]")
|
||||
}
|
||||
return join(out)
|
||||
}
|
||||
|
||||
// schedulerArmInvalid refuses an arm that did not reach its own refusal. Each
|
||||
// arm exists to isolate one cause, and one that reached a different cause was
|
||||
// intercepted by a check above it — reporting that as this arm's answer is how
|
||||
// a probe measures the wrong ceiling and calls it evidence.
|
||||
func schedulerArmInvalid(arm string, before Observation) string {
|
||||
want := string(wantedCause(arm))
|
||||
if before.WaitCauses[want] == 0 {
|
||||
return "no item was refused under " + want + "; the arm reached " + orNone(causeSummary(before))
|
||||
}
|
||||
for cause := range before.WaitCauses {
|
||||
if cause != want {
|
||||
return "an item was also refused under " + cause + ", so this arm does not isolate " + want
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"reasonix/internal/contract/provider"
|
||||
)
|
||||
|
||||
// newScripted builds the provider with its synchronisation channels open. The
|
||||
// scheduler arms coordinate through them; every other arm never reads them.
|
||||
func newScripted(arm string) *scripted {
|
||||
return &scripted{arm: arm, held: make(chan struct{}), release: make(chan struct{})}
|
||||
}
|
||||
|
||||
// probeModelRef is both the catalog ref and the model name boot is given, so
|
||||
// syntheticEntryFromResolver matches it without touching any config file.
|
||||
const probeModelRef = "probe/scripted"
|
||||
|
||||
// probeAltModelRef is a second identity the same scripted provider answers to.
|
||||
// Without one, every layer of the model resolution collapses onto the same
|
||||
// string and a probe cannot tell which layer produced it.
|
||||
const probeAltModelRef = "probe/alt"
|
||||
|
||||
// scripted answers completions from a fixed script so the construct phase can
|
||||
// establish host state without a network call. A request carrying no tools is a
|
||||
// host-internal completion (the compaction summary); everything else is a turn.
|
||||
type scripted struct {
|
||||
// arm is which run this provider is scripting. Only the fan-out arms read
|
||||
// it: the pair fleet is the same dispatch either way, and the arm decides
|
||||
// whether its second item finishes or is still executing at death.
|
||||
arm string
|
||||
turnCalls atomic.Int64
|
||||
// retodo latches the one reply that replaces the task list. The sentinel
|
||||
// stays in history after the call, so without it every later round would
|
||||
// rewrite the list again.
|
||||
retodo atomic.Bool
|
||||
// asked latches the one reply that opens a question and, in the same round,
|
||||
// tries to write. Calling ask ends the round, so the write is the effect a
|
||||
// decision is holding back.
|
||||
asked atomic.Bool
|
||||
// Which fleets have been opened. A fleet's aggregate returns into the same
|
||||
// round that dispatched it, so without a latch the reply that follows reads
|
||||
// its own sentinel off the still-current user turn and dispatches again.
|
||||
fleets dispatched
|
||||
// opened says the arm's own fan-out has gone out, whichever sentinel opened
|
||||
// it: the terminal and derive arms then hold the turn rather than replying,
|
||||
// so the process still dies inside one.
|
||||
opened atomic.Bool
|
||||
// The scheduler arms dispatch two fleets and must not race them. held is
|
||||
// closed by the holder's own child, which only runs once its slot is
|
||||
// granted; release lets the arm free capacity while a refusal stands.
|
||||
holding sync.Once
|
||||
released sync.Once
|
||||
held chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
// dispatched is the set of fleets already opened, keyed by the sentinel that
|
||||
// opened each. One named state rather than a latch per fleet: they share a
|
||||
// lifetime and a meaning, and every new fan-out was adding another flag.
|
||||
type dispatched struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]bool
|
||||
}
|
||||
|
||||
// held reports whether something already recorded this name, without claiming
|
||||
// it. The lone-delegation arms read it to ask a negative: that a child the
|
||||
// scheduler should be holding back never reached the provider at all.
|
||||
func (d *dispatched) held(name string) bool {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.seen[name]
|
||||
}
|
||||
|
||||
// first reports whether this is the sentinel's first dispatch, and records it.
|
||||
func (d *dispatched) first(sentinel string) bool {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.seen[sentinel] {
|
||||
return false
|
||||
}
|
||||
if d.seen == nil {
|
||||
d.seen = map[string]bool{}
|
||||
}
|
||||
d.seen[sentinel] = true
|
||||
return true
|
||||
}
|
||||
|
||||
// releaseOnce frees the child waiting on release. Idempotent: an arm may reach
|
||||
// the moment to release through more than one path.
|
||||
func (s *scripted) releaseOnce() { s.released.Do(func() { close(s.release) }) }
|
||||
|
||||
func (s *scripted) Name() string { return "probe" }
|
||||
|
||||
func (s *scripted) Stream(ctx context.Context, req provider.Request) (<-chan provider.Chunk, error) {
|
||||
ch := make(chan provider.Chunk, 8)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for _, c := range s.script(ctx, req) {
|
||||
select {
|
||||
case ch <- c:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (s *scripted) script(ctx context.Context, req provider.Request) []provider.Chunk {
|
||||
if len(req.Tools) == 0 {
|
||||
return append(text(summaryBody), done())
|
||||
}
|
||||
// A delegated run answers first: the parent's transcript quotes every fleet
|
||||
// argument, so any later branch that scanned the conversation would make
|
||||
// the parent answer as one of its own children.
|
||||
if sentinel := childSentinel(req); sentinel != "" {
|
||||
return s.childScript(ctx, sentinel)
|
||||
}
|
||||
if chunks, ok := s.fanOut(req); ok {
|
||||
return chunks
|
||||
}
|
||||
if s.opens(req, sleepSentinel) {
|
||||
return append(sleepCall(), done())
|
||||
}
|
||||
if s.opensRun(req, taskSentinel) {
|
||||
if queuedTaskArm(s.arm) {
|
||||
// The holder's own child says when the ceiling is occupied.
|
||||
// Dispatching before that lets the delegation win the race and start.
|
||||
<-s.held
|
||||
}
|
||||
return append(loneTaskCall(s.arm), done())
|
||||
}
|
||||
if s.opensRun(req, ephemeralSentinel) {
|
||||
return append(ephemeralCall(s.arm), done())
|
||||
}
|
||||
if s.opensRun(req, skillSentinel) {
|
||||
if queuedSkillArm(s.arm) {
|
||||
// The holder's own child says when the ceiling is occupied.
|
||||
// Dispatching before that lets the skill win the race and start.
|
||||
<-s.held
|
||||
}
|
||||
return append(skillCall(s.arm), done())
|
||||
}
|
||||
// The one arm whose turn has to close signs the host's steps off first, one
|
||||
// per round, because a turn that called a tool cannot deliver while the list
|
||||
// is open. The latch is read last: a round that could not answer keeps it.
|
||||
if closesItsTurn(s.arm) && s.opened.Load() && hasTool(req.Tools, "complete_step") {
|
||||
for _, id := range probeStepIDs() {
|
||||
if s.fleets.first("sign-" + id) {
|
||||
return append(signStep(id), done())
|
||||
}
|
||||
}
|
||||
}
|
||||
// A terminal arm needs its fan-out to finish and its turn to stay open: a
|
||||
// closed turn would settle the marker, and the process would no longer be
|
||||
// dying inside one.
|
||||
if (terminalArm(s.arm) || deriveArm(s.arm)) && s.opened.Load() {
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
if s.turnCalls.Add(1) == 1 && hasTool(req.Tools, "todo_write") {
|
||||
return append(todoCall(firstTodos()), done())
|
||||
}
|
||||
if hasTool(req.Tools, "todo_write") && requestMentions(req, retodoSentinel) && !s.retodo.Swap(true) {
|
||||
return append(todoCall(secondTodos()), done())
|
||||
}
|
||||
if requestMentions(req, askSentinel) && !s.asked.Swap(true) {
|
||||
return append(askAndWrite(), done())
|
||||
}
|
||||
return append(text(echoMarker(req)+turnBody), done())
|
||||
}
|
||||
|
||||
// A turn is tagged twice: the user side by markerPattern, the assistant reply
|
||||
// by echoPattern. A fold folds assistant work first, so a one-sided tag would
|
||||
// let the surviving user turn hide the reply that was dropped beside it.
|
||||
var (
|
||||
markerPattern = regexp.MustCompile(`PROBE-MARK-\d+`)
|
||||
echoPattern = regexp.MustCompile(`PROBE-ECHO-\d+`)
|
||||
)
|
||||
|
||||
func marker(n int) string { return fmt.Sprintf("PROBE-MARK-%03d", n) }
|
||||
|
||||
// markersIn returns the tags a message set still shows, in order.
|
||||
func markersIn(texts []string, pat *regexp.Regexp) []string {
|
||||
var out []string
|
||||
for _, t := range texts {
|
||||
out = append(out, pat.FindAllString(t, -1)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// echoMarker answers a turn with the assistant-side tag for the same number.
|
||||
func echoMarker(req provider.Request) string {
|
||||
for _, m := range slices.Backward(req.Messages) {
|
||||
found := markerPattern.FindAllString(m.Content, -1)
|
||||
if len(found) == 0 {
|
||||
continue
|
||||
}
|
||||
return "PROBE-ECHO-" + found[len(found)-1][len("PROBE-MARK-"):] + " reply. "
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func hasTool(schemas []provider.ToolSchema, name string) bool {
|
||||
for _, t := range schemas {
|
||||
if t.Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func text(s string) []provider.Chunk {
|
||||
return []provider.Chunk{{Type: provider.ChunkText, Text: s}}
|
||||
}
|
||||
|
||||
func done() provider.Chunk { return provider.Chunk{Type: provider.ChunkDone} }
|
||||
|
||||
// todoCall writes the list the resume phase looks for. The step ids are the
|
||||
// point: they are the host identity a completion has to cite. A first write
|
||||
// may not open with a completed item — the host rejects that shape, and a
|
||||
// rejected call would leave the row unmeasured rather than answered.
|
||||
func todoCall(todos []map[string]any) []provider.Chunk {
|
||||
args, _ := json.Marshal(map[string]any{"todos": todos})
|
||||
return []provider.Chunk{{
|
||||
Type: provider.ChunkToolCall,
|
||||
ToolCall: &provider.ToolCall{ID: "probe_todo_" + strconv.Itoa(len(todos)), Name: "todo_write", Arguments: string(args)},
|
||||
}}
|
||||
}
|
||||
|
||||
// retodoSentinel asks the scripted reply to replace the task list, so a probe
|
||||
// can move host step identity from one set of ids to another.
|
||||
const retodoSentinel = "PROBE-RETODO"
|
||||
|
||||
// askSentinel asks for a question the host must put to a person, alongside the
|
||||
// write that question is holding back.
|
||||
const (
|
||||
askSentinel = "PROBE-ASK"
|
||||
deferredEffect = "probe-deferred-effect.txt"
|
||||
deferredContents = "this write was not held back by the decision"
|
||||
)
|
||||
|
||||
// askAndWrite opens a question and asks to write in the same round. The round
|
||||
// ends at the question, so the write must not run — not while the answer is
|
||||
// pending, and not because a later process reloaded the session.
|
||||
func askAndWrite() []provider.Chunk {
|
||||
ask, _ := json.Marshal(map[string]any{"questions": []map[string]any{{
|
||||
"header": "Boundary",
|
||||
"question": "Which side of the fold should the probe measure?",
|
||||
"reason": "user_decision",
|
||||
"options": []map[string]any{
|
||||
{"label": "Below the fold", "description": "removes folded history"},
|
||||
{"label": "Above the fold", "description": "leaves coverage intact"},
|
||||
},
|
||||
}}})
|
||||
write, _ := json.Marshal(map[string]string{"path": deferredEffect, "content": deferredContents})
|
||||
return []provider.Chunk{
|
||||
{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{ID: "probe_ask_1", Name: "ask", Arguments: string(ask)}},
|
||||
{Type: provider.ChunkToolCall, ToolCall: &provider.ToolCall{ID: "probe_write_1", Name: "write_file", Arguments: string(write)}},
|
||||
}
|
||||
}
|
||||
|
||||
func requestMentions(req provider.Request, needle string) bool {
|
||||
for _, m := range slices.Backward(req.Messages) {
|
||||
if strings.Contains(m.Content, needle) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func firstTodos() []map[string]any {
|
||||
return []map[string]any{
|
||||
{"content": "Establish probe state", "status": "in_progress", "activeForm": "Establishing probe state", "step_id": "probe_step_01"},
|
||||
{"content": "Cross the process boundary", "status": "pending", "step_id": "probe_step_02"},
|
||||
{"content": "Report the resurrection matrix", "status": "pending", "step_id": "probe_step_03"},
|
||||
}
|
||||
}
|
||||
|
||||
// secondTodos replaces the pending work while the in_progress item stays: the
|
||||
// host refuses to drop an item that is in flight, so this is what a list
|
||||
// rewrite actually looks like. The two new ids are what a stale note misses.
|
||||
func secondTodos() []map[string]any {
|
||||
return []map[string]any{
|
||||
{"content": "Establish probe state", "status": "in_progress", "activeForm": "Establishing probe state", "step_id": "probe_step_01"},
|
||||
{"content": "Split the frozen body", "status": "pending", "step_id": "probe_step_21"},
|
||||
{"content": "Hold the ordering contract", "status": "pending", "step_id": "probe_step_22"},
|
||||
}
|
||||
}
|
||||
|
||||
// turnBody is fat on purpose: the fold keeps a recent verbatim tail, so thin
|
||||
// replies leave no region between the pinned head and that tail.
|
||||
var turnBody = "Probe state established. " + strings.Repeat("The probe records deterministic filler so the fold has material to work on. ", 60)
|
||||
|
||||
var summaryBody = strings.Join([]string{
|
||||
"## Standing facts & constraints",
|
||||
"The runtime-resume probe is measuring what survives a process boundary.",
|
||||
"",
|
||||
"## Goal",
|
||||
"Establish host state, exit, and observe what a new process can still prove.",
|
||||
}, "\n")
|
||||
|
||||
func resolverFor(prov *scripted) *provider.StaticResolver {
|
||||
return &provider.StaticResolver{
|
||||
Descriptors: []provider.Descriptor{{
|
||||
Ref: probeModelRef, DisplayName: "probe", Model: "scripted",
|
||||
ContextWindow: 128_000, Tools: true,
|
||||
}, {
|
||||
Ref: probeAltModelRef, DisplayName: "probe-alt", Model: "scripted-alt",
|
||||
ContextWindow: 128_000, Tools: true,
|
||||
}},
|
||||
Providers: map[string]provider.Provider{probeModelRef: prov, probeAltModelRef: prov},
|
||||
}
|
||||
}
|
||||
|
||||
func errUnexpected(what string, got any) error {
|
||||
return fmt.Errorf("probe could not establish %s (got %v); the arm measures nothing and is reported invalid", what, got)
|
||||
}
|
||||
Loaded 100 of 2453 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user