Compare commits

...
3 changed files with 164 additions and 125 deletions

No files matched your search

+52 -122
View File
@@ -1,91 +1,11 @@
name: Tests
# Builds, type-checks and runs every test suite on a clean GitHub runner, and records the file
# list each published package would contain. Each suite is its own job so one failure does not
# hide the others. No provider credentials or operator configuration are supplied.
# Package suites deny external networking; scenario runtime has no external route.
on:
push:
branches: [main]
pull_request:
branches: [main]
workflow_dispatch:
on: workflow_dispatch
permissions:
contents: read
concurrency:
group: tests-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build-and-package:
o1-transport-checkpoint:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-node@v4
with:
node-version: 22
cache: npm
- run: npm ci
- run: npm run build
- name: Record the published file list of each package
run: |
set -euo pipefail
mkdir -p pack-lists
for p in cli daemon tui; do
(cd packages/$p && npm pack --dry-run --json --ignore-scripts) \
| node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{for(const f of JSON.parse(s)[0].files)console.log(f.path)})' \
| sort > pack-lists/$p.txt
echo "$p: $(wc -l < pack-lists/$p.txt) files"
done
- uses: actions/upload-artifact@v4
with:
name: pack-lists
path: pack-lists/
typecheck:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-node@v4
with:
node-version: 22
cache: npm
- run: npm ci
- run: npm run lint
repo-checks:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-node@v4
with:
node-version: 22
cache: npm
- run: npm ci
- run: npm run test:repo
package-tests:
if: ${{ github.event_name != 'push' }}
# macOS, the platform OpenRig runs on: several daemon and cli tests read macOS ps columns,
# file-inode behaviour and release tags.
runs-on: macos-latest
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
package: [daemon, cli, tui, ui]
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
with:
@@ -95,48 +15,58 @@ jobs:
with:
node-version: 22
cache: npm
- run: brew install tmux
- run: npm ci
- name: Build the packages the tests import
run: npm run build
- name: Run the suite without ambient credentials or external network
- name: Record source bindings and retain candidate
run: |
suite_root=$(mktemp -d "$RUNNER_TEMP/opr-test.XXXXXX")
suite_root=$(cd "$suite_root" && pwd -P)
mkdir -p "$suite_root/home" "$suite_root/bin"
# Same system bytes, nonprivileged mode; no system file is changed.
/usr/bin/install -m 0755 /bin/ps "$suite_root/bin/ps"
# Only this job's temporary sockets (including tmux) are reachable.
env -i HOME="$suite_root/home" PATH="$PATH" TMPDIR="$suite_root" TMUX_TMPDIR="$suite_root" \
CI=true TERM=xterm-256color \
sandbox-exec -D IPC_ROOT="$suite_root" -p '(version 1)(allow default)(deny network-outbound)(allow network-outbound (remote ip "localhost:*"))(allow network-outbound (subpath (param "IPC_ROOT")))' \
/bin/bash -s "${{ matrix.package }}" <<'SUITE'
set -euo pipefail
node scripts/ci-macos-process-probe.mjs
# The production readers use bare ps. Select real system bytes only after
# self/owned-child observation and network/IPC confinement are proved.
export PATH="$TMPDIR/bin:$PATH"
test "$(command -v ps)" = "$TMPDIR/bin/ps"
exec npm run test -w "packages/$1"
SUITE
installed-scenario:
runs-on: ubuntu-latest
timeout-minutes: 35
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-node@v4
with:
node-version: 22
cache: npm
- run: npm ci
- name: Installed package, daemon restart, and seeded durability failure
run: bash scripts/run-pr-scenarios.sh
mkdir -p "$RUNNER_TEMP/o1-evidence" "$RUNNER_TEMP/o1-candidate"
git rev-parse HEAD HEAD^{tree} > "$RUNNER_TEMP/o1-evidence/candidate-binding.txt"
git rev-parse ae65fe7865c20a32bb503f7e88e39ead08826154^{tree} > "$RUNNER_TEMP/o1-evidence/base-binding.txt"
sha256sum packages/daemon/test/issue498-o1-checkpoint.test.ts > "$RUNNER_TEMP/o1-evidence/fixture-sha256.txt"
cp packages/daemon/src/domain/session-transport.ts "$RUNNER_TEMP/o1-candidate/session-transport.ts"
git diff ae65fe7865c20a32bb503f7e88e39ead08826154 -- packages/daemon/src/domain/session-transport.ts > "$RUNNER_TEMP/o1-evidence/candidate.patch"
- name: Measure current main through the production transport seam
id: baseline
continue-on-error: true
run: |
git show ae65fe7865c20a32bb503f7e88e39ead08826154:packages/daemon/src/domain/session-transport.ts > packages/daemon/src/domain/session-transport.ts
CHECKPOINT_OUTPUT="$RUNNER_TEMP/o1-evidence/base" npm exec -w packages/daemon -- vitest run test/issue498-o1-checkpoint.test.ts --reporter verbose
- name: Measure the bounded candidate with identical fixture bytes
id: candidate
continue-on-error: true
run: |
cp "$RUNNER_TEMP/o1-candidate/session-transport.ts" packages/daemon/src/domain/session-transport.ts
CHECKPOINT_OUTPUT="$RUNNER_TEMP/o1-evidence/candidate" npm exec -w packages/daemon -- vitest run test/issue498-o1-checkpoint.test.ts --reporter verbose
- name: Compare recorded effects without calling ordinary refusals a success
if: always()
env:
BASE_OUTCOME: ${{ steps.baseline.outcome }}
CANDIDATE_OUTCOME: ${{ steps.candidate.outcome }}
run: |
python3 - <<'PYCODE'
import json,os,pathlib
root=pathlib.Path(os.environ['RUNNER_TEMP'])/'o1-evidence'
base=json.loads((root/'base/observations.json').read_text())
candidate=json.loads((root/'candidate/observations.json').read_text())
assert len(base)==len(candidate)==36
rows=[]
for a,b in zip(base,candidate):
assert (a['id'],a['hook'],a['pane'],a['intent'])==(b['id'],b['hook'],b['pane'],b['intent'])
rows.append({'id':b['id'],'hook':b['hook'],'intent':b['intent'],
'base_ok':a['result']['ok'],'candidate_ok':b['result']['ok'],
'base_reason':a['result'].get('reason'),'candidate_reason':b['result'].get('reason'),
'base_pastes':a['pasteCount'],'candidate_pastes':b['pasteCount'],
'base_keys':a['keysCount'],'candidate_keys':b['keysCount']})
regressions=[r for r in rows if r['intent']=='deliver' and r['base_ok'] and not r['candidate_ok']]
report={'measurement_steps':[os.environ['BASE_OUTCOME'],os.environ['CANDIDATE_OUTCOME']],
'rows':rows,'new_ordinary_refusals':regressions,'stop_for_tradeoff':bool(regressions)}
(root/'comparison.json').write_text(json.dumps(report,indent=2)+'\n')
print(json.dumps({'records_per_cut':len(base),'new_ordinary_refusals':len(regressions),
'stop_for_tradeoff':bool(regressions),'measurement_steps':report['measurement_steps']}))
assert report['measurement_steps']==['success','success']
PYCODE
- uses: actions/upload-artifact@v4
if: always()
with:
name: installed-scenario-evidence
path: dist/pr-scenarios/
name: issue498-o1-checkpoint
path: ${{ runner.temp }}/o1-evidence/
if-no-files-found: error
@@ -3201,8 +3201,10 @@ export class SessionTransport {
hookActivity.reason !== "permission_request_pending" &&
hookActivity.reason !== "permission_prompt_cleared"
) {
// Fresh hook (within the 15s send window): authoritative for any state.
if (this.hookFreshForSend(hookActivity, now)) {
const freshForSend = this.hookFreshForSend(hookActivity, now);
// A fresh positive prompt or unknown hook keeps its existing verdict. Idle/running hooks
// still allow delivery, but cannot override the pane's positive current-prompt veto.
if (freshForSend && hookActivity.state !== "idle" && hookActivity.state !== "running") {
return hookActivity;
}
// OPR.0.4.3.28 Part A — a stale-but-latest `idle` hook (older than the 15s
@@ -3213,7 +3215,7 @@ export class SessionTransport {
// A stale NON-idle hook (running/needs_input >15s) still falls through to
// the pane probe below. No 15s widen; scoped to stale!==true so a
// truly-abandoned seat (>5min) still degrades to the probe.
if (hookActivity.state === "idle") {
if (hookActivity.state === "idle" || (freshForSend && hookActivity.state === "running")) {
// Guard code-review 2026-07-02 (Blocker 1): a NARROW real-time veto
// before trusting the stale-idle hook — never paste+Enter onto a VISIBLE
// picker/permission prompt. Only a POSITIVE needs_input from the pane
@@ -0,0 +1,107 @@
// Bounded O1 checkpoint, not a native-runtime or cause-2 acceptance test.
// Real SessionTransport + SQLite hook store; captures/input effects are explicitly modeled.
import { afterAll, describe, expect, it, vi } from "vitest";
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { RigRepository } from "../src/domain/rig-repository.js";
import { SessionRegistry } from "../src/domain/session-registry.js";
import { SessionTransport, classifyPaneActivity } from "../src/domain/session-transport.js";
import { AgentActivityStore } from "../src/domain/agent-activity-store.js";
import { EventBus } from "../src/domain/event-bus.js";
import type { TmuxAdapter } from "../src/adapters/tmux.js";
import { createFullTestDb } from "./helpers/test-app.js";
const idle = "Done.\n❯ \n ⏵⏵ accept edits on (shift+tab to cycle)";
const choice = "Choose a color\n❯ 1. RED\n 2. BLUE\nEnter to select";
const equalRender = "Earlier result\n❯ 1. Fix the bug\n 2. Explain it\n❯ \n⏵⏵ accept edits on";
const cases = [
{ id: "ordinary-idle", intent: "deliver", pane: idle },
{ id: "ordinary-working", intent: "deliver", pane: "Working on task...\n⠋ Processing\nesc to interrupt" },
{ id: "unknown", intent: "deliver", pane: "xyzzy no prompt here" },
{ id: "null-capture", intent: "deliver", pane: null },
{ id: "failed-capture", intent: "deliver", pane: idle, throws: true },
{ id: "current-choice", intent: "refuse", pane: choice },
{ id: "current-permission", intent: "refuse", pane: "Do you want to allow this operation?\n❯ 1. Yes\n 2. No" },
{ id: "permission-question-only", intent: "refuse", pane: "Do you want to proceed with this deploy?\n gpt-5.5 · Context [████ ] · ~/code" },
{ id: "plain-history", intent: "deliver", pane: "Earlier request: 1. Fix the bug\n• Done.\n" + idle },
{ id: "marked-history", intent: "deliver", pane: "❯ 1. Fix the bug\n• Done.\n" + idle },
{ id: "quoted-menu", intent: "deliver", pane: "Report of the previous dialog:\n❯ 1. Yes\n 2. No\nThis is quoted text, not an active dialog.\n" + idle },
{ id: "literal-permission-text", intent: "deliver", pane: "The test fixture literally prints: Do you want to run the tests?\n• Done.\n" + idle },
{ id: "multiline-history", intent: "deliver", pane: "❯ 1. RED\n2. BLUE\n• Done.\n" + idle },
{ id: "identical-render-current", intent: "refuse", pane: equalRender },
{ id: "identical-render-history", intent: "deliver", pane: equalRender },
{ id: "late-choice-after-check", intent: "residual", pane: idle, late: true },
{ id: "explicit-override", intent: "deliver", pane: choice, override: true },
{ id: "force-current-choice", intent: "refuse", pane: choice, force: true },
] as const;
const records: unknown[] = [];
afterAll(() => {
const dir = process.env.CHECKPOINT_OUTPUT;
if (!dir) throw new Error("CHECKPOINT_OUTPUT must name this CI job's private evidence directory");
mkdirSync(dir, { recursive: true });
writeFileSync(join(dir, "observations.json"), JSON.stringify(records, null, 2) + "\n");
});
describe("498 O1 same-input checkpoint", () => {
for (const hook of ["Stop", "UserPromptSubmit"] as const) {
it.each(cases)(`${hook}: $id`, async fixture => {
const db = createFullTestDb();
try {
const now = new Date("2026-10-19T13:00:00.000Z");
const rigRepo = new RigRepository(db);
const sessionRegistry = new SessionRegistry(db);
const eventBus = new EventBus(db);
const store = new AgentActivityStore({ db, eventBus });
const rig = rigRepo.createRig("checkpoint");
const node = rigRepo.addNode(rig.id, "dev.test", { role: "worker", runtime: "claude-code" });
const session = sessionRegistry.registerSession(node.id, "dev-test@checkpoint");
sessionRegistry.updateStatus(session.id, "running");
sessionRegistry.updateBinding(node.id, { tmuxSession: "dev-test@checkpoint" });
store.recordHookEvent({ runtime: "claude-code", sessionName: "dev-test@checkpoint", hookEvent: hook,
occurredAt: new Date(now.getTime() - 1000).toISOString() });
const effects: Array<{ kind: string; value?: unknown }> = [];
let pane: string | null = fixture.pane;
const capture = vi.fn(async () => {
effects.push({ kind: "capture" });
if ("throws" in fixture) throw new Error("controlled capture failure");
return pane;
});
const sendText = vi.fn(async (_target: string, text: string) => {
effects.push({ kind: "paste", value: text });
if ("late" in fixture) pane = choice;
return { ok: true as const };
});
const sendKeys = vi.fn(async (_target: string, keys: string[]) => {
effects.push({ kind: "keys", value: keys }); return { ok: true as const };
});
const tmux = {
hasSession: async () => true,
probeSession: async () => ({ state: "present" as const }),
capturePaneContent: capture, sendText, sendKeys,
getPaneCommand: async () => null, getPanePid: async () => null,
listSessions: async () => [], listWindows: async () => [], listPanes: async () => [],
createSession: async () => ({ ok: true as const }), killSession: async () => ({ ok: true as const }),
startPipePane: async () => ({ ok: true as const }), stopPipePane: async () => ({ ok: true as const }),
} as unknown as TmuxAdapter;
const transport = new SessionTransport({ db, rigRepo, sessionRegistry, eventBus,
agentActivityStore: store, tmuxAdapter: tmux, now: () => now, sleep: async () => {} });
const result = await transport.send("dev-test@checkpoint", "Status update from a peer", {
...("force" in fixture ? { force: true } : {}),
...("override" in fixture ? { dangerouslyInteract: true, reason: "deliberate synthetic input", actorSession: "peer@checkpoint" } : {}),
});
records.push({ id: fixture.id, hook, intent: fixture.intent, pane: fixture.pane,
paneVerdict: classifyPaneActivity(fixture.pane ?? ""),
hookVerdict: store.getLatestForNode({ sessionName: "dev-test@checkpoint", now }),
result, effects, pasteCount: sendText.mock.calls.length, keysCount: sendKeys.mock.calls.length,
overrideCount: (db.prepare("SELECT COUNT(*) AS n FROM events WHERE type='transport.prompt_override'").get() as { n: number }).n });
// Check measurement integrity only. Policy comparisons consume every recorded result offline.
expect(sendText.mock.calls.length).toBeLessThanOrEqual(1);
if (!result.ok) {
expect(sendText).not.toHaveBeenCalled();
expect(sendKeys).not.toHaveBeenCalled();
} else expect(sendText).toHaveBeenCalledTimes(1);
} finally { db.close(); }
});
}
});