mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-11 17:32:22 -10:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
41990912f2 | ||
|
|
c45eef811a |
No files matched your search
+52
-122
@@ -1,91 +1,11 @@
|
||||
name: Tests
|
||||
|
||||
# Builds, type-checks and runs every test suite on a clean GitHub runner, and records the file
|
||||
# list each published package would contain. Each suite is its own job so one failure does not
|
||||
# hide the others. No provider credentials or operator configuration are supplied.
|
||||
# Package suites deny external networking; scenario runtime has no external route.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
on: workflow_dispatch
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: tests-${{ github.event_name }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-and-package:
|
||||
o1-transport-checkpoint:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npm run build
|
||||
- name: Record the published file list of each package
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p pack-lists
|
||||
for p in cli daemon tui; do
|
||||
(cd packages/$p && npm pack --dry-run --json --ignore-scripts) \
|
||||
| node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{for(const f of JSON.parse(s)[0].files)console.log(f.path)})' \
|
||||
| sort > pack-lists/$p.txt
|
||||
echo "$p: $(wc -l < pack-lists/$p.txt) files"
|
||||
done
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: pack-lists
|
||||
path: pack-lists/
|
||||
|
||||
typecheck:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npm run lint
|
||||
|
||||
repo-checks:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npm run test:repo
|
||||
|
||||
package-tests:
|
||||
if: ${{ github.event_name != 'push' }}
|
||||
# macOS, the platform OpenRig runs on: several daemon and cli tests read macOS ps columns,
|
||||
# file-inode behaviour and release tags.
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
package: [daemon, cli, tui, ui]
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
@@ -95,48 +15,58 @@ jobs:
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
- run: brew install tmux
|
||||
- run: npm ci
|
||||
- name: Build the packages the tests import
|
||||
run: npm run build
|
||||
- name: Run the suite without ambient credentials or external network
|
||||
- name: Record source bindings and retain candidate
|
||||
run: |
|
||||
suite_root=$(mktemp -d "$RUNNER_TEMP/opr-test.XXXXXX")
|
||||
suite_root=$(cd "$suite_root" && pwd -P)
|
||||
mkdir -p "$suite_root/home" "$suite_root/bin"
|
||||
# Same system bytes, nonprivileged mode; no system file is changed.
|
||||
/usr/bin/install -m 0755 /bin/ps "$suite_root/bin/ps"
|
||||
# Only this job's temporary sockets (including tmux) are reachable.
|
||||
env -i HOME="$suite_root/home" PATH="$PATH" TMPDIR="$suite_root" TMUX_TMPDIR="$suite_root" \
|
||||
CI=true TERM=xterm-256color \
|
||||
sandbox-exec -D IPC_ROOT="$suite_root" -p '(version 1)(allow default)(deny network-outbound)(allow network-outbound (remote ip "localhost:*"))(allow network-outbound (subpath (param "IPC_ROOT")))' \
|
||||
/bin/bash -s "${{ matrix.package }}" <<'SUITE'
|
||||
set -euo pipefail
|
||||
node scripts/ci-macos-process-probe.mjs
|
||||
# The production readers use bare ps. Select real system bytes only after
|
||||
# self/owned-child observation and network/IPC confinement are proved.
|
||||
export PATH="$TMPDIR/bin:$PATH"
|
||||
test "$(command -v ps)" = "$TMPDIR/bin/ps"
|
||||
exec npm run test -w "packages/$1"
|
||||
SUITE
|
||||
|
||||
installed-scenario:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 35
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- name: Installed package, daemon restart, and seeded durability failure
|
||||
run: bash scripts/run-pr-scenarios.sh
|
||||
mkdir -p "$RUNNER_TEMP/o1-evidence" "$RUNNER_TEMP/o1-candidate"
|
||||
git rev-parse HEAD HEAD^{tree} > "$RUNNER_TEMP/o1-evidence/candidate-binding.txt"
|
||||
git rev-parse ae65fe7865c20a32bb503f7e88e39ead08826154^{tree} > "$RUNNER_TEMP/o1-evidence/base-binding.txt"
|
||||
sha256sum packages/daemon/test/issue498-o1-checkpoint.test.ts > "$RUNNER_TEMP/o1-evidence/fixture-sha256.txt"
|
||||
cp packages/daemon/src/domain/session-transport.ts "$RUNNER_TEMP/o1-candidate/session-transport.ts"
|
||||
git diff ae65fe7865c20a32bb503f7e88e39ead08826154 -- packages/daemon/src/domain/session-transport.ts > "$RUNNER_TEMP/o1-evidence/candidate.patch"
|
||||
- name: Measure current main through the production transport seam
|
||||
id: baseline
|
||||
continue-on-error: true
|
||||
run: |
|
||||
git show ae65fe7865c20a32bb503f7e88e39ead08826154:packages/daemon/src/domain/session-transport.ts > packages/daemon/src/domain/session-transport.ts
|
||||
CHECKPOINT_OUTPUT="$RUNNER_TEMP/o1-evidence/base" npm exec -w packages/daemon -- vitest run test/issue498-o1-checkpoint.test.ts --reporter verbose
|
||||
- name: Measure the bounded candidate with identical fixture bytes
|
||||
id: candidate
|
||||
continue-on-error: true
|
||||
run: |
|
||||
cp "$RUNNER_TEMP/o1-candidate/session-transport.ts" packages/daemon/src/domain/session-transport.ts
|
||||
CHECKPOINT_OUTPUT="$RUNNER_TEMP/o1-evidence/candidate" npm exec -w packages/daemon -- vitest run test/issue498-o1-checkpoint.test.ts --reporter verbose
|
||||
- name: Compare recorded effects without calling ordinary refusals a success
|
||||
if: always()
|
||||
env:
|
||||
BASE_OUTCOME: ${{ steps.baseline.outcome }}
|
||||
CANDIDATE_OUTCOME: ${{ steps.candidate.outcome }}
|
||||
run: |
|
||||
python3 - <<'PYCODE'
|
||||
import json,os,pathlib
|
||||
root=pathlib.Path(os.environ['RUNNER_TEMP'])/'o1-evidence'
|
||||
base=json.loads((root/'base/observations.json').read_text())
|
||||
candidate=json.loads((root/'candidate/observations.json').read_text())
|
||||
assert len(base)==len(candidate)==36
|
||||
rows=[]
|
||||
for a,b in zip(base,candidate):
|
||||
assert (a['id'],a['hook'],a['pane'],a['intent'])==(b['id'],b['hook'],b['pane'],b['intent'])
|
||||
rows.append({'id':b['id'],'hook':b['hook'],'intent':b['intent'],
|
||||
'base_ok':a['result']['ok'],'candidate_ok':b['result']['ok'],
|
||||
'base_reason':a['result'].get('reason'),'candidate_reason':b['result'].get('reason'),
|
||||
'base_pastes':a['pasteCount'],'candidate_pastes':b['pasteCount'],
|
||||
'base_keys':a['keysCount'],'candidate_keys':b['keysCount']})
|
||||
regressions=[r for r in rows if r['intent']=='deliver' and r['base_ok'] and not r['candidate_ok']]
|
||||
report={'measurement_steps':[os.environ['BASE_OUTCOME'],os.environ['CANDIDATE_OUTCOME']],
|
||||
'rows':rows,'new_ordinary_refusals':regressions,'stop_for_tradeoff':bool(regressions)}
|
||||
(root/'comparison.json').write_text(json.dumps(report,indent=2)+'\n')
|
||||
print(json.dumps({'records_per_cut':len(base),'new_ordinary_refusals':len(regressions),
|
||||
'stop_for_tradeoff':bool(regressions),'measurement_steps':report['measurement_steps']}))
|
||||
assert report['measurement_steps']==['success','success']
|
||||
PYCODE
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: installed-scenario-evidence
|
||||
path: dist/pr-scenarios/
|
||||
name: issue498-o1-checkpoint
|
||||
path: ${{ runner.temp }}/o1-evidence/
|
||||
if-no-files-found: error
|
||||
@@ -3201,8 +3201,10 @@ export class SessionTransport {
|
||||
hookActivity.reason !== "permission_request_pending" &&
|
||||
hookActivity.reason !== "permission_prompt_cleared"
|
||||
) {
|
||||
// Fresh hook (within the 15s send window): authoritative for any state.
|
||||
if (this.hookFreshForSend(hookActivity, now)) {
|
||||
const freshForSend = this.hookFreshForSend(hookActivity, now);
|
||||
// A fresh positive prompt or unknown hook keeps its existing verdict. Idle/running hooks
|
||||
// still allow delivery, but cannot override the pane's positive current-prompt veto.
|
||||
if (freshForSend && hookActivity.state !== "idle" && hookActivity.state !== "running") {
|
||||
return hookActivity;
|
||||
}
|
||||
// OPR.0.4.3.28 Part A — a stale-but-latest `idle` hook (older than the 15s
|
||||
@@ -3213,7 +3215,7 @@ export class SessionTransport {
|
||||
// A stale NON-idle hook (running/needs_input >15s) still falls through to
|
||||
// the pane probe below. No 15s widen; scoped to stale!==true so a
|
||||
// truly-abandoned seat (>5min) still degrades to the probe.
|
||||
if (hookActivity.state === "idle") {
|
||||
if (hookActivity.state === "idle" || (freshForSend && hookActivity.state === "running")) {
|
||||
// Guard code-review 2026-07-02 (Blocker 1): a NARROW real-time veto
|
||||
// before trusting the stale-idle hook — never paste+Enter onto a VISIBLE
|
||||
// picker/permission prompt. Only a POSITIVE needs_input from the pane
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
// Bounded O1 checkpoint, not a native-runtime or cause-2 acceptance test.
|
||||
// Real SessionTransport + SQLite hook store; captures/input effects are explicitly modeled.
|
||||
import { afterAll, describe, expect, it, vi } from "vitest";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { RigRepository } from "../src/domain/rig-repository.js";
|
||||
import { SessionRegistry } from "../src/domain/session-registry.js";
|
||||
import { SessionTransport, classifyPaneActivity } from "../src/domain/session-transport.js";
|
||||
import { AgentActivityStore } from "../src/domain/agent-activity-store.js";
|
||||
import { EventBus } from "../src/domain/event-bus.js";
|
||||
import type { TmuxAdapter } from "../src/adapters/tmux.js";
|
||||
import { createFullTestDb } from "./helpers/test-app.js";
|
||||
|
||||
const idle = "Done.\n❯ \n ⏵⏵ accept edits on (shift+tab to cycle)";
|
||||
const choice = "Choose a color\n❯ 1. RED\n 2. BLUE\nEnter to select";
|
||||
const equalRender = "Earlier result\n❯ 1. Fix the bug\n 2. Explain it\n❯ \n⏵⏵ accept edits on";
|
||||
const cases = [
|
||||
{ id: "ordinary-idle", intent: "deliver", pane: idle },
|
||||
{ id: "ordinary-working", intent: "deliver", pane: "Working on task...\n⠋ Processing\nesc to interrupt" },
|
||||
{ id: "unknown", intent: "deliver", pane: "xyzzy no prompt here" },
|
||||
{ id: "null-capture", intent: "deliver", pane: null },
|
||||
{ id: "failed-capture", intent: "deliver", pane: idle, throws: true },
|
||||
{ id: "current-choice", intent: "refuse", pane: choice },
|
||||
{ id: "current-permission", intent: "refuse", pane: "Do you want to allow this operation?\n❯ 1. Yes\n 2. No" },
|
||||
{ id: "permission-question-only", intent: "refuse", pane: "Do you want to proceed with this deploy?\n gpt-5.5 · Context [████ ] · ~/code" },
|
||||
{ id: "plain-history", intent: "deliver", pane: "Earlier request: 1. Fix the bug\n• Done.\n" + idle },
|
||||
{ id: "marked-history", intent: "deliver", pane: "❯ 1. Fix the bug\n• Done.\n" + idle },
|
||||
{ id: "quoted-menu", intent: "deliver", pane: "Report of the previous dialog:\n❯ 1. Yes\n 2. No\nThis is quoted text, not an active dialog.\n" + idle },
|
||||
{ id: "literal-permission-text", intent: "deliver", pane: "The test fixture literally prints: Do you want to run the tests?\n• Done.\n" + idle },
|
||||
{ id: "multiline-history", intent: "deliver", pane: "❯ 1. RED\n2. BLUE\n• Done.\n" + idle },
|
||||
{ id: "identical-render-current", intent: "refuse", pane: equalRender },
|
||||
{ id: "identical-render-history", intent: "deliver", pane: equalRender },
|
||||
{ id: "late-choice-after-check", intent: "residual", pane: idle, late: true },
|
||||
{ id: "explicit-override", intent: "deliver", pane: choice, override: true },
|
||||
{ id: "force-current-choice", intent: "refuse", pane: choice, force: true },
|
||||
] as const;
|
||||
|
||||
const records: unknown[] = [];
|
||||
afterAll(() => {
|
||||
const dir = process.env.CHECKPOINT_OUTPUT;
|
||||
if (!dir) throw new Error("CHECKPOINT_OUTPUT must name this CI job's private evidence directory");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(dir, "observations.json"), JSON.stringify(records, null, 2) + "\n");
|
||||
});
|
||||
|
||||
describe("498 O1 same-input checkpoint", () => {
|
||||
for (const hook of ["Stop", "UserPromptSubmit"] as const) {
|
||||
it.each(cases)(`${hook}: $id`, async fixture => {
|
||||
const db = createFullTestDb();
|
||||
try {
|
||||
const now = new Date("2026-10-19T13:00:00.000Z");
|
||||
const rigRepo = new RigRepository(db);
|
||||
const sessionRegistry = new SessionRegistry(db);
|
||||
const eventBus = new EventBus(db);
|
||||
const store = new AgentActivityStore({ db, eventBus });
|
||||
const rig = rigRepo.createRig("checkpoint");
|
||||
const node = rigRepo.addNode(rig.id, "dev.test", { role: "worker", runtime: "claude-code" });
|
||||
const session = sessionRegistry.registerSession(node.id, "dev-test@checkpoint");
|
||||
sessionRegistry.updateStatus(session.id, "running");
|
||||
sessionRegistry.updateBinding(node.id, { tmuxSession: "dev-test@checkpoint" });
|
||||
store.recordHookEvent({ runtime: "claude-code", sessionName: "dev-test@checkpoint", hookEvent: hook,
|
||||
occurredAt: new Date(now.getTime() - 1000).toISOString() });
|
||||
const effects: Array<{ kind: string; value?: unknown }> = [];
|
||||
let pane: string | null = fixture.pane;
|
||||
const capture = vi.fn(async () => {
|
||||
effects.push({ kind: "capture" });
|
||||
if ("throws" in fixture) throw new Error("controlled capture failure");
|
||||
return pane;
|
||||
});
|
||||
const sendText = vi.fn(async (_target: string, text: string) => {
|
||||
effects.push({ kind: "paste", value: text });
|
||||
if ("late" in fixture) pane = choice;
|
||||
return { ok: true as const };
|
||||
});
|
||||
const sendKeys = vi.fn(async (_target: string, keys: string[]) => {
|
||||
effects.push({ kind: "keys", value: keys }); return { ok: true as const };
|
||||
});
|
||||
const tmux = {
|
||||
hasSession: async () => true,
|
||||
probeSession: async () => ({ state: "present" as const }),
|
||||
capturePaneContent: capture, sendText, sendKeys,
|
||||
getPaneCommand: async () => null, getPanePid: async () => null,
|
||||
listSessions: async () => [], listWindows: async () => [], listPanes: async () => [],
|
||||
createSession: async () => ({ ok: true as const }), killSession: async () => ({ ok: true as const }),
|
||||
startPipePane: async () => ({ ok: true as const }), stopPipePane: async () => ({ ok: true as const }),
|
||||
} as unknown as TmuxAdapter;
|
||||
const transport = new SessionTransport({ db, rigRepo, sessionRegistry, eventBus,
|
||||
agentActivityStore: store, tmuxAdapter: tmux, now: () => now, sleep: async () => {} });
|
||||
const result = await transport.send("dev-test@checkpoint", "Status update from a peer", {
|
||||
...("force" in fixture ? { force: true } : {}),
|
||||
...("override" in fixture ? { dangerouslyInteract: true, reason: "deliberate synthetic input", actorSession: "peer@checkpoint" } : {}),
|
||||
});
|
||||
records.push({ id: fixture.id, hook, intent: fixture.intent, pane: fixture.pane,
|
||||
paneVerdict: classifyPaneActivity(fixture.pane ?? ""),
|
||||
hookVerdict: store.getLatestForNode({ sessionName: "dev-test@checkpoint", now }),
|
||||
result, effects, pasteCount: sendText.mock.calls.length, keysCount: sendKeys.mock.calls.length,
|
||||
overrideCount: (db.prepare("SELECT COUNT(*) AS n FROM events WHERE type='transport.prompt_override'").get() as { n: number }).n });
|
||||
// Check measurement integrity only. Policy comparisons consume every recorded result offline.
|
||||
expect(sendText.mock.calls.length).toBeLessThanOrEqual(1);
|
||||
if (!result.ok) {
|
||||
expect(sendText).not.toHaveBeenCalled();
|
||||
expect(sendKeys).not.toHaveBeenCalled();
|
||||
} else expect(sendText).toHaveBeenCalledTimes(1);
|
||||
} finally { db.close(); }
|
||||
});
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user