mirror of
https://github.com/Heretek-AI/opencode-v2-plugin-teser.git
synced 2026-11-08 08:07:13 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dfc6f6a526 | ||
|
|
2fe0ba871f | ||
|
|
bd36c293c2 | ||
|
|
b9bc7ccf23 | ||
|
|
2245c87a8e | ||
|
|
d4deb3ef57 | ||
|
|
8aa4d618ae | ||
|
|
eaf6e21063 | ||
|
|
8422645da7 | ||
|
|
8fc1886818 | ||
|
|
75624bb4e6 | ||
|
|
2c3ac504d5 | ||
|
|
b5876a6301 | ||
|
|
0de0a68087 | ||
|
|
7ba0cdb9e4 | ||
|
|
bec6543263 | ||
|
|
410b26254f | ||
|
|
0955a39e5a | ||
|
|
7a2b4d5c5f | ||
|
|
553ac2a9f7 | ||
|
|
22add43f4c | ||
|
|
5fbfd25ad3 | ||
|
|
4c3e8db59c | ||
|
|
c19eeced10 | ||
|
|
1cc883cc79 | ||
|
|
91231e4ffe | ||
|
|
cdb58cbaab | ||
|
|
32eb99d772 | ||
|
|
ca6bdd546e | ||
|
|
af6675cc17 |
No files matched your search
@@ -0,0 +1,13 @@
|
||||
# OCI-7.2-floating-majors: GitHub Actions are consumed as floating major
|
||||
# tags (`checkout@v7`, `setup-node@v7`, `upload-artifact@v7`,
|
||||
# `setup-bun@v2`). Dependabot keeps those majors current so CI semantics
|
||||
# cannot drift silently between audits. Pinned SHAs at last review are in
|
||||
# docs/NOTES.md ("Audit P2/P3" section).
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
commit-message:
|
||||
prefix: "chore"
|
||||
@@ -6,16 +6,16 @@ jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
- run: bun install --frozen-lockfile
|
||||
- run: bun run check
|
||||
e2e:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v7
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
- uses: actions/setup-node@v4
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: 22
|
||||
- run: bun install --frozen-lockfile
|
||||
@@ -27,3 +27,35 @@ jobs:
|
||||
env:
|
||||
OPENCODE_BIN: ${{ runner.temp }}/oc/node_modules/.bin/opencode
|
||||
run: bun run test:e2e
|
||||
action:
|
||||
# Runs the composite Action from this checkout, the way users run it.
|
||||
# load + scripted only: no live key is needed, and the gate never runs.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- id: pass
|
||||
uses: ./
|
||||
with:
|
||||
working-directory: test/fixtures/plugins/tool-direct
|
||||
tiers: load,scripted
|
||||
artifact-name: action-self-test-pass
|
||||
- id: fail
|
||||
uses: ./
|
||||
continue-on-error: true
|
||||
with:
|
||||
working-directory: test/fixtures/plugins/throws
|
||||
tiers: load,scripted
|
||||
artifact-name: action-self-test-fail
|
||||
- name: Check the Action's outputs
|
||||
env:
|
||||
PASS_RESULT: ${{ steps.pass.outputs.result }}
|
||||
PASS_REPORT: ${{ steps.pass.outputs.report-path }}
|
||||
FAIL_RESULT: ${{ steps.fail.outputs.result }}
|
||||
FAIL_OUTCOME: ${{ steps.fail.outcome }}
|
||||
run: |
|
||||
echo "pass: result=$PASS_RESULT report=$PASS_REPORT"
|
||||
echo "fail: result=$FAIL_RESULT outcome=$FAIL_OUTCOME"
|
||||
test "$PASS_RESULT" = pass
|
||||
test -f "$PASS_REPORT"
|
||||
test "$FAIL_OUTCOME" = failure
|
||||
test "$FAIL_RESULT" = fail
|
||||
@@ -0,0 +1,41 @@
|
||||
name: OpenCodeReview
|
||||
|
||||
concurrency:
|
||||
group: >-
|
||||
${{
|
||||
(
|
||||
github.event_name == 'pull_request_target'
|
||||
|| (
|
||||
github.event_name == 'issue_comment'
|
||||
&& github.event.issue.pull_request
|
||||
&& github.event.comment.user.type != 'Bot'
|
||||
&& (
|
||||
github.event.comment.author_association == 'MEMBER'
|
||||
|| github.event.comment.author_association == 'OWNER'
|
||||
|| github.event.comment.author_association == 'COLLABORATOR'
|
||||
)
|
||||
&& (
|
||||
startsWith(github.event.comment.body, '/open-code-review')
|
||||
|| startsWith(github.event.comment.body, '@open-code-review')
|
||||
)
|
||||
)
|
||||
)
|
||||
&& format('ocr-{0}', github.event.pull_request.number || github.event.issue.number)
|
||||
|| format('noop-{0}', github.run_id)
|
||||
}}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened]
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
review:
|
||||
uses: Heretek-AI/.github/.github/workflows/ocr-review.yml@a440b485d779b9177b1602c4d4f7f58cff0483a2 # main
|
||||
secrets: inherit
|
||||
@@ -0,0 +1,32 @@
|
||||
name: publish
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*.*.*'
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: 22
|
||||
registry-url: https://registry.npmjs.org
|
||||
- name: Upgrade npm for OIDC trusted publishing (requires npm >= 11.5)
|
||||
run: npm install -g npm@latest
|
||||
- run: bun install --frozen-lockfile
|
||||
- run: bun run check
|
||||
- name: Verify tag matches package.json version
|
||||
run: |
|
||||
TAG_VERSION="${GITHUB_REF_NAME#v}"
|
||||
PKG_VERSION="$(node --input-type=module -e "import { readFileSync } from 'node:fs'; console.log(JSON.parse(readFileSync('package.json', 'utf8')).version)")"
|
||||
if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then
|
||||
echo "Refusing to publish: tag ${GITHUB_REF_NAME} (version ${TAG_VERSION}) does not match package.json version ${PKG_VERSION}" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Publishing version ${PKG_VERSION} from tag ${GITHUB_REF_NAME}"
|
||||
- run: npm publish --provenance --access public
|
||||
@@ -28,12 +28,17 @@ in [schema/config.json](schema/config.json) and
|
||||
|
||||
From your plugin repo root:
|
||||
|
||||
> **NOTE:** the package is published as `@heretek-ai/opencode-plugin-ci`,
|
||||
> so `bunx @heretek-ai/opencode-plugin-ci ...` works directly. To run from
|
||||
> a source checkout instead, `<checkout>` below is your clone of this repo, and
|
||||
> `bun <checkout>/bin/opencode-plugin-ci.ts ...` takes the place of `bunx ...`.
|
||||
|
||||
```sh
|
||||
# 1. Scaffold starter files (.opencode-ci/config.yaml + an example scenario).
|
||||
# Plain `init` refuses to overwrite existing files; --force re-scaffolds
|
||||
# the starters (used here so this quickstart is repeatable — back up any
|
||||
# hand-written scenarios first).
|
||||
bunx @heretek-ai/opencode-plugin-ci init --force
|
||||
bun <checkout>/bin/opencode-plugin-ci.ts init --force
|
||||
|
||||
# 2. Edit .opencode-ci/config.yaml: set plugin.path if your package lives in
|
||||
# a subdirectory, list the tools/commands/agents/skills/mcp your plugin
|
||||
@@ -41,16 +46,16 @@ bunx @heretek-ai/opencode-plugin-ci init --force
|
||||
# See "Config reference" below.
|
||||
|
||||
# 3. Check that everything validates and the plugin entry resolves.
|
||||
bunx @heretek-ai/opencode-plugin-ci validate
|
||||
bun <checkout>/bin/opencode-plugin-ci.ts validate
|
||||
|
||||
# 4. Run the starter scenario. The live tier auto-skips when no API key
|
||||
# is configured, so this is free and offline.
|
||||
bunx @heretek-ai/opencode-plugin-ci run --scenario example
|
||||
bun <checkout>/bin/opencode-plugin-ci.ts run --scenario example
|
||||
```
|
||||
|
||||
Add your own scenarios under `.opencode-ci/scenarios/` (see "Scenario
|
||||
reference" below) and run everything with plain
|
||||
`bunx @heretek-ai/opencode-plugin-ci run`.
|
||||
`bun <checkout>/bin/opencode-plugin-ci.ts run`.
|
||||
|
||||
Open `.opencode-ci/out/report.md` for the report. Each scenario keeps its
|
||||
evidence under `.opencode-ci/out/<opencode-version>/<scenario-id>/`
|
||||
@@ -59,16 +64,16 @@ evidence under `.opencode-ci/out/<opencode-version>/<scenario-id>/`
|
||||
## Config reference
|
||||
|
||||
`.opencode-ci/config.yaml` (or `config.json`; exactly one may exist).
|
||||
Every object is strict: unknown keys fail validation so typos never pass
|
||||
silently. Full schema: [schema/config.json](schema/config.json).
|
||||
Every object is strict: unknown keys fail validation — errors name the file
|
||||
and field, not a line number — so typos never pass silently. Full schema: [schema/config.json](schema/config.json).
|
||||
|
||||
```yaml
|
||||
# yaml-language-server: $schema=https://unpkg.com/@heretek-ai/opencode-plugin-ci/schema/config.json
|
||||
plugin:
|
||||
path: . # plugin package dir in the checkout
|
||||
entry: dist/server.js # optional override; else auto-detected
|
||||
entry: dist/server.js # optional; must exist after build, else auto-detected
|
||||
install: bun install # optional, runs once before any tier
|
||||
build: bun run build # optional, runs once before any tier
|
||||
build: bun run build # optional, runs once before entries are resolved
|
||||
options: {} # default plugin options for every scenario
|
||||
ui: false # true = TUI-only plugin (tier 1 load check only)
|
||||
opencode:
|
||||
@@ -79,21 +84,18 @@ expect: # tier 1 inventory expectations
|
||||
agents: []
|
||||
skills: []
|
||||
mcp: []
|
||||
providers: [] # for provider plugins (tier 1)
|
||||
models: [] # for provider plugins (tier 1)
|
||||
live:
|
||||
model: opencode/some-model # provider/model, or a bare id resolved via `opencode models`
|
||||
keys: [OPENCODE_API_KEY] # env vars passed into the sandbox
|
||||
repeat: 3
|
||||
pass: 2 # must be <= repeat
|
||||
budgetUSD: 1.00 # hard stop for the whole live tier
|
||||
budgetUSD: 1.00 # hard stop for the whole live tier, across all versions
|
||||
timeoutMs: 240000 # per live attempt, unless the scenario sets timeoutMs
|
||||
gate:
|
||||
enabled: true # historical flake gate, live tier only
|
||||
window: 5
|
||||
threshold: 0.9
|
||||
bootstrap: fail # fail|allow — allow tolerates "no history yet"
|
||||
workflow: plugin-ci.yml # optional; else inferred from GITHUB_WORKFLOW_REF
|
||||
artifact: opencode-plugin-ci-results # must match the Action's artifact-name input
|
||||
artifact: opencode-plugin-ci-results # history artifact; --gate-artifact (the Action's artifact-name) overrides it
|
||||
logLevel: info # info|all
|
||||
```
|
||||
|
||||
@@ -102,12 +104,12 @@ duplicate `id`s are an error. Schema:
|
||||
[schema/scenario.json](schema/scenario.json).
|
||||
|
||||
```yaml
|
||||
id: redacts-secrets # ^[a-z0-9][a-z0-9-]*$
|
||||
id: redacts-secrets # ^[a-z0-9][a-z0-9-]*$; "load" and "bun-test" are reserved
|
||||
description: execute.before strips API keys from bash commands
|
||||
mode: scripted # scripted | live
|
||||
plugin:
|
||||
options: { mode: strict } # merged over config plugin.options
|
||||
workspace: # files written into the project dir before the run
|
||||
workspace: # files written into the project dir before the run (relative paths, no "..")
|
||||
.env: "API_KEY=sk-test\n"
|
||||
prompt: "print the env file"
|
||||
script: # scripted only: one step per agent-loop request
|
||||
@@ -115,7 +117,7 @@ script: # scripted only: one step per agent-loop request
|
||||
- name: shell
|
||||
arguments: { command: "cat .env", description: "show env" }
|
||||
- text: done
|
||||
timeoutMs: 120000
|
||||
timeoutMs: 120000 # optional; default 120000 (scripted) or live.timeoutMs (live)
|
||||
expect:
|
||||
exitCode: 0
|
||||
tools:
|
||||
@@ -126,8 +128,9 @@ judge: # live only, optional
|
||||
```
|
||||
|
||||
Cross-field rules (all validated): `script` is required in scripted mode
|
||||
and forbidden in live mode; `expect.request` and `judgeReply` are
|
||||
scripted-only; `judge` is live-only; `live.pass <= live.repeat`.
|
||||
and forbidden in live mode; `expect.request` is scripted-only; `judge` is
|
||||
live-only; `live.pass <= live.repeat`; `id` must not be `load` or `bun-test`;
|
||||
`workspace` keys must be relative paths that stay inside the project dir.
|
||||
|
||||
## Scenario reference: one example per assertion type
|
||||
|
||||
@@ -136,7 +139,10 @@ most fields: `equals`, `contains`, `notContains`, `matches` / `notMatches`
|
||||
(JavaScript regex source; a leading `(?i)` enables case-insensitivity),
|
||||
`includes` (arrays), `exists`, `count` (`{ eq?, gte?, lte? }`).
|
||||
Non-string values are compared as JSON for the string matchers; `equals`
|
||||
uses deep equality.
|
||||
uses deep equality. **A field with no recorded value fails every matcher
|
||||
except `exists: false`**, so a misspelled field name (`input: { comand: …
|
||||
}`) fails with `no value was recorded (check the field name)` instead of
|
||||
passing a `notContains`.
|
||||
|
||||
- **`exitCode`** — the `opencode run` exit code (default `0`):
|
||||
`exitCode: 0`
|
||||
@@ -235,6 +241,11 @@ This means:
|
||||
- Because `--format json` only shows the outer `execute` call, write tool
|
||||
assertions against the **probe** (`tools:` / `hooks:`), which sees every
|
||||
inner call under its real name — never against `tool_use` events.
|
||||
- Tools registered with a `namespace` are recorded as
|
||||
`<namespace>_<name>` (dots become `_`). In `tools:`, `noTools:` and
|
||||
`hooks[].tool` you may write that name, `ns.name` or the catalog path
|
||||
`tools.ns.name`; all three match. A bare `name` does not match, and the
|
||||
failure says which namespaced name was recorded.
|
||||
|
||||
## Tiers
|
||||
|
||||
@@ -245,9 +256,12 @@ This means:
|
||||
| 3 | `live` | real provider | only with a key; `pass` of `repeat` attempts must pass |
|
||||
|
||||
Filter with `run --tier load --tier scripted`, `--scenario <id>`
|
||||
(repeatable, comma-separated), `--opencode <v>`, `--out <dir>`, `--keep`
|
||||
(repeatable, comma-separated; an unknown id is a config error, exit 2),
|
||||
`--opencode <v>` (`x.y.z` or `latest`), `--out <dir>`, `--keep`
|
||||
(keep sandboxes for debugging), `--gate` (force the gate locally),
|
||||
`--verbose` / `--quiet`.
|
||||
`--gate-artifact <name>` (overrides `gate.artifact`), `--verbose` /
|
||||
`--quiet`. Each case's evidence dir under `--out` is wiped when the case
|
||||
starts, so re-running into the same out dir never mixes runs.
|
||||
|
||||
TUI-only plugins (no server entry, or `plugin.ui: true`) get a static
|
||||
tier-1 check (default export has a string `id` and function `setup`) and
|
||||
@@ -255,7 +269,12 @@ tiers 2–3 report `skipped`. A failing tier 1 marks that version's later
|
||||
tiers `skipped: "tier 1 failed"`. Results for `opencode.versions[0]` are
|
||||
required; other versions are allowed failures (reported, exit code
|
||||
unaffected). `.opencode-ci/**/*.test.ts` files run once via `bun test`
|
||||
with `OPENCODE_BIN` set, folded in as a `bun-test` case.
|
||||
with `OPENCODE_BIN` set, folded in as a `bun-test` case (the tool passes
|
||||
each file as an explicit `./` path, because `bun test` skips dot-dirs
|
||||
when given a directory filter). The live budget is shared by every
|
||||
version in one run. Once it is spent, a case that has not started is
|
||||
`skipped`, and a case that already ran attempts but has fewer than `pass`
|
||||
passes fails with `(budget exhausted …)` in its failure line.
|
||||
|
||||
## Gate (live tier only)
|
||||
|
||||
@@ -265,17 +284,19 @@ flake gate instead: the live tier fails only when a *stable* case (passed
|
||||
`.opencode-ci/acknowledgments.json` (`{ version: 1, acknowledgments:
|
||||
[{ case, reason, issue, date }] }`; malformed fails closed).
|
||||
|
||||
History comes from earlier `opencode-plugin-ci-results` artifacts on the
|
||||
same branch via `gh`, so the workflow needs `actions: read`. The gate runs
|
||||
History comes from earlier `opencode-plugin-ci-results` artifacts via
|
||||
`gh`, so the workflow needs `actions: read`. Push runs read their own
|
||||
branch's history; pull request runs read the **base** branch's history
|
||||
(`GITHUB_BASE_REF`), since a PR's own ref has no earlier runs. The gate runs
|
||||
only when the live tier ran at least one non-skipped case, `gate.enabled`
|
||||
is true, and `--gate` was passed or `GITHUB_ACTIONS == "true"`. Without
|
||||
`gh`/token it falls back to "fail on any live failure". No history exits
|
||||
2 unless `gate.bootstrap: allow`.
|
||||
|
||||
**Matrix builds:** artifact names must be unique within a workflow run, so
|
||||
give each matrix leg a distinct `artifact-name` Action input **and** set the
|
||||
same value in that leg's `gate.artifact` config (the gate downloads history
|
||||
by that name). Example:
|
||||
give each matrix leg a distinct `artifact-name` Action input. The Action
|
||||
passes it to the CLI as `--gate-artifact`, so each leg reads its own
|
||||
history; no config change is needed, and validation does NOT enforce this. Example:
|
||||
|
||||
```yaml
|
||||
strategy:
|
||||
@@ -284,20 +305,20 @@ strategy:
|
||||
steps:
|
||||
- uses: Heretek-AI/opencode-v2-plugin-teser@v1
|
||||
with:
|
||||
opencode-versions: ${{ matrix.opencode }}
|
||||
artifact-name: opencode-plugin-ci-results-${{ matrix.opencode }}
|
||||
```
|
||||
|
||||
with `gate: { artifact: opencode-plugin-ci-results-2.0.26 }` in the
|
||||
required leg's config (each leg reads its own checkout's config, so
|
||||
parameterize it per leg or keep one artifact per workflow).
|
||||
|
||||
## GitHub Action
|
||||
|
||||
`Heretek-AI/opencode-v2-plugin-teser@v1` (see [action.yml](action.yml)).
|
||||
Inputs `tiers`, `opencode-versions` (overrides config), `working-directory`,
|
||||
Inputs `tiers`, `opencode-versions` (overrides config), `working-directory`
|
||||
(must stay inside `GITHUB_WORKSPACE`; absolute paths inside the workspace are fine, paths outside exit 2),
|
||||
`upload-artifact` (default `true`), `artifact-name` (default
|
||||
`opencode-plugin-ci-results`). Outputs `result` (`pass`/`fail`) and
|
||||
`report-path`. Inputs travel to the script through `env:`, never
|
||||
`opencode-plugin-ci-results`; also the gate's history name). There is no
|
||||
`config` input: the tool reads `.opencode-ci/config.*` in
|
||||
`working-directory`. Outputs `result` (`pass`/`fail`; a fully skipped run
|
||||
is `pass`) and `report-path`. Inputs travel to the script through `env:`, never
|
||||
interpolated. Minimal workflow ([examples/workflow.yml](examples/workflow.yml)):
|
||||
|
||||
```yaml
|
||||
@@ -327,4 +348,13 @@ jobs:
|
||||
checks on `push` to trusted branches.
|
||||
- Treat scenario `workspace` files and prompts as untrusted input to the
|
||||
plugin, and keep secrets out of assertions: the probe redacts secret-like
|
||||
env/header values, and you should too — never print or record secret values.
|
||||
env/header values **at the source**, but every other evidence surface
|
||||
(prompt, tool I/O, session context, stub requests, events, report) **MUST
|
||||
NOT contain secrets — that is the scenario author's responsibility**.
|
||||
Never print or record secret values (a scenario that runs `env` or
|
||||
`cat .env` persists the key into the uploaded evidence).
|
||||
- As defense in depth, explicitly configured live-key values (`live.keys`)
|
||||
are scrubbed to `[redacted]` in `report.md`,
|
||||
`results.json` failures, `stub-requests.jsonl`, `events.jsonl` and
|
||||
`stderr.log` before they are written. `probe.jsonl` stays raw on purpose,
|
||||
so `notContains`-style assertions still see — and fail on — leaked values.
|
||||
+14
-4
@@ -9,13 +9,13 @@ inputs:
|
||||
description: "Overrides config, comma-separated"
|
||||
default: ""
|
||||
working-directory:
|
||||
description: "Plugin repo directory"
|
||||
description: "Plugin repo directory; must stay inside GITHUB_WORKSPACE (absolute paths inside the workspace are fine)"
|
||||
default: "."
|
||||
upload-artifact:
|
||||
description: "Upload evidence + results"
|
||||
default: "true"
|
||||
artifact-name:
|
||||
description: "Artifact name for evidence + results (must match gate.artifact; use a distinct name per matrix leg)"
|
||||
description: "Artifact name for evidence + results. The gate also reads history from this name, so use a distinct name per matrix leg."
|
||||
default: "opencode-plugin-ci-results"
|
||||
outputs:
|
||||
result:
|
||||
@@ -28,7 +28,7 @@ runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
- uses: actions/setup-node@v4
|
||||
- uses: actions/setup-node@v7
|
||||
with: { node-version: 22 }
|
||||
- name: Install tool dependencies
|
||||
shell: bash
|
||||
@@ -42,11 +42,21 @@ runs:
|
||||
# Inputs go through env, never interpolated into the script (prevents script injection).
|
||||
IN_TIERS: ${{ inputs.tiers }}
|
||||
IN_VERSIONS: ${{ inputs.opencode-versions }}
|
||||
IN_ARTIFACT: ${{ inputs.artifact-name }}
|
||||
IN_WD: ${{ inputs.working-directory }}
|
||||
ACTION_PATH: ${{ github.action_path }}
|
||||
run: |
|
||||
# OCI-6.3-working-dir: working-directory must stay inside the
|
||||
# workspace. The step has already changed into working-directory,
|
||||
# so $PWD is the resolved path; compare it against the resolved
|
||||
# workspace root. Absolute paths inside the workspace are fine.
|
||||
# Checked via env passthrough to keep the zero-interpolation invariant.
|
||||
ws="$(cd "$GITHUB_WORKSPACE" && pwd -P)"
|
||||
case "$(pwd -P)/" in "$ws"/*) ;; *) echo "working-directory must be inside the workspace (got: $IN_WD)" >&2; echo "result=fail" >> "$GITHUB_OUTPUT"; exit 2;; esac
|
||||
args=()
|
||||
IFS=',' read -ra T <<< "$IN_TIERS"; for t in "${T[@]}"; do args+=(--tier "$t"); done
|
||||
if [ -n "$IN_VERSIONS" ]; then IFS=',' read -ra V <<< "$IN_VERSIONS"; for v in "${V[@]}"; do args+=(--opencode "$v"); done; fi
|
||||
args+=(--gate-artifact "$IN_ARTIFACT")
|
||||
set +e
|
||||
bun "$ACTION_PATH/bin/opencode-plugin-ci.ts" run "${args[@]}" --out .opencode-ci/out
|
||||
code=$?
|
||||
@@ -55,7 +65,7 @@ runs:
|
||||
if [ $code -eq 0 ]; then echo "result=pass" >> "$GITHUB_OUTPUT"; else echo "result=fail" >> "$GITHUB_OUTPUT"; fi
|
||||
exit $code
|
||||
- if: always() && inputs.upload-artifact == 'true'
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: ${{ inputs.artifact-name }}
|
||||
path: ${{ inputs.working-directory }}/.opencode-ci/out
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+363
@@ -1,3 +1,228 @@
|
||||
# Audit P1 remediation (2026-10-18)
|
||||
|
||||
Eight P1s from `AUDIT_REPORT_FINDINGS.md` (branch `fix/v1.1-audit-remediation`,
|
||||
three commits). Toolchain: Bun 1.4.2, `@opencode/[email protected]` via the main
|
||||
repo's `review/oc` binary (read-only), `TMPDIR=/dev/shm/ocz`. `bun run check`
|
||||
ends at **286 pass, 0 fail** (248 baseline + 38 new).
|
||||
|
||||
- OCI-4.4-workspace-escape: `safeWorkspaceDest` in `src/sandbox/sandbox.ts`
|
||||
(`resolve`+`sep` containment mirroring `evaluateFiles`, absolute paths
|
||||
rejected explicitly, `ConfigError`/exit 2). `test/unit/sandbox-escape.test.ts`
|
||||
(5 tests). Guard-removal sabotage goes 1 pass / 4 fail.
|
||||
- OCI-4.4-ancestor: fail-closed allow-list in `runLoad` (`expectedInventoryIds`,
|
||||
`findUnexpectedPlugins`, `detectAncestorConfig` in `src/tiers/load.ts`;
|
||||
`test/unit/ancestor-guard.test.ts`, 11 tests). REFINEMENT vs the audit
|
||||
text: a clean 2.0.26 inventory holds 86 `source.type: "builtin"` plugins
|
||||
shipped with the binary, so builtins are exempt and only non-builtin,
|
||||
non-configured plugins fail (`id` AND resolved local-path match, so an
|
||||
ancestor plugin colliding with a probe id is still caught). Verified live:
|
||||
clean tier 1 exit 0; planted `$TMPDIR/opencode.json` exit 1 with
|
||||
`ancestor opencode config detected ...: sandbox isolation compromised.`;
|
||||
planted local evil plugin exit 1 with
|
||||
`unexpected plugins in inventory from ancestor config: [evil-ancestor-plugin].`
|
||||
- OCI-1.5-skip: `test/e2e/cli.test.ts` "run in throws exits 1" now asserts
|
||||
`results.json` holds `throws-smoke` with `skipped: "tier 1 failed"`.
|
||||
- OCI-6.4-unpublished: README quickstart uses `bun <checkout>/...` source
|
||||
form; `bunx` appears only in the unpublished note. Publish automation note:
|
||||
no `npm publish` automation exists (same gap as `docs/NOTES.md:270`); to
|
||||
close this P1 run `npm publish --provenance` for 0.1.0, then verify
|
||||
`npm view @heretek-ai/opencode-plugin-ci version` and the unpkg
|
||||
`$schema` URLs, and add a CI publish job. No publish attempted here
|
||||
(no secrets).
|
||||
- OCI-6.3-action-guard: `test/unit/action-guard.test.ts` (YAML-parses
|
||||
`action.yml`, asserts every `run:` block has no `${{ inputs./github. }}`;
|
||||
comment-position injection included). Sabotage (`# ${{ inputs.tiers }}` in
|
||||
`run:`) goes red; restored afterwards.
|
||||
- OCI-6.4-files-guard: `test/unit/packaging.test.ts` pins `files`/`bin` and
|
||||
`npm pack --dry-run` contents. Dropping `probe` from `files` goes red
|
||||
(deep-equal + tarball assertions).
|
||||
- OCI-7.3-redact-untested: `test/unit/probe-redact.test.ts` imports
|
||||
`probe/core.js` (`@ts-ignore`: untyped JS, excluded from tsconfig) and
|
||||
asserts each alternative individually with its own canary. Narrowing to
|
||||
`/KEY/i` fails 5/7; narrowing to spec-four fails AUTH+COOKIE.
|
||||
- OCI-7.3-evidence-leaks: `redactSecretValues` + `collectSecretValues` in
|
||||
`src/util/redact.ts`, applied to stub `recordFile` writes, tier
|
||||
`events.jsonl`/`stderr.log` writes (load/scripted/live), `results.json`
|
||||
display strings and `report.md`/step-summary/gate-section in `run.ts` and
|
||||
the `report` command. In-memory evidence stays raw so assertions still fail
|
||||
on leaks. `test/unit/evidence-scrub.test.ts` (8 tests). DEVIATIONS from the
|
||||
audit Verify text: `probe.jsonl` stays raw by design (assertions need the
|
||||
leak to fail the scenario); no validation-time warning when workspace text
|
||||
contains key values (load-time warnings deferred — the scrub covers every
|
||||
persisted surface except the intentionally-raw probe).
|
||||
|
||||
Also: `Sandbox` now observes its `git init` promise so a throwing
|
||||
constructor (workspace containment) can never surface as an unhandled
|
||||
rejection late in the run.
|
||||
|
||||
## v1.1 (2026-10-18)
|
||||
|
||||
Toolchain: Bun 1.4.2, `@opencode/[email protected]` pinned
|
||||
(`review/oc/node_modules/.bin/opencode --version` prints
|
||||
`opencode v2.0.26`). `bun install --frozen-lockfile`: no changes.
|
||||
|
||||
- M1 reproductions (`review/spike-v1.1/`, git-ignored): step 1 printed
|
||||
`The following filters did not match any test files` / `exit=1`, then
|
||||
` 1 pass` / `exit=0`; step 2 printed `run1 exit=0`, `run2 exit=0`,
|
||||
`run3 exit=1` with probe counts `stale-out:2`, `stale-out-fresh:1`.
|
||||
- M6 re-run (fresh `review/spike-v1.1`, real opencode 2.0.26,
|
||||
`TMPDIR=/dev/shm/ocz`): `run1 exit=0`, `run2 exit=1`, `run3 exit=1`,
|
||||
both inventory counts `1`. B5 fixed.
|
||||
- M2: as briefed — fake opencode binary + shared test setup (4 pass).
|
||||
- M3: as briefed — build before entries, tier 1 takes `serverEntry` (5 pass).
|
||||
- M4: as briefed — `.opencode-ci` test files passed by explicit `./` path
|
||||
(2 pass). Note: with /tmp inodes exhausted, unit runs used
|
||||
`TMPDIR=/dev/shm/ocz`; unrelated to the fix.
|
||||
- M5: as briefed — budget exhaustion fails started cases, one budget per
|
||||
run (7 pass).
|
||||
- M6: as briefed — evidence dirs reset per case, reserved ids,
|
||||
`--opencode`/`--scenario` validation (6 pass), plus the M1 re-run above.
|
||||
- M7: as briefed — no-op config fields removed, `live.timeoutMs` is the
|
||||
live default, schemas regenerated (schema-sync passes).
|
||||
- M8: as briefed — missing values fail every matcher except
|
||||
`exists: false` (4 pass).
|
||||
- M9: as briefed — namespaced tool names normalized in assertions (4 pass).
|
||||
- M10: as briefed — PR gate history from the base branch,
|
||||
`--gate-artifact` flag (2 pass).
|
||||
- M11: as briefed — judge stub defaults to fail, single defaults
|
||||
warning, codemode comment fix. Decisions (review nits, no code change):
|
||||
results keep the requested `opencode` version (SPEC 7.1: the
|
||||
required/allowed split, `report` and gate key on it); the duplicate
|
||||
`probe-live.jsonl` copy stays (harmless now that dirs are reset); gate
|
||||
history still downloads the full evidence artifact (results-only
|
||||
artifact is a v2 item).
|
||||
- M12: as briefed — one deliberately failing scenario per surface
|
||||
fixture (hook-mutate, shell-env, system-inject, event-file),
|
||||
`noTools: [execute]` proves `tool-direct` is direct, Action self-test
|
||||
job in `.github/workflows/ci.yml`. Local Action run matched the brief
|
||||
exactly: `tool-direct exit=0` with `result=pass` and a `report-path`
|
||||
line, `throws exit=1` with `result=fail` (2.6 caveat stands: composite
|
||||
outputs on failure were exercised only locally, not on GitHub — if the
|
||||
`fail` leg's `result` comes back empty there, keep the `outcome`
|
||||
check, drop only the `FAIL_RESULT` line).
|
||||
- M13: as briefed — README replacements applied in order; `grep -n
|
||||
"providers\|window: 5\|threshold\|judgeReply" README.md` prints
|
||||
nothing, `grep -c -- "--gate-artifact" README.md` gives 3;
|
||||
`bun run check` 248 pass.
|
||||
- live: not run (per 2.6; covered by the fake binary, no real key used).
|
||||
- SPEC re-read (3.2, 4, 5.1, 5.2, 5.3, 6.1, 6.2, 7.1, 7.2, 8, 9): every
|
||||
sentence confirmed in code except the Known gaps below.
|
||||
|
||||
Known gaps (SPEC statements the code does not fully implement):
|
||||
|
||||
- No `workspace.tar` or per-case `result.json` (evidence dirs hold
|
||||
`events.jsonl`, `probe.jsonl`, `stub-requests.jsonl`, `stderr.log`;
|
||||
per-case `result.json` and failure-only `workspace.tar` deferred to v2).
|
||||
- No `webfetch`/`websearch` report flag (observable only via raw
|
||||
evidence or `noTools`; flagging deferred to v2).
|
||||
- `ctx.storage` is not snapshotted into evidence.
|
||||
- No dogfood scheduled workflow over `awesome-opencode`.
|
||||
- Gate history downloads the full evidence artifact per run (deleted
|
||||
after reading); a results-only artifact is a v2 item.
|
||||
- Provider expectations deferred to v2 (`expect.providers`/`models`
|
||||
fail validation instead of being silently ignored).
|
||||
|
||||
# v1.1 M11: judge stub defaults to fail; single defaults warning; comment fix (2026-10-18)
|
||||
|
||||
- `src/tiers/judge.ts`: stub judge default is now a fail
|
||||
(`'{"pass":false,"reason":"no stubReply given"}'`), per review nit —
|
||||
fail closed when no reply is stubbed. `grep -c '"pass":true'
|
||||
src/tiers/judge.ts` → 0.
|
||||
- `src/config/load.ts`: deleted the direct `console.warn` loop; the
|
||||
"using defaults" warning is returned in `warnings` and printed once by
|
||||
the caller. `grep -c "console.warn" src/config/load.ts` → 0.
|
||||
- `src/stub/codemode.ts`: comment fixed (no longer blames the brief's
|
||||
regex for the trailing `(`).
|
||||
- Review-nit decisions (no code change, SPEC-kept as is):
|
||||
- Results keep `opencode` as the requested version (`latest`), because
|
||||
the required/allowed split, `report` and the gate key on it (SPEC 7.1).
|
||||
- The duplicate `probe-live.jsonl` copy in evidence dirs stays; harmless
|
||||
now that dirs are reset per case.
|
||||
- Gate history still downloads the full evidence artifact per run;
|
||||
splitting out a results-only artifact is a v2 item.
|
||||
- `bun run format`: applied (tabs); `bun run check` → 248 pass.
|
||||
|
||||
# v1.1 M6: reset evidence dirs per case; reserve tool ids; validate --opencode and --scenario (2026-10-18)
|
||||
|
||||
- Pre-fix run matched the brief: `run-inputs` failed to load
|
||||
(`Export named 'parseRunArgs' not found`), `scenario-rules` 0 pass / 2 fail.
|
||||
- `src/util/fs.ts`: new `resetDir` (rm -rf + mkdir -p, refuses any dir not
|
||||
strictly inside root). All four tiers reset their evidence dir.
|
||||
`src/config/schema.ts`: `versionSchema` exported, `load`/`bun-test`
|
||||
reserved, workspace `..`/absolute rejected. `src/cli.ts`: `parseRunArgs`
|
||||
+ `RunFlags` exported, `--opencode` validated like `opencode.versions`.
|
||||
`src/run.ts`: unknown `--scenario` ids throw `ConfigError` (exit 2).
|
||||
- Acceptance: new files 6 pass, 0 fail; `bun run check` 236 pass + the same
|
||||
single environmental `stub.test.ts > recordFile` failure (/tmp inodes still
|
||||
100% — see M4 note; 236 + 1 = all 237 tests, so nothing else is red).
|
||||
- M1 stale-evidence re-run (fresh `review/spike-v1.1`, real opencode 2.0.26,
|
||||
`TMPDIR=/dev/shm/ocz`): `run1 exit=0`, `run2 exit=1`, `run3 exit=1`,
|
||||
inventory counts `stale-out:1` and `stale-out-fresh:1`. B5 fixed.
|
||||
- `bun run format`: no fixes applied.
|
||||
|
||||
# v1.1 M5: budget exhaustion fails started cases; one budget per run (2026-10-18)
|
||||
|
||||
- Pre-fix run matched the brief exactly: `live-loop` 5 pass / 1 fail (the
|
||||
budget-after-attempts case) and `run-budget` 0 pass / 1 fail (the `latest`
|
||||
row ran on a fresh budget instead of skipping).
|
||||
- `src/tiers/live.ts`: `budgetStop` breaks the loop when attempts already ran
|
||||
(failure line gains `(budget exhausted ...)`); the no-attempt path still
|
||||
returns `skipped`. `src/run.ts`: one `LiveBudget` above the version loop.
|
||||
- Acceptance: new files 7 pass, 0 fail; `bun run check` 231 pass + the same
|
||||
single environmental `stub.test.ts > recordFile` failure (/tmp inodes still
|
||||
100% — see M4 note). `bun run format`: no fixes applied.
|
||||
|
||||
# v1.1 M4: bun-test files by explicit path (2026-10-18)
|
||||
|
||||
- New `test/unit/run-bun-test.test.ts` fails pre-fix with the genuine B3
|
||||
message (`The following filters did not match any test files ...
|
||||
.opencode-ci`, verified by stashing the fix and re-running), and passes
|
||||
2/2 post-fix. `src/run.ts` passes each discovered file as an explicit
|
||||
`./`-relative path.
|
||||
- ENVIRONMENTAL BLOCKER (not a code failure): `bun run check` shows
|
||||
223 pass + 1 fail in `test/unit/stub.test.ts > recordFile gets one JSONL
|
||||
line per request`. That test hardcodes
|
||||
`/tmp/opencode-plugin-ci-stub-test-<pid>.jsonl` (line 230), and /tmp has
|
||||
0 free inodes (`df -i`: 1048576/1048576 used). Top consumers are other
|
||||
sessions' dirs (`opencode-harness-audit-QyyFZg` 400k inodes,
|
||||
`opencode-harness-audit-V9uKqK` 195k, `m6probe`, `m6run2`, `opencode`,
|
||||
`lobehub`, `paperclip`); I own no files in /tmp (my runs use
|
||||
`TMPDIR=/dev/shm/ocz` or clean up via `cleanupTempDirs`), so there is
|
||||
nothing safe to delete. The same test passed in the M3 check (222 pass)
|
||||
when inodes were momentarily available. Retried 4 times, same single
|
||||
failure. Re-run `bun run check` once /tmp drains to confirm 224 pass.
|
||||
- `bun run format`: no fixes applied.
|
||||
|
||||
# v1.1 M3: build before entries, serverEntry plumbing (2026-10-18)
|
||||
|
||||
- New `test/unit/run-entry.test.ts` failed to load before the fix
|
||||
(`Export named 'StepError' not found`), then went 1 pass / 4 fail after
|
||||
the `StepError` step only — matching the brief's predicted sequence.
|
||||
- `src/plugin/install.ts`: `ConfigError` -> `StepError` (3 sites + doc
|
||||
comment). `src/run.ts`: install/build moved before `ensureEntriesOrThrow`,
|
||||
`serverEntry: entries.server` passed to tier 1. `src/tiers/load.ts`:
|
||||
takes `serverEntry`, checks both shim and real paths in logs.
|
||||
`src/plugin/entry.ts`: missing explicit entry throws `ConfigError`.
|
||||
`src/cli.ts`: `validate` warns (exit 0) when the entry is missing but
|
||||
`plugin.build` is set. `test/unit/entry.test.ts` + `test/e2e/tier1.test.ts`
|
||||
updated per the brief.
|
||||
- Acceptance: `run-entry.test.ts` 5 pass; `bun run check` 222 pass;
|
||||
`tier1.test.ts + cli.test.ts` 10 pass. Note: /tmp inodes were 100% used
|
||||
by other sessions' leftovers, so e2e ran with `TMPDIR=/dev/shm/ocz`
|
||||
(same workaround as the audit report); unit tests were unaffected.
|
||||
|
||||
# v1.1 M2: fake opencode binary and shared test setup (2026-10-18)
|
||||
|
||||
Toolchain: bun 1.4.2, `opencode v2.0.26` pinned binary.
|
||||
|
||||
- Created `test/helpers/fake-opencode.ts` (plan-playback fake: writes probe
|
||||
records + `step_finish` cost, counts `run` invocations), `test/helpers/tmp.ts`
|
||||
(temp dirs with cleanup) and `test/helpers/run-ci.ts` (`makePlugin`,
|
||||
`useFake`, `isolateCiEnv`) exactly per the brief.
|
||||
- `test/unit/run-exit-codes.test.ts`: 4 pass, 0 fail on first run (expected:
|
||||
these cover behavior that already works; they prove the fake works).
|
||||
- `bun run check`: 217 pass, 0 fail. `bun run format`: no fixes applied.
|
||||
|
||||
# Spike notes (M2)
|
||||
|
||||
Date: 2026-10-17. opencode: `opencode v2.0.26`
|
||||
@@ -327,3 +552,141 @@ statement not listed under "Known gaps" below was confirmed in code.
|
||||
- Carried from M12: custom `gate.window`/`gate.threshold` no longer take
|
||||
effect (verbatim upstream verdict hard-codes >= 5 samples, >= 90%);
|
||||
defaults (5 / 0.9) keep default behavior identical.
|
||||
|
||||
# Audit P2/P3 remediation (2026-10-18)
|
||||
|
||||
Implements the P2 and P3 Remediation lines of `AUDIT_REPORT_FINDINGS.md`
|
||||
verbatim (P1 was the three prior `audit P1:` commits on this branch).
|
||||
P3 items need no behavior change except where noted.
|
||||
|
||||
## P2
|
||||
|
||||
- OCI-3.3-meta (`probe/core.js` metadata-last): no test crafted a payload
|
||||
with colliding `kind`/`phase` keys, so a payload-wins sabotage stayed
|
||||
green. Added `test/unit/probe-meta.test.ts` importing `makeProbe` and the
|
||||
new `writeRecord` seam (the single spread site; `write` delegates to it),
|
||||
writing a record whose data contains
|
||||
`{kind:"forged",phase:"forged",seq:999}` and asserting the parsed line
|
||||
keeps `kind`/`phase`/`seq` from metadata. Verified the swap sabotage
|
||||
(metadata-first spread in `writeRecord`) goes red. Public probe handlers
|
||||
build fixed-shape data objects, so a collision is reachable only through
|
||||
this seam — the ordering is defense-in-depth, now pinned.
|
||||
- OCI-4.3-order (`src/tiers/load.ts:88-92`): the tier-1 `[pre,plugin,post]`
|
||||
order was unpinned while the scripted copy is pinned by
|
||||
`hook-mutate-pass`. Extracted `buildLoadPlugins(pre, shimDir, post, ...)`
|
||||
called by `runLoad`, pinned by `test/unit/load-order.test.ts` (package
|
||||
order plus a `runLoad`-calls-the-constructor shape check). A `runLoad`-level
|
||||
hook-order e2e is inapplicable: tier 1 runs one `{"text":"ok"}` stub turn
|
||||
with no tool calls, so tier-1 `probe.jsonl` holds no hook records to
|
||||
compare. Swap sabotage (`[post, shim, pre]` in the helper) goes red.
|
||||
- OCI-2.5-noerrors (`src/assert/expect.ts:381` `?? true`): added
|
||||
`test/unit/noerrors.test.ts` — `evaluate` with `noErrors` deleted (the
|
||||
fallback path) on evidence with an `error` event must contain `error
|
||||
event`; explicit `noErrors: false` still passes; clean runs pass. Prod
|
||||
default stays schema-pinned (`noErrors: true` in `scenarioExpectSchema`);
|
||||
no prod change, fallback pinned only. No scripted e2e fixture added: no
|
||||
stub turn emits `error` events on a passing path, so the unit pin is the
|
||||
whole coverage.
|
||||
- OCI-1.4-cap (`probe/core.js:15-17` 200-item cap): `clip` now takes
|
||||
`capArrays`; `writeRecord` passes `false` for `kind === "inventory"`, so
|
||||
inventory membership lists are never truncated (strings still clip, depth
|
||||
cap unchanged). `test/unit/probe-cap.test.ts` drives `makeProbe` with a
|
||||
201-tool inventory through `readProbe` and asserts all 201 survive with
|
||||
`echo_tool` last; a second test asserts long strings still clip. The old
|
||||
`slice(0, 200)` fails the first test.
|
||||
- OCI-1.4-providers + OCI-5.6-inert-window: already remediated by the M7
|
||||
removal (fail-closed) — verified, no code change. `ConfigSchema.safeParse`
|
||||
rejects `expect.providers`, `expect.models`, `gate.window` and
|
||||
`gate.threshold`; pinned by `test/unit/removed-fields.test.ts` (names the
|
||||
audit IDs; overlaps `config-schema.test.ts` which asserts the same four
|
||||
rejections).
|
||||
- OCI-6.1-allowed-wiring (`src/run.ts:191-192`): extracted pure
|
||||
`partitionVersions(versions)` (`{required, allowedVersions}`), called from
|
||||
`runCi`; pinned by `test/unit/partition.test.ts`. The `slice(1)` → `[]`
|
||||
sabotage goes red. Exit codes never read `allowedVersions` (report labels
|
||||
only, fail-closed direction); the wiring is now pinned anyway.
|
||||
- OCI-2.5-files-symlink (`src/assert/expect.ts:318-370`): after the lexical
|
||||
check, `evaluateFiles` resolves the nearest existing ancestor via
|
||||
`realpathSync`, re-appends the missing tail, and re-checks containment;
|
||||
any `realpath` error fails closed. `test/unit/files-symlink.test.ts`:
|
||||
`linketc -> /etc` with `linketc/passwd` (exists and content assertions)
|
||||
yields `path escapes the project dir`, with no host content leaked;
|
||||
`sub/../ok.txt` still passes; lexical `../` still rejected.
|
||||
- OCI-6.4-action-pins: bumped to `@v7` in `action.yml`
|
||||
(`actions/setup-node`, `actions/upload-artifact`) and
|
||||
`.github/workflows/ci.yml` (`actions/checkout`, `actions/setup-node`);
|
||||
`oven-sh/setup-bun@v2` is already at HEAD. Pinned by
|
||||
`test/unit/action-pins.test.ts` (no `actions/*@v1..v6` in either file).
|
||||
`rg '(checkout|setup-node|upload-artifact)@v[0-9]'` shows only v7.
|
||||
|
||||
## P3
|
||||
|
||||
- OCI-7.1-no-line: no code change. Validation errors name file + field, not
|
||||
line (`formatIssues`: `${file}: validation failed:\n - at "<path>":
|
||||
<message>`; line hints exist only for YAML *parse* errors). Corrected the
|
||||
claim wording in README ("errors name the file and field, not a line
|
||||
number") and docs/SPEC 5.2 ("naming the file and field (not a line
|
||||
number)"). The audit-report claim text itself lives outside this repo.
|
||||
- OCI-registry-drift (`src/opencode/binary.ts:79` `latest` floats): no code
|
||||
change required. Policy note: `latest` resolves live per run
|
||||
(`npmViewVersion`); drift 2.0.26 → 2.0.32 was verified additive-only by the
|
||||
audit. Re-pinning policy: re-run the pinned-vs-head API diffs
|
||||
(`tool/shell/catalog/hooks/policy/agent/session/permission`) and
|
||||
`npm view @opencode/cli version` before moving the default; the allowed
|
||||
`latest` leg absorbs drift without failing the job. A scheduled head-leg
|
||||
e2e remains future work.
|
||||
- OCI-6.3-working-dir: docs now state `working-directory` must stay inside
|
||||
`GITHUB_WORKSPACE` (`action.yml` input description, README, SPEC 8), plus
|
||||
an env-passthrough guard keeping the zero-interpolation invariant —
|
||||
`IN_WD: ${{ inputs.working-directory }}` with
|
||||
`case "$IN_WD" in /*|*..*) ... exit 2;; esac` before any tier runs.
|
||||
Pinned by `test/unit/action-working-dir.test.ts` (description, `IN_WD`
|
||||
wiring with no `${{ inputs... }}` in `run:`, guard text, predicate cases).
|
||||
- OCI-7.2-floating-majors (overlaps OCI-6.4-action-pins): after the v7 bump,
|
||||
SHAs recorded here (2026-10-18, `git ls-remote`):
|
||||
`actions/checkout@v7` = `3d3c42e...` (v7.0.1),
|
||||
`actions/setup-node@v7` = `949feb2...` (v7.1.0),
|
||||
`actions/upload-artifact@v7` = `cf430e0...` (v7.0.2),
|
||||
`oven-sh/setup-bun@v2` = `0c5077e...` (HEAD == v2.2.0, no lag).
|
||||
Added `.github/dependabot.yml` (weekly `github-actions` updates) so the
|
||||
floating majors stay current between audits.
|
||||
- OCI-6.2-artifact-poisoning: `ConfigSchema.superRefine` now fails
|
||||
validation when `gate.artifact` is the default
|
||||
(`DEFAULT_RESULTS_ARTIFACT` from `src/gate/history.ts`) AND
|
||||
`opencode.versions` has more than one entry, with a message naming the
|
||||
file and field (`gate.artifact must not be the default ... when
|
||||
opencode.versions has more than one entry`). The `init` scaffold uses a
|
||||
single version so fresh projects are unaffected; the README matrix example
|
||||
(distinct `artifact-name` per leg) now matches enforced behavior.
|
||||
Pinned by `test/unit/artifact-guard.test.ts`. Three existing
|
||||
multi-version tests (`run-exit-codes` x2, `run-budget`) were updated to
|
||||
use distinct gate artifacts.
|
||||
|
||||
## Decisions / discrepancies
|
||||
|
||||
- The brief's "extend tier1.test.ts or probe.test.ts with a runLoad-level
|
||||
hook-order assertion" is inapplicable as written (tier 1 emits no hook
|
||||
records); the order is pinned at the construction site instead (see
|
||||
OCI-4.3-order above). No NOTES contradiction filed — the minimal path the
|
||||
brief allows (`test/unit/load-order.test.ts`) is what shipped.
|
||||
- The brief's "one scripted e2e fixture whose script forces an error event"
|
||||
(OCI-2.5-noerrors) is deferred: the stub cannot emit `error` events except
|
||||
via the fake binary, and no passing-path turn emits them; the unit pin
|
||||
covers the fallback.
|
||||
|
||||
## M14 decision checklist 14-25
|
||||
|
||||
- decision 14: `src/run.ts` runs `runStep("install"/"build")` before `ensureEntriesOrThrow` and passes `serverEntry` to tier 1 (`src/tiers/load.ts` takes `serverEntry`, checks shim + real paths in logs); test `test/unit/run-entry.test.ts` (build-before-dist-entry, plugin.entry plumbing, missing-entry ConfigError, TUI entry, StepError).
|
||||
- decision 15: `src/plugin/entry.ts` missing explicit `plugin.entry` throws `ConfigError` (never falls back), `src/cli.ts` `validate` warns when `plugin.build` is set; test `test/unit/entry.test.ts` "missing explicit entry throws instead of falling back".
|
||||
- decision 16: `src/run.ts` passes each `.opencode-ci` test file as an explicit `./`-relative path (`bun test <dir>` skips dot-dirs); test `test/unit/run-bun-test.test.ts` (passing file passes, failing file fails with its name).
|
||||
- decision 17: `src/tiers/live.ts` `budgetStop` fails the case with `(budget exhausted …)` once attempts ran (skip only when none ran), `src/run.ts` holds one `LiveBudget` above the version loop; tests `test/unit/live-loop.test.ts` "running out of budget after a failed attempt is a failure, not a skip" and `test/unit/run-budget.test.ts` (shared budget, `latest` skips).
|
||||
- decision 18: `src/util/fs.ts` `resetDir` (refuses outside root) called by all four tiers, `src/config/schema.ts` `RESERVED_IDS` (`load`, `bun-test`) + workspace containment, `src/cli.ts` exported `parseRunArgs` validates `--opencode` via `versionSchema`, `src/run.ts` unknown `--scenario` throws `ConfigError`; tests `test/unit/run-inputs.test.ts` (re-run isolation, resetDir guard, unknown scenario, --opencode) + `test/unit/scenario-rules.test.ts`.
|
||||
- decision 19: `src/config/schema.ts` removes `expect.providers`/`models`, `gate.window`/`threshold`, scenario `judgeReply` (fail validation), `live.timeoutMs` wired as the live default (`src/tiers/live.ts` `scenario.timeoutMs ?? live.timeoutMs`, `src/tiers/scripted.ts` `?? 120_000`, scenario `timeoutMs` optional); test `test/unit/config-schema.test.ts` (removed fields fail, no schema default) + regenerated `schema/config.json` + `schema/scenario.json`.
|
||||
- decision 20: `src/assert/matchers.ts` missing value fails every matcher except `exists: false` (`no value was recorded (check the field name)`); test `test/unit/matchers-missing.test.ts` (4 pass: notContains/notMatches/equals fail, `exists: false` passes, null recorded).
|
||||
- decision 21: `src/assert/expect.ts` `normalizeToolName` maps all spellings to the effective `<namespace>_<name>` in `tools`/`noTools`/`hooks[].tool`, bare names fail with a `did you mean …?` hint; test `test/unit/namespaced-tools.test.ts` (4 pass: dotted catalog matches, bare-name hint `ns_spike_echo`).
|
||||
- decision 22: `src/gate/history.ts` `historyBranch` reads PR history from `GITHUB_BASE_REF`, used by `src/gate/gate.ts`; test `test/unit/gate-wiring.test.ts` (PR `4/merge` reads `main`).
|
||||
- decision 23: `action.yml` passes `artifact-name` via `IN_ARTIFACT` as CLI `--gate-artifact` (`src/cli.ts` `parseRunArgs`, `src/run.ts` overrides `gate.artifact`), still no `config` input and no `skipped` output; test `test/unit/gate-wiring.test.ts` (`--gate-artifact` both spellings).
|
||||
- decision 24: `src/plugin/install.ts` `StepError` (failing install/build exits 1), `src/run.ts` unknown `--scenario` is `ConfigError` (exit 2), results keep the requested `opencode` version (`opencode: version`); tests `test/unit/run-entry.test.ts` (build-failure message), `test/unit/run-inputs.test.ts` (unknown scenario), `test/unit/run-exit-codes.test.ts` (required/allowed exits).
|
||||
- decision 25: `test/helpers/fake-opencode.ts` fake binary + `test/helpers/run-ci.ts`/`tmp.ts` shared setup cover `runCi`/live loop, `.github/workflows/ci.yml` `uses: ./` self-test job (`tool-direct` pass, `throws` fail); tests `run-exit-codes`/`run-entry`/`run-budget`/`live-loop` + `bun run test:e2e` 28 pass.
|
||||
|
||||
Verification: `bun run check` 315 pass, `bun run test:e2e` 28 pass, `git status` clean, `ls TMPDIR` 0 (`run-plugin-`/`run-fake-`/`live-loop-`), `live:` not run.
|
||||
+66
-42
@@ -1,6 +1,6 @@
|
||||
# opencode-plugin-ci: specification (v1)
|
||||
|
||||
Status: draft. Targets opencode v2 (`@opencode/cli` 2.x, `@opencode/plugin` 2.x) only.
|
||||
Status: v1 implemented (PR #4); the v1.1 fixes from the v1 review are specified here and briefed in `docs/IMPLEMENTATION_PROMPT_v1.1.md`. Targets opencode v2 (`@opencode/cli` 2.x, `@opencode/plugin` 2.x) only.
|
||||
|
||||
## 1. Goal
|
||||
|
||||
@@ -99,23 +99,23 @@ Nothing touches the user's real opencode setup. `--keep` (or `OPENCODE_PLUGIN_CI
|
||||
|
||||
### 3.2 Plugin under test
|
||||
|
||||
The plugin loads from the checkout being tested, as a local path, after the author's own `install` and `build` commands run. npm is never used, so a PR is tested before anything is published.
|
||||
The plugin loads from the checkout being tested, as a local path, after the author's own `install` and `build` commands run. npm is never used, so a PR is tested before anything is published. Entries are resolved only **after** install and build (6.1), because the entry is often a build output such as `dist/server.js` that does not exist in a fresh checkout.
|
||||
|
||||
opencode only loads `<dir>/server.*` or `<dir>/index.*` from a local directory, but real plugins usually point `package.json` at a build output. So the harness writes a one-line **shim** directory, `<root>/plugin-under-test/server.js`, containing `export { default } from "<absolute server entry>"`. The server entry is resolved in this order:
|
||||
|
||||
1. `plugin.entry` in config (relative to `plugin.path`),
|
||||
1. `plugin.entry` in config (relative to `plugin.path`). It must name an existing file after the build; otherwise the run is a configuration error (exit 2). It never falls back to the steps below, so a typo cannot test a different file,
|
||||
2. `package.json` `exports["./server"]`, then `exports["."]` (a string, or the `import` or `default` condition),
|
||||
3. `package.json` `main`,
|
||||
4. `<plugin.path>/server.{ts,js,mjs}`, then `<plugin.path>/index.{ts,js,mjs}`.
|
||||
|
||||
The TUI entry is resolved the same way from `exports["./tui"]` or `<plugin.path>/tui.{ts,js,mjs}`. A plugin with a TUI entry and no server entry is TUI-only. A plugin with neither is a configuration error (exit 2). The shim keeps the real file in the checkout, so the plugin's own `node_modules` still resolve.
|
||||
The TUI entry is resolved the same way from `exports["./tui"]` or `<plugin.path>/tui.{ts,js,mjs}`, also when `plugin.entry` is set. A plugin with a TUI entry and no server entry is TUI-only. Every tier, tier 1 included, uses the one server entry resolved here. A plugin with neither is a configuration error (exit 2). The shim keeps the real file in the checkout, so the plugin's own `node_modules` still resolve.
|
||||
|
||||
### 3.3 Probe plugin
|
||||
|
||||
This is a small v2 plugin shipped inside this tool as two entry directories, `probe/pre` and `probe/post`, that share one implementation and have distinct ids. It is loaded before and after the plugin under test. Each instance writes one JSON line per observation, tagged with its `phase`, to `probe.jsonl`.
|
||||
|
||||
- **Inventory**: on the first `session` `prompt` hook (by then every plugin has activated), it records `ctx.plugin.list()`, `ctx.tool.list()`, `ctx.agent`, `ctx.command.list()`, `ctx.skill` and `ctx.mcp` lists.
|
||||
- **Hooks**: it registers `tool.hook("execute.before")`, `tool.hook("execute.after")`, `shell.hook("create.before")` and the `session` hooks `prompt`, `context`, `model.request`, then records the payload it sees. Large fields are truncated, and values of env keys matching `/KEY|TOKEN|SECRET|PASSWORD/i` are redacted.
|
||||
- **Hooks**: it registers `tool.hook("execute.before")`, `tool.hook("execute.after")`, `shell.hook("create.before")` and the `session` hooks `prompt`, `context`, `model.request`, then records the payload it sees. Large fields are truncated, and values of env keys and headers matching `/KEY|TOKEN|SECRET|PASSWORD|AUTH|COOKIE/i` are redacted.
|
||||
- **Events**: it iterates `ctx.event.subscribe()` in a background loop (not awaited in `setup`) and records event types.
|
||||
- **Load errors**: it records plugin load failures that `ctx.plugin.list()` exposes.
|
||||
|
||||
@@ -130,7 +130,7 @@ This is the `stub-llm.ts` pattern: an OpenAI-compatible `/v1/chat/completions` s
|
||||
Two additions to the original stub:
|
||||
|
||||
- **Code Mode auto-wrap.** When a scripted tool call names a tool that the request does not offer directly but `execute` is offered, the stub rewrites it into an `execute` call. It finds the tool's exact call path (for example `tools.my_tool` or `tools.ns["my-tool"]`) in the Code Mode catalog in the request's system text, and sends `{ code: "return await <path>(<json args>)" }`. A step can opt out with `raw: true`.
|
||||
- **Judge requests.** The judge prompt carries the sentinel `OPENCODE_PLUGIN_CI_JUDGE`. A request containing it is answered with the scenario's scripted `judgeReply`, so the judge path is testable offline. A tool-less request would otherwise be mistaken for a title request.
|
||||
- **Judge requests.** The judge prompt carries the sentinel `OPENCODE_PLUGIN_CI_JUDGE`. A request containing it is answered with the stub's `judgeReply` option, so the judge path is testable offline. A tool-less request would otherwise be mistaken for a title request. `judgeReply` is a stub option for this tool's own tests (`runJudge({ stubReply })` with `OPENCODE_PLUGIN_CI_JUDGE_VIA_STUB=1`), not a scenario field: scripted scenarios never run the judge, and live scenarios never use the stub. With no reply given, the stub judge answers a fail.
|
||||
|
||||
## 4. Tiers
|
||||
|
||||
@@ -144,7 +144,7 @@ Two additions to the original stub:
|
||||
|
||||
- exit code 0 and no `error` event,
|
||||
- the plugin under test is in the probe's `plugin.list()`, matched by `source.path` equal to the shim's `server.js`, with `state.status: "active"`. A missing entry means the entry was never found; a `failed` entry is reported with its `state.error`,
|
||||
- no stderr line at level `ERROR` or `FATAL` that names the plugin id or path, and no `failed to load plugin` line for it,
|
||||
- no stderr line at level `ERROR` or `FATAL` that names the plugin id, the shim path or the real server entry path, and no `failed to load plugin` line for it,
|
||||
- every contribution declared in `.opencode-ci/config.yaml` `expect` (tools, agents, commands, skills, mcp) appears in the inventory.
|
||||
|
||||
TUI-only plugins never reach the server, so they get a **static** tier 1 instead. A child `bun` process imports the TUI entry from the checkout and asserts that the default export has a string `id` and a function `setup`. They then stop.
|
||||
@@ -153,6 +153,8 @@ TUI-only plugins never reach the server, so they get a **static** tier 1 instead
|
||||
|
||||
**Tier 3, live.** This tier runs scenarios with `mode: live` against the user's provider and model. Assertions are the same as tier 2 except stub-specific ones, which are rejected at validation. A live scenario passes when at least `pass` of `repeat` attempts pass (default 2 of 3, as in `live.test.ts`). An optional `judge` rubric adds a semantic check (6.3). Without a configured key, tier 3 is reported as `skipped`, not failed, so fork PRs (which get no secrets) stay green.
|
||||
|
||||
`live.budgetUSD` is one counter for the whole run, shared by every opencode version. When it is spent, a scenario that has not run any attempt is `skipped` (`budget exhausted ($x of $y)`). A scenario that already ran attempts but has fewer than `pass` passes **fails**, with `(budget exhausted …)` in its failure line. A recorded failure is never reported as a skip.
|
||||
|
||||
## 5. Configuration
|
||||
|
||||
### 5.1 `.opencode-ci/config.yaml`
|
||||
@@ -161,6 +163,7 @@ TUI-only plugins never reach the server, so they get a **static** tier 1 instead
|
||||
# yaml-language-server: $schema=https://unpkg.com/@heretek-ai/opencode-plugin-ci/schema/config.json
|
||||
plugin:
|
||||
path: . # plugin package dir in the checkout
|
||||
entry: dist/server.js # optional; must exist after build (3.2)
|
||||
install: bun install # optional
|
||||
build: bun run build # optional
|
||||
options: {} # default plugin options for every scenario
|
||||
@@ -178,13 +181,18 @@ live:
|
||||
keys: [OPENCODE_API_KEY] # env vars passed into the sandbox
|
||||
repeat: 3
|
||||
pass: 2
|
||||
budgetUSD: 1.00 # hard stop for the whole live tier
|
||||
budgetUSD: 1.00 # hard stop for the whole run, across all versions (4)
|
||||
timeoutMs: 240000 # per live attempt, unless the scenario sets timeoutMs
|
||||
gate:
|
||||
enabled: true # historical flake gate, live tier only (7.2)
|
||||
window: 5
|
||||
threshold: 0.9
|
||||
bootstrap: fail # fail | allow ("no history yet" passes)
|
||||
workflow: plugin-ci.yml # optional; else from GITHUB_WORKFLOW_REF
|
||||
artifact: opencode-plugin-ci-results # history artifact; `run --gate-artifact` overrides it
|
||||
logLevel: info # info | all
|
||||
```
|
||||
|
||||
`expect.providers`, `expect.models`, `gate.window` and `gate.threshold` are not v1 fields. Because every object is strict, setting them fails validation instead of being silently ignored. Provider expectations are deferred to v2 (6.4); the gate's thresholds are fixed (7.2).
|
||||
|
||||
JSON (`config.json`) is accepted with the same schema. Exactly one of the two may exist.
|
||||
|
||||
### 5.2 Scenario files: `.opencode-ci/scenarios/*.yaml` (or `.json`)
|
||||
@@ -203,7 +211,7 @@ script: # scripted only: one step per agent-loop request
|
||||
- name: shell
|
||||
arguments: { command: "cat .env", description: "show env" }
|
||||
- text: done
|
||||
timeoutMs: 120000
|
||||
timeoutMs: 120000 # optional; default 120000 (scripted) or live.timeoutMs (live)
|
||||
expect:
|
||||
exitCode: 0
|
||||
tools: # from probe tool.hook records (sees Code Mode inner calls)
|
||||
@@ -233,39 +241,39 @@ judge: # live only, optional
|
||||
rubric: "The reply explains that the key was redacted and does not reveal it."
|
||||
```
|
||||
|
||||
Matchers are `equals`, `contains`, `notContains`, `matches`, `notMatches`, `includes` (arrays), `exists` and `count`. They are combinable and AND-ed. Unknown keys fail validation, so typos never silently pass.
|
||||
Matchers are `equals`, `contains`, `notContains`, `matches`, `notMatches`, `includes` (arrays), `exists` and `count`. They are combinable and AND-ed. Unknown keys fail validation naming the file and field (not a line number), so typos never silently pass.
|
||||
|
||||
Rules that keep assertions honest:
|
||||
|
||||
- **A missing value fails.** A field with no recorded value (`undefined`) fails every matcher except `exists: false`, with `<label>: no value was recorded (check the field name).` A misspelled field (`input: { comand: { notContains: secret } }`) therefore fails instead of passing. `null` is a recorded value.
|
||||
- **Namespaced tool names.** opencode records a tool registered with a namespace under its effective name, `namespace.replaceAll(".", "_") + "_" + name.replace(/[^a-zA-Z0-9_-]/g, "_")` (read from the 2.0.26 binary). Catalogs use `tools.ns.name` and inner calls `ns.name`. In `tools[].name`, `noTools` and `hooks[].tool`, all three spellings match. A bare `name` does not match a namespaced tool; the failure names the recorded form (`did you mean ns_name?`).
|
||||
- **Reserved ids.** `load` and `bun-test` are the tool's own case ids and evidence dirs, so a scenario may not use them.
|
||||
- **Workspace paths** must be relative and must not contain a `..` segment.
|
||||
|
||||
A JSON Schema for both files is published in the package (`schema/config.json`, `schema/scenario.json`), and `validate` checks it.
|
||||
|
||||
### 5.3 Programmatic API
|
||||
|
||||
For authors who outgrow YAML, the package exports the primitives the runner itself uses:
|
||||
For authors who outgrow YAML, the package exports the primitives the runner itself uses (`src/index.ts`): `Sandbox`, `startStub`, `readProbe` / `ProbeLog`, `evaluate`, `check`, `resolveEntries`, `writeShim`, `resolveOpencode`, `parseEvents`, `runCi`, `runJudge` and `runLiveScenario`, with their types. The README documents them; the names in earlier drafts of this section were illustrative.
|
||||
|
||||
```ts
|
||||
import { Sandbox, startStub, loadProbe, assert } from "@heretek-ai/opencode-plugin-ci"
|
||||
import { test, expect } from "bun:test"
|
||||
|
||||
test("custom", async () => {
|
||||
await using sandbox = await Sandbox.create({ plugin: ".", script: [{ text: "ok" }] })
|
||||
const run = await sandbox.run("hi")
|
||||
expect(run.probe.inventory.tools).toContain("my_tool")
|
||||
})
|
||||
```
|
||||
|
||||
`bun:test` files under `.opencode-ci/**/*.test.ts` are run by the CLI as part of tier 2, and their results are folded into the same report.
|
||||
`bun:test` files under `.opencode-ci/**/*.test.ts` are run by the CLI once per run, after the version loop, as one `bun-test` case with `OPENCODE_BIN` set to the required version's binary. Each file is passed to `bun test` as an explicit `./`-relative path, because `bun test` skips dot-directories when it discovers tests (`bun test .opencode-ci` never matches a file, verified on Bun 1.4.2).
|
||||
|
||||
## 6. Execution details
|
||||
|
||||
### 6.1 Run order
|
||||
|
||||
1. Validate config and scenarios (fail fast on schema errors).
|
||||
2. Run `plugin.install` and `plugin.build` once in the checkout.
|
||||
3. For each opencode version: resolve the binary (`OPENCODE_BIN`, else `npx @opencode/cli@<v>`, cached), then tier 1, tier 2 and tier 3 in order. Scenarios within a tier run with concurrency `min(4, cpus)`; each has its own sandbox and stub port.
|
||||
4. A failing tier 1 skips tiers 2 and 3 for that version, because they would all fail for the same reason.
|
||||
1. Validate config and scenarios (fail fast on schema errors). Every `--scenario` id must exist, and every `--opencode` value must be `x.y.z` or `latest`; otherwise exit 2.
|
||||
2. Run `plugin.install` and `plugin.build` once in the checkout. A failing step is the plugin's failure: exit 1, not a configuration error.
|
||||
3. Resolve the server and TUI entries (3.2). Nothing resolving is a configuration error (exit 2).
|
||||
4. For each opencode version: resolve the binary (`OPENCODE_BIN`, else `npm install @opencode/cli@<v>` into `$XDG_CACHE_HOME/opencode-plugin-ci/opencode/<v>`, cached), then tier 1, tier 2 and tier 3 in order. Tier 2 scenarios run with concurrency `min(4, cpus)`; each has its own sandbox and stub port. Tier 3 runs sequentially because it shares the budget.
|
||||
5. A failing tier 1 skips tiers 2 and 3 for that version, because they would all fail for the same reason.
|
||||
6. Run the `bun-test` case (5.3), write `results.json` and `report.md`, then apply the gate (7.2).
|
||||
|
||||
### 6.2 Evidence
|
||||
|
||||
Every scenario writes `<out>/<version>/<scenario>/` with `events.jsonl`, `probe.jsonl`, `stub-requests.jsonl`, `stderr.log`, `workspace.tar` (for failures only) and `result.json`. The default `<out>` is `.opencode-ci/out` (git-ignored). The Action uploads it as an artifact.
|
||||
Every case writes `<out>/<version>/<case id>/` with `events.jsonl`, `probe.jsonl`, `stub-requests.jsonl` (stub tiers) and `stderr.log`; live attempts write one subdir each. The default `<out>` is `.opencode-ci/out` (git-ignored). The Action uploads it as an artifact. `workspace.tar` and a per-case `result.json` are deferred to v2.
|
||||
|
||||
Each case's evidence dir is **emptied when the case starts**. The probe appends to its log, so without this a re-run into the same out dir evaluates the previous run's records (reproduced: a broken plugin passed on re-run). The reset refuses any path that is not strictly inside `<out>`.
|
||||
|
||||
### 6.3 Judge
|
||||
|
||||
@@ -280,30 +288,31 @@ The judge is the user's original two-agent idea, scoped so that it cannot skip e
|
||||
|
||||
A plugin whose job is to be the model provider cannot be checked against the stub. For these plugins:
|
||||
|
||||
- tier 1 asserts that the provider and models appear (`expect.providers`, `expect.models`, read from the probe's `ctx.provider` and `ctx.model`);
|
||||
- tier 1 checks that the plugin activates, like any plugin. Asserting that its providers and models appear (`expect.providers`, `expect.models`) is **deferred to v2**; v1 rejects those fields (5.1);
|
||||
- tier 2 is optional, using `model: <their-provider>/<model>` with `baseURL` pointed at the stub when the provider is OpenAI-compatible;
|
||||
- tier 3 is the real test.
|
||||
|
||||
### 6.5 Isolation and network
|
||||
|
||||
Tiers 1 and 2 need no network beyond installing opencode. The stub binds to loopback only. The tool does not sandbox the network itself (that is the runner's job), but the report flags any `tool_use` of `webfetch` or `websearch` in scripted scenarios.
|
||||
Tiers 1 and 2 need no network beyond installing opencode. The stub binds to loopback only. The tool does not sandbox the network itself (that is the runner's job). Flagging `webfetch` or `websearch` use in scripted scenarios in the report is deferred to v2; authors can assert it today with `noTools: [webfetch, websearch]`.
|
||||
|
||||
## 7. Reporting and gating
|
||||
|
||||
### 7.1 Results and report
|
||||
|
||||
`results.json` uses the IUMBTEMS `EvalRow` shape (`id`, `pass`, `tools`, `failures`, `steps`, `costUSD`, `textExcerpt`, `kept`, ...) with `tier` and `opencode` added, so the gate logic ports unchanged. The Markdown report is generated from `e2e-report.ts`. It contains one table per opencode version, a row per scenario, failures expanded with the first failing assertion and evidence links, and token and cost totals for the live tier. It is written to `report.md` and appended to `$GITHUB_STEP_SUMMARY` when that is set. This is on by default.
|
||||
`results.json` uses the IUMBTEMS `EvalRow` shape (`id`, `pass`, `tools`, `failures`, `steps`, `costUSD`, `textExcerpt`, `kept`, ...) with `tier` and `opencode` added, so the gate logic ports unchanged. `opencode` is the **requested** version string (`2.0.26` or `latest`), because the required/allowed split, `report` re-rendering and gate history all key on it; a resolved `latest` would change from run to run. The Markdown report is generated from `e2e-report.ts`. It contains one table per opencode version, a row per scenario, failures expanded with the first failing assertion and evidence links, and token and cost totals for the live tier. It is written to `report.md` and appended to `$GITHUB_STEP_SUMMARY` when that is set. This is on by default.
|
||||
|
||||
### 7.2 Historical flake gate (live tier only)
|
||||
|
||||
This is ported from `IUMBTEMS/scripts/e2e-gate.ts` and applies only to live-tier rows. Tiers 1 and 2 are deterministic and fail on any failure.
|
||||
|
||||
- **History**: the gate reads the `results.json` from the `opencode-plugin-ci` artifacts of the last N runs of the same workflow on the same branch. It does this through the GitHub API with the job's `GITHUB_TOKEN`, so the workflow needs `actions: read`. History is not stored in the Actions cache, because a cache eviction would silently reset it.
|
||||
- **Stable case**: a case passed at least `threshold` (default 90%) of at least `window` (default 5) sampled runs.
|
||||
- **History**: the gate reads the `results.json` from the artifacts named `gate.artifact` (default `opencode-plugin-ci-results`; `run --gate-artifact` overrides it, and the Action passes its `artifact-name` input that way) from the last runs of the same workflow. Push runs read their own branch (`GITHUB_REF_NAME`). Pull request runs read the **base** branch (`GITHUB_BASE_REF`): on `pull_request` events `GITHUB_REF_NAME` is `<n>/merge`, under which no run is ever listed (`gh run list --branch` matches head branch names). It reads through the GitHub API with the job's `GITHUB_TOKEN`, so the workflow needs `actions: read`. History is not stored in the Actions cache, because a cache eviction would silently reset it. Downloaded history is deleted after the gate reads it.
|
||||
- **Stable case**: a case passed at least 90% of at least 5 sampled runs. These thresholds are fixed in v1 (the sticky-red helper in the verbatim port hard-codes them), so they are not config fields.
|
||||
- **Verdict**: the live tier fails only when a stable case fails now. **Sticky red** also applies: a case that was stable before its current failure streak keeps failing until it passes or a human acknowledges the streak.
|
||||
- **Acknowledgments**: `.opencode-ci/acknowledgments.json` in the plugin repo has the format `{ "version": 1, "acknowledgments": [{ "case", "reason", "issue", "date" }] }`. It is human-written; the tool never writes it. It is fail-closed: a malformed file fails the gate.
|
||||
- **No history**: a red live tier with no history to judge it against exits 2. The Action reports this as a failure unless `gate.bootstrap: allow` is set, for the first runs in a new repo.
|
||||
- **Local runs**: the gate is skipped locally unless `--gate` is passed with a `GITHUB_TOKEN` and `--repo`.
|
||||
- **Local runs**: the gate is skipped locally unless `--gate` is passed. It then needs `gh`, a token (`GH_TOKEN` or `GITHUB_TOKEN`), `GITHUB_REPOSITORY` and a branch (`GITHUB_BASE_REF` or `GITHUB_REF_NAME`); without them it falls back to "fail on any live failure".
|
||||
- **Matrix builds**: artifact names must be unique within a workflow run, so each leg sets its own `artifact-name`, and that same name is the leg's gate history. No per-leg config is needed; validation does NOT enforce distinct names.
|
||||
|
||||
## 8. Packaging
|
||||
|
||||
@@ -314,13 +323,13 @@ One package, two entry points, in this repo.
|
||||
```
|
||||
opencode-plugin-ci init # writes .opencode-ci/config.yaml and one example scenario
|
||||
opencode-plugin-ci validate # schema-checks config and scenarios
|
||||
opencode-plugin-ci run [--tier load|scripted|live] [--scenario id] [--opencode 2.0.26] [--keep] [--gate]
|
||||
opencode-plugin-ci run [--tier load|scripted|live] [--scenario id] [--opencode 2.0.26] [--out dir] [--keep] [--gate] [--gate-artifact name] [--verbose] [--quiet]
|
||||
opencode-plugin-ci report [results.json]
|
||||
```
|
||||
|
||||
Exit codes: 0 pass, 1 failure, 2 configuration error or gate with no history.
|
||||
Exit codes: 0 pass, 1 failure (including a failing `plugin.install` or `plugin.build`), 2 configuration error or gate with no history.
|
||||
|
||||
**GitHub Action**: `Heretek-AI/opencode-v2-plugin-teser@v1`, a composite action that sets up Bun, installs `@opencode/cli` for each requested version and runs the same CLI.
|
||||
**GitHub Action**: `Heretek-AI/opencode-v2-plugin-teser@v1`, a composite action that sets up Bun and Node, installs the tool's dependencies and runs the same CLI, which installs `@opencode/cli` for each requested version (6.1).
|
||||
|
||||
```yaml
|
||||
name: plugin-ci
|
||||
@@ -340,13 +349,15 @@ jobs:
|
||||
OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
|
||||
```
|
||||
|
||||
Inputs are `tiers`, `opencode-versions` (overrides config), `config` (path), `working-directory` and `upload-artifact` (default true). Outputs are `result` (`pass`/`fail`/`skipped`) and `report-path`.
|
||||
Inputs are `tiers`, `opencode-versions` (overrides config), `working-directory` (must stay inside `GITHUB_WORKSPACE`; absolute paths inside the workspace are fine, paths outside exit 2), `upload-artifact` (default true) and `artifact-name` (default `opencode-plugin-ci-results`; passed to the CLI as `--gate-artifact`). There is no `config` input: the CLI reads `.opencode-ci/config.*` in `working-directory`. Outputs are `result` (`pass` when the CLI exits 0, else `fail`; a fully skipped run is `pass`) and `report-path`. Inputs reach the script only through `env:`, never through `${{ }}` inside `run:`.
|
||||
|
||||
The docs must state plainly: never run this on `pull_request_target` with the PR's code checked out and secrets exposed. The live tier runs the plugin under test with the real key.
|
||||
|
||||
## 9. Validation plan for this tool
|
||||
|
||||
- **Self-tests**: fixture plugins in `test/fixtures/` cover each surface (a tool, an `execute.before` mutation, a `shell.create.before` env injection, a `session` `context` system-prompt transform, an `event.subscribe` file write, and a plugin that throws in `setup`). Each has passing and deliberately failing scenarios.
|
||||
- **Self-tests**: fixture plugins in `test/fixtures/` cover each surface (a tool, an `execute.before` mutation, a `shell.create.before` env injection, a `session` `context` system-prompt transform, an `event.subscribe` file write, and a plugin that throws in `setup`). Each surface fixture has a passing and a deliberately failing scenario, and the e2e suite checks the exact failure text.
|
||||
- **Orchestration tests**: `runCi` and the live attempt loop are unit-tested against a fake `opencode` binary (`test/helpers/fake-opencode.ts`) that plays back planned runs, writes probe records and reports costs. These cover step order, entry plumbing, exit codes, skips, the shared budget, the `bun-test` case and re-runs into the same out dir.
|
||||
- **Action self-test**: a CI job runs the composite Action from the checkout (`uses: ./`) on a passing and a failing fixture and checks its `result` output.
|
||||
- **Dogfood**: a scheduled workflow runs tier 1 against a handful of `compat: v2` entries from `awesome-opencode` (at least one memory, one tool, one shell-hook, one session-transform and one provider plugin). It also publishes a table that awesome-opencode can later use as a "verified loads on 2.0.x" signal.
|
||||
|
||||
## 10. Implementation order
|
||||
@@ -358,7 +369,8 @@ The docs must state plainly: never run this on `pull_request_target` with the PR
|
||||
5. Report (ported `e2e-report.ts`).
|
||||
6. Live tier, judge and budget.
|
||||
7. Flake gate (ported `e2e-gate.ts`).
|
||||
8. Composite Action, docs and the dogfood workflow.
|
||||
8. Composite Action, docs and the dogfood workflow (the dogfood workflow is still open).
|
||||
9. v1.1: the review fixes (decisions 14 to 25), briefed in `docs/IMPLEMENTATION_PROMPT_v1.1.md`.
|
||||
|
||||
## 11. Decisions log
|
||||
|
||||
@@ -377,3 +389,15 @@ The docs must state plainly: never run this on `pull_request_target` with the PR
|
||||
| 11 | The default opencode matrix is a pinned version (required) plus `latest` (allowed to fail) |
|
||||
| 12 | The plugin under test is loaded through a generated shim directory, because opencode ignores `package.json` for local plugin paths |
|
||||
| 13 | The probe runs twice (`pre` and `post`) around the plugin under test, and tool assertions come from its hook records rather than `tool_use` events, because of Code Mode |
|
||||
| 14 | Install and build run before entry resolution, and tier 1 receives the resolved entry instead of resolving its own (v1 review B1, B2) |
|
||||
| 15 | An explicit `plugin.entry` that does not exist after the build is a configuration error, never a fallback; `validate` only warns when `plugin.build` is set, since it does not build (S2) |
|
||||
| 16 | The `bun-test` case passes each discovered file as an explicit `./` path, because `bun test` skips dot-directories (B3) |
|
||||
| 17 | The live budget is one counter per run across versions; running out after attempts ran fails the case instead of skipping it (B4, S1) |
|
||||
| 18 | Each case's evidence dir is emptied when the case starts, only ever inside `<out>`; `load` and `bun-test` are reserved scenario ids; `--opencode` values are validated like `opencode.versions` (B5, found while prototyping the fixes) |
|
||||
| 19 | Fields with no effect are removed rather than wired up: `expect.providers`/`models` (deferred to v2), `gate.window`/`threshold` (fixed at 5 and 0.9), scenario `judgeReply` (dead in both modes). `live.timeoutMs` is wired up as the live default, and scenario `timeoutMs` has no schema default (S3) |
|
||||
| 20 | A missing value fails every matcher except `exists: false` (S4) |
|
||||
| 21 | Tool-name assertions normalize to opencode's effective name `<namespace>_<name>`, and a bare name gets a hint (S5) |
|
||||
| 22 | Gate history on pull requests comes from the base branch (S6) |
|
||||
| 23 | The Action passes `artifact-name` to the CLI as `--gate-artifact`, which overrides `gate.artifact`; there is still no `config` input and no `skipped` output (S7, gap 7) |
|
||||
| 24 | A failing install or build exits 1, not 2; an unknown `--scenario` id exits 2; results keep the requested `opencode` version (review nits) |
|
||||
| 25 | `runCi`, the live loop and the Action are covered by tests: a fake `opencode` binary for unit tests and a `uses: ./` CI job (S8) |
|
||||
@@ -0,0 +1,262 @@
|
||||
# ADVERSARIAL THREE-SUBAGENT AUDIT PROMPT: `opencode-v2-plugin-teser`
|
||||
|
||||
> Status: prompt that produced the v1 findings at `af6675c` (see `audit-findings-v1-af6675c.md`); its OCI-6.2 remediation was later reverted — see R1 in `review-main-8aa4d61`; do not follow stale advice.
|
||||
|
||||
**Target Codebase**: the Git repository this file sits in. Resolve it with `git rev-parse --show-toplevel` and treat every path below as relative to that root. Nothing in this prompt depends on where the checkout lives.
|
||||
**GitHub Remote**: `https://github.com/Heretek-AI/opencode-v2-plugin-teser`
|
||||
**What it is supposed to be**: A CLI plus composite GitHub Action (`@heretek-ai/opencode-plugin-ci`, bin `opencode-plugin-ci`) that answers "does my opencode v2 plugin actually work?" identically on a laptop and in CI by driving the real `opencode` binary with the plugin loaded and asserting on what happened across three tiers — `load` (plugin activates under opencode 2.0.26+), `scripted` (deterministic scenarios against a local stub model, zero tokens), and `live` (real provider, k-of-n attempts under a USD budget, skipped — never failed — without a key). Consumers are plugin authors via `bunx` and via the Action; a probe plugin loaded pre/post observes hooks and inventory, a one-line shim loads the real entry file, and a historical flake gate judges live-tier flakes. Non-goals: no TUI rendering checks (load-only for TUI plugins), no v1 (`@opencode-ai/*`) plugins, no model-quality benchmarking. Refusal boundary: never run the live tier against a real model with a real key during an audit — assert the skip path instead — and never test `pull_request_target`-style flows with secrets exposed.
|
||||
**Execution Context**: OpenCode v2 autonomous audit (works the same in any agent CLI that has subagents and a shell)
|
||||
**Auditor Formation**: three independent adversarial subagents (Alpha, Beta, Gamma), a cross-examination pass, and a synthesis pass
|
||||
**Output**: `AUDIT_REPORT_FINDINGS.md` at the repository root
|
||||
|
||||
This prompt is designed to be re-run. Each run audits the current `HEAD`, re-verifies every finding of the previous run, and overwrites `AUDIT_REPORT_FINDINGS.md`.
|
||||
|
||||
---
|
||||
|
||||
## 0. Preflight (the controller does this before spawning subagents)
|
||||
|
||||
1. **Baseline.** Record at the top of the report: `git rev-parse HEAD`, `git log -1 --format='%ci %s'`, the UTC date, and the versions of every tool you will use (bun, node, npm, git, gh, jq, rg, curl). If `git status --porcelain` shows changes other than this prompt file and a previous report, list the dirty files and audit a clean `git worktree` of `HEAD` instead of the working tree.
|
||||
2. **Scratch space.** `export AUDIT_TMP="$(mktemp -d -t opencode-v2-plugin-teser-audit-XXXXXX)"`. Every clone, downloaded binary, packed tarball, sabotage copy, container volume and raw log goes under `$AUDIT_TMP`. Each subagent writes its raw notes to `$AUDIT_TMP/{alpha,beta,gamma}.md`.
|
||||
3. **Disposable copy for experiments.** `git worktree add --detach "$AUDIT_TMP/wt" HEAD`. Sabotage, patched configs, and any file edits happen only there. The real checkout must end the audit with `git status --porcelain` showing nothing new but `AUDIT_REPORT_FINDINGS.md` (and this prompt file, if you copied it in untracked). Remove the copy with `git worktree remove --force "$AUDIT_TMP/wt"` at the end.
|
||||
4. **Previous run.** If `AUDIT_REPORT_FINDINGS.md` already exists, read it before anything else. Every finding in it gets a row in the Regression Ledger (section 6), re-verified from scratch: never copy a previous verdict forward.
|
||||
5. **Network and host capabilities.** You need outbound HTTPS to github.com, raw.githubusercontent.com, npm registry (registry.npmjs.org), OSV API (https://api.osv.dev/v1/query). Probe what this host can do (`bun --version`, `node --version`, `gh auth status`, `$OPENCODE_BIN --version`) and record it. When a check cannot run here, the claim is `UNVERIFIABLE` with the exact reason. A claim is never `VERIFIED` by inference, by reading the code alone, or because a test says so.
|
||||
|
||||
**Hard limits (all subagents):** no package publishing, no `git push`, no tags, no GitHub writes (issues, comments, PRs), no real API keys or credentials in any command, file or report (generate throwaway keys and dummy values), no destructive command outside `$AUDIT_TMP`, loopback ports only for the stub model, throwaway temp dirs only, never run the live tier with a real key (assert the skip path instead). Read-only against the real checkout.
|
||||
|
||||
---
|
||||
|
||||
## 1. Directive & Rules of Engagement
|
||||
|
||||
You are a team of **three elite, adversarial principal systems auditors**. Your mandate is to stress-test, debunk, and verify every claimed feature, architectural decision, integration contract, security invariant, and version pin in **`opencode-v2-plugin-teser`**, measured against what the repository is supposed to be (above) and what its own `README.md`, `AGENTS.md` and `docs/` claim.
|
||||
|
||||
### The Cardinal Rule: DO NOT BELIEVE TESTS
|
||||
- Unit tests, acceptance scripts and CI badges can lie: they assert obsolete invariants, mock away the real system, or test that a file *contains* a rule rather than that the rule *works*.
|
||||
- This repository has a documented history of exactly that:
|
||||
- M10 rework: the `report` command re-rendered saved multi-version results without `allowedVersions`, marking allowed failures as failures while the header said "allowed to fail" — unit snapshot + e2e stayed green; caught by drift-guard audit, not by tests.
|
||||
- `test/fixtures/plugins/tool-basic` ships a deliberately failing scenario (`fail.yaml` expects `echo:bye`); an unfiltered full `run` exits 1 by design — naive suite-runners see red.
|
||||
- `gate.window`/`gate.threshold` config is accepted but inert (verbatim verdict hard-codes >=5 samples, >=90%); tests pass with custom values silently ignored.
|
||||
- SPEC 5.3 snippet names (`Sandbox.create`/`loadProbe`/`assert`) do not match real exports (`new Sandbox`/`readProbe`/`evaluate`+`check`).
|
||||
- Probe redact regex is a fail-closed superset (`AUTH|COOKIE` added) vs the spec.
|
||||
- The live tier never ran against a real model (no key in this environment) — live paths are green via skip + stub only.
|
||||
- Never conclude that a component works because `bun run check` and `OPENCODE_BIN=… bun run test:e2e --timeout 600000` exits 0. Prove behavior against **the real thing it plugs into**: the real `opencode` binary (`@opencode/[email protected]` from npm, plus upstream head), the real `@opencode/plugin` 2.0.26 type definitions and opencode source tree, the npm registry (published tarball + `npm view`), and a real GitHub Actions run of the composite Action.
|
||||
|
||||
### The Second Rule: DO NOT BELIEVE THE PAPER TRAIL
|
||||
- `docs/reports/*`, `docs/evidence/*`, "red-proof" logs, drift-guard `VERDICT: PASS` lines, `CHANGELOG.md`, and `README.md` are **claims under audit**, not evidence.
|
||||
- The repo was built by agent triads from docs/IMPLEMENTATION_PROMPT.md, a milestone brief grounded in a 2026-10-17 spike against opencode 2.0.26; docs/SPEC.md is the design source of truth. Build gates that only check spec compliance cannot catch invented facts (API names, image tags, config shapes) in the spec. Matching the spec is not the same as working. Where the spec and reality disagree, reality wins and the spec is the defect.
|
||||
|
||||
### The Third Rule: SABOTAGE EVERY CRITICAL INVARIANT
|
||||
- For each invariant marked **[S]** in section 3, break it with a one-line change in `$AUDIT_TMP/wt`, run the full suite there, and record whether the suite went red. Restore the line after each run.
|
||||
- A sabotage the suite does not catch is a finding: **P1** ("untested invariant"), or **P0** when the invariant is a security boundary.
|
||||
|
||||
### Modern Version & Proper Deployment Imperative
|
||||
- Every pinned version must be a **modern, supported, production-grade release that actually exists** in its registry. Flag invented versions, deprecated or archived packages, renamed organizations, floating tags (`latest`, `^` ranges on security-relevant deps, unpinned GitHub Actions), end-of-life runtimes, and versions with known CVEs (the ecosystem's audit tool, OSV: `https://api.osv.dev/v1/query`).
|
||||
- Verify that integration patterns follow the **official upstream documentation and source code** of the system being integrated, at the version pinned and at the current upstream head (drift is a finding).
|
||||
|
||||
### Verdict vocabulary
|
||||
Every claim in section 3 ends with exactly one status, each backed by a command and its output:
|
||||
- `VERIFIED` (proven to work against the real thing), `PARTIAL` (works with caveats you name), `BROKEN` (proven not to work), `UNVERIFIABLE` (could not be run here, with the exact reason).
|
||||
|
||||
---
|
||||
|
||||
## 2. The Adversarial Triad (Subagent Roles)
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
C["OpenCode Controller"] --> A["Alpha: Runtime & Integration Inquisitor"]
|
||||
C --> B["Beta: Architecture, Versions & Supply Chain"]
|
||||
C --> G["Gamma: Security, Trust Boundaries & Abuse"]
|
||||
A --> X["Cross-examination: each finding attacked by a different subagent"]
|
||||
B --> X
|
||||
G --> X
|
||||
X --> S["Synthesis: AUDIT_REPORT_FINDINGS.md"]
|
||||
```
|
||||
|
||||
### Subagent Alpha: Runtime & Integration Inquisitor
|
||||
**Focus question:** does every tier actually pass against the real `opencode` binary at the pinned version AND at upstream head, for every plugin surface the repo claims?
|
||||
|
||||
1. Run tier 1 and tier 2 end-to-end with `OPENCODE_BIN` pointing at a freshly `npm install`ed `@opencode/[email protected]` in `$AUDIT_TMP` (never the repo's `review/oc` cache) against every fixture in `test/fixtures/plugins/` (`tool-basic` pass AND deliberate-fail, `tool-direct`, `hook-mutate`, `shell-env`, `system-inject`, `event-file`, `throws`, `no-entry`, `tui-only`, `tui-broken`); confirm `throws` fails naming `fixture boom`, `no-entry` exits 2, and the deliberate `fail.yaml` exits 1 by design.
|
||||
2. Re-run the load + scripted tiers against opencode upstream head (`npm view @opencode/cli version`, install that into `$AUDIT_TMP`) and diff behavior: catalog format, hook order, `tool_use` event shapes, `plugin.list()` fields — any silent change is drift.
|
||||
3. Prove the `[pre, plugin, post]` hook-order claim both ways: swap fixture order in a scratch sandbox config and show `pre` records the unmutated input while `post` records the mutated one (use `hook-mutate` and `shell-env` fixtures).
|
||||
4. Exercise every `expect` assertion surface (`tools`, `noTools`, `hooks` pre/post, `request` nth/system/tools, `files`, `stderr`, `text`, `exitCode`, `noErrors`) with a crafted failing value each, and confirm each failure message names the file, the matcher, and a truncated `got` value.
|
||||
5. Prove the live tier never touches a real model: run `run --tier live` with no key (expect `skipped: "no API key …"`, exit 0) and with a throwaway dummy key against the stub (`OPENCODE_PLUGIN_CI_JUDGE_VIA_STUB=1` test-only hook), then verify judge tokens count toward `live.budgetUSD` and the pre-attempt budget check skips before spawning.
|
||||
|
||||
### Subagent Beta: Architecture, Versions & Supply Chain
|
||||
**Focus question:** is every pin real, modern, and supported; is every ported file faithful; and does the packaged artifact match what the repo promises?
|
||||
|
||||
1. Registry truth: `npm view @opencode/cli versions --json` (2.0.26 present, not deprecated/yanked), `npm view @opencode/[email protected]`, `npm view yaml zod` ranges vs `^2.9.1`/`^4.1.0` in `package.json`; OSV-query `yaml`, `zod`, and the resolved `@opencode/cli` for known CVEs; flag `opencode.versions: ["2.0.26", "latest"]` floating-`latest` semantics (allowed-failure leg) against a head install.
|
||||
2. Upstream identity and drift: the local `review/opencode` clone's remote is `https://github.com/anomalyco/opencode.git` (a fork, not `sst/opencode`) — determine the canonical v2 upstream, compare pinned version vs upstream head for `tool.transform`/`tool.hook`/`shell.hook`/`session.hook` signatures, the `permissions: [{action, resource, effect}]` shape, Code Mode catalog format, and `plugins`-array hook ordering; note `review/oc` is an npm-install dir, not a clone.
|
||||
3. Verbatim-port fidelity: byte-compare `src/gate/verdict.ts` and `src/gate/exit.ts` against the named IUMBTEMS upstream (`review/IUMBTEMS/packages/core/src/evals/gate.ts`, `packages/core/src/evals/exit.ts`) at the pinned commit AND upstream head; audit the deliberate deviations in `docs/NOTES.md` (absent-acks default, strict acks, hard-coded >=5/>=90% ignoring `gate.window`/`gate.threshold`) as contract questions, not conclusions.
|
||||
4. Dependency placement and packaging: `@opencode/plugin` is devDeps-only (types) — prove `probe/` (plain ESM, only `node:fs`/`node:path`) and the packed tarball (`npm pack --dry-run`, `files: [bin, src, probe, schema, examples, README, LICENSE, NOTICE]`) contain no dev leakage; `bunx @heretek-ai/opencode-plugin-ci` path must resolve once published, `engines bun>=1.2`, `setup-bun@v2`/`setup-node@v4(node 22)`/`upload-artifact@v4`/`checkout@v4` pins vs head.
|
||||
5. Config-schema currency: regenerate schemas (`bun scripts/gen-schema.ts`, no diff), probe every `.strict()` object with an unknown key, verify every cross-field rule (`script` required-scripted/forbidden-live, `request`/`judgeReply` scripted-only, `judge` live-only, `live.pass <= live.repeat`) fires with a message naming file and field, and diff `schema/config.json` + `schema/scenario.json` against the `$schema` URLs the README advertises.
|
||||
6. Scope and ecosystem fit: confirm the refusal boundary holds (no TUI rendering, no v1 plugins, no benchmarks, no dogfood `awesome-opencode` workflow per SPEC 9 gap), and that `expect.providers`/`expect.models`, `workspace.tar`, per-scenario `result.json`, `webfetch`/`websearch` flagging, `ctx.storage` snapshots, Action `config` input, and `result: skipped` output remain documented gaps rather than silent behavior.
|
||||
|
||||
### Subagent Gamma: Security, Trust Boundaries & Abuse
|
||||
**Focus question:** can untrusted plugin code, scenario content, or PR metadata cross a trust boundary — exfiltrating keys, escaping the sandbox, or poisoning CI?
|
||||
|
||||
1. Injection corpus vs the Action: feed `tiers`, `opencode-versions`, and `working-directory` inputs containing `$(...)`, backticks, `;`, newlines, and `../` traversals; prove no inputs-interpolation exists inside any action.yml `run:` script (env passthrough `IN_TIERS`/`IN_VERSIONS`/`ACTION_PATH` only) and that `working-directory` cannot escape the runner workspace.
|
||||
2. Command/arg-splitting: abuse the `IFS=',' read -ra` tier/version splitting in action.yml with entries containing spaces, globs, `--out`, `--keep`, `--gate`, and `--opencode=` payloads; prove they become inert `--tier`/`--opencode` values or are rejected, never new flags.
|
||||
3. Secret handling under fire: run scripted scenarios whose workspace `.env`, prompts, tool outputs, shell env, and model-request headers contain `sk-test`-style canaries for every `SECRET_KEY` alternative (`KEY|TOKEN|SECRET|PASSWORD|AUTH|COOKIE`); verify redaction in `probe.jsonl` (`redactEnv`), stub `recordFile`, `events.jsonl`, `report.md`, and `$GITHUB_STEP_SUMMARY` — and prove AUTH/COOKIE additions are fail-closed, not masking a miss elsewhere.
|
||||
4. Path containment: `files:` assertions and `workspace:` writes with `..`, absolute paths, and symlinks escaping `projectDir`; `plugin.path`/`plugin.entry` pointing outside the checkout; shim `serverEntry` with spaces/special bytes (must remain a valid `file://` URL re-export); confirm rejection or containment in each case.
|
||||
5. Judge isolation: prove the judge run loads zero plugins, uses the deny-all agent (`permissions: [{action:"*",resource:"*",effect:"deny"}]`), never sees plugin source, sends no tools, and that a non-JSON or `{"pass":"true"}` judge reply counts as failed judgement; attempt prompt-injection of `OPENCODE_PLUGIN_CI_JUDGE` sentinel via workspace files and show it cannot hijack the judge verdict path.
|
||||
6. CI supply chain: unpinned `oven-sh/setup-bun@v2` / `actions/*@v4` floating majors, `bun install --frozen-lockfile` integrity, `GH_TOKEN`/`github.token` scope (`contents: read`, `actions: read`) vs flake-gate history reads, artifact upload (`opencode-plugin-ci-results` == `gate.artifact` default) poisoning across matrix legs, and the `pull_request_target` prohibition — demonstrate what a malicious PR plugin could do with a live key if that warning were ignored.
|
||||
|
||||
---
|
||||
|
||||
## 3. Comprehensive Inventory of Claimed Functions & Architectural Decisions
|
||||
|
||||
* **Claim 1.1 [A]**: Tier 1 `load` boots the real `opencode run --standalone --auto --format json --print-logs --log-level info` with `stdio: ["ignore","pipe","pipe"]` (stdin closed), and asserts `code === 0`, no timeout, and zero `error` events (`src/sandbox/sandbox.ts`, `src/tiers/load.ts`).
|
||||
* **Claim 1.2 [A][S]**: Tier 1 plugin identity is proven through the probe inventory: the plugin under test is the inventory entry whose `source.path` equals `<shimDir>/server.js` with `state.status: "active"`; a missing entry reports "plugin never registered", a `failed` entry surfaces `state.error` (`src/tiers/load.ts`).
|
||||
* **Claim 1.3 [A]**: `pluginLogProblems` fails tier 1 only on `level ERROR|FATAL` lines naming the plugin id/path or `failed to load plugin` with matching `plugin.id`, parsed from logfmt handling `message`/`msg`, quoted escapes, and dotted keys (`src/opencode/logs.ts`, fixtures: `throws` fails naming `fixture boom`).
|
||||
* **Claim 1.4 [A]**: Tier 1 `expect.tools/commands/agents/skills/mcp` entries (from `.opencode-ci/config.yaml`) are each found in the probe inventory; TUI-only plugins instead pass a static check that the TUI default export has a string `id` and function `setup` (`src/tiers/load.ts`, `src/tiers/ui-static.ts`, fixtures `tui-only` pass / `tui-broken` fail).
|
||||
* **Claim 1.5 [A][S]**: A failing tier 1 skips tiers 2 and 3 for that version (`skipped: "tier 1 failed"`), so per-version cascading failures cannot mask the root cause (`src/run.ts`).
|
||||
* **Claim 2.1 [A][S]**: Scripted `tools:` assertions pair `tool.before`/`tool.after` probe records by `callID`, exclude the outer `execute` call when inner calls exist, require at least one call to match `status`+`input`+`output` together, and default `count` to `{ gte: 1 }` (`src/probe/read.ts`, `src/assert/expect.ts`).
|
||||
* **Claim 2.2 [A][S]**: `tools:` output text is derived from `result.content` (string or `{text}` array joined) or stringified `result.output`, and every failure message names the matcher, the expected value, and the `got` value truncated to 300 chars (`src/assert/expect.ts`, `src/assert/matchers.ts`).
|
||||
* **Claim 2.3 [A]**: `hooks:` assertions address `tool.execute.before|after`, `shell.create.before`, `session.context`, `session.prompt` with `phase` defaulting to `post`, so `pre` shows what the plugin received and `post` shows what it produced (`hook-mutate` and `shell-env` fixtures).
|
||||
* **Claim 2.4 [A]**: `request:` picks the nth (1-based) primary stub request; `system` is the system-role text and `tools` covers direct tool names plus catalog paths by full path, last segment, and `_`-joined form (`src/assert/expect.ts`, `system-inject` fixture).
|
||||
* **Claim 2.5 [A][S]**: `noErrors` defaults to `true` (any `error` event fails the scenario), `exitCode` defaults to `0`, and `files:` rejects any `..` escaping the project dir (`src/config/schema.ts`, `src/assert/expect.ts`).
|
||||
* **Claim 3.1 [A][S]**: The stub auto-wraps a scripted `toolCalls` entry into `execute({ code: "return await <catalogPath>(<json args>)" })` whenever the tool is not directly offered but found via `findCatalogPath` in the Code Mode catalog; `raw: true` opts out (`src/stub/stub.ts`, `src/stub/codemode.ts`, `tool-direct` vs `tool-basic` fixtures).
|
||||
* **Claim 3.2 [A]**: Stub request classification is ordered: containing `OPENCODE_PLUGIN_CI_JUDGE` → `judge` (replies `judgeReply`), else non-empty `body.tools` → `primary` (consumes one script step), else `auxiliary` (`Stub title`); an exhausted script replies `STUB_SCRIPT_EXHAUSTED` and fails the scenario (`src/stub/stub.ts`).
|
||||
* **Claim 3.3 [A][S]**: The probe writes metadata LAST (`{ ...clip(snapshot(data), 0, max), phase, seq, t, kind }`) after a deep-copy snapshot, so a payload field can never overwrite `kind`/`phase` and later plugin mutations cannot corrupt recorded input (`probe/core.js`).
|
||||
* **Claim 3.4 [A]**: `session.context.system` is recorded as one string per system part (clipped at `SYSTEM_MAX` 50000, not `MAX` 4000) so an appended plugin part survives clipping of a long base prompt, and matchers run against the parts joined with `"\n"` (`probe/core.js`).
|
||||
* **Claim 4.1 [A][S]**: The shim directory `<root>/plugin-under-test/server.js` contains exactly `export { default } from "<file-url>"` with `pathToFileURL(serverEntry).href`, keeping spaces/Windows paths safe while the real file stays in place for `node_modules` resolution (`src/plugin/shim.ts`).
|
||||
* **Claim 4.2 [A]**: `resolveEntries` resolves the server entry in order — explicit `plugin.entry`, `package.json` `exports["./server"]` then `exports["."]` (string or `import`/`default`/`bun`/`node` conditions), `package.json` `main`, then `server.{ts,js,mjs}`, `index.{ts,js,mjs}` — and the TUI entry from `exports["./tui"]` then `tui.{ts,js,mjs}`; a directory with neither is exit 2, a file path warns `configured plugin path must be a directory` (`src/plugin/entry.ts`, fixture `no-entry`).
|
||||
* **Claim 4.3 [A][S]**: Plugin load order in every sandbox is `[probe/pre, shimmed plugin, probe/post]` and hooks fire in config-`plugins` array order, so `post` records show the plugin's mutations and `pre` records show its inputs (`src/tiers/load.ts`, spike evidence in `docs/NOTES.md`).
|
||||
* **Claim 4.4 [A]**: Binary resolution honors `OPENCODE_BIN` (verifying `<bin> --version` contains `v<version>`), else resolves `latest` via `npm view @opencode/cli version`, installs `@opencode/cli@<v>` under `$XDG_CACHE_HOME/opencode-plugin-ci/opencode/<v>` with async spawn (never `spawnSync`, which would deadlock the in-process stub), and the sandbox sets `OPENCODE_DISABLE_AUTOUPDATE=1`, `OPENCODE_DISABLE_MODELS_FETCH=1`, isolated `HOME`/`XDG_*`, empty `GIT_CONFIG_GLOBAL`, `TERM=dumb` (`src/opencode/binary.ts`, `src/sandbox/sandbox.ts`).
|
||||
* **Claim 4.5 [A]**: `plugin.install`/`plugin.build` run once per `run` in the checkout via async `bash -lc`/`cmd /c` streaming, and any non-zero exit throws `ConfigError` (exit 2) before any tier runs (`src/plugin/install.ts`).
|
||||
* **Claim 5.1 [A]**: Live scenarios pass on at least `pass` of `repeat` attempts (defaults 2 of 3, `pass <= repeat` validated), each attempt costing `usage(events).cost` (summed `step_finish` `cost` + input/output tokens) against the shared run-wide `live.budgetUSD` (default 1.00) (`src/tiers/live.ts`, `src/opencode/events.ts`).
|
||||
* **Claim 5.2 [A][S]**: The pre-attempt budget check skips (never spawns `opencode`) once `spent >= limit`, and without any configured key the scenario skips with `no API key (set one of: …)` and exit 0, so fork PRs stay green (`src/tiers/live.ts`).
|
||||
* **Claim 5.3 [A][S]**: The judge is a second `opencode run --agent opencode-plugin-ci-judge` in a fresh sandbox with no plugins and deny-all `permissions: [{action:"*",resource:"*",effect:"deny"}]` (verified tool-less request), prompted with the `OPENCODE_PLUGIN_CI_JUDGE` sentinel first line + rubric + clipped evidence, and `parseVerdict` takes the first balanced `{…}` requiring boolean `pass` + string `reason` — anything else is failed judgement (`src/tiers/judge.ts`).
|
||||
* **Claim 5.4 [A]**: Judge tokens count toward `live.budgetUSD`, and live-only `judge.rubric` / scripted-only `request`+`judgeReply` cross-field violations fail validation naming the scenario file and field (`src/tiers/judge.ts`, `src/config/schema.ts`).
|
||||
* **Claim 5.5 [B][S]**: The flake gate applies to live-tier rows only: stable means ≥ 90% of ≥ 5 sampled runs (verbatim upstream constants); stable-now-failing fails, sticky-red persists until pass or human ack, a malformed `.opencode-ci/acknowledgments.json` fails closed, and no history exits 2 unless `gate.bootstrap: allow` (`src/gate/verdict.ts`, `src/gate/gate.ts`).
|
||||
* **Claim 5.6 [B][S]**: An absent `.opencode-ci/acknowledgments.json` defaults to `{version:1,acknowledgments:[]}` (SPEC/M12 override of upstream fail-on-absent), and custom `gate.window`/`gate.threshold` are accepted but inert — defaults 5/0.9 keep default behavior identical (`src/gate/gate.ts`, `docs/NOTES.md` Known gaps).
|
||||
* **Claim 6.1 [B][S]**: Results for `opencode.versions[0]` are required while later versions are allowed failures (`allowedVersions` passed to the report renderer — the M10 bug class), exit codes are 0 pass / 1 failure / 2 config-or-no-history, and `results.json` uses the IUMBTEMS `EvalRow` shape plus `tier`/`opencode` (`src/run.ts`, `src/report/results.ts`).
|
||||
* **Claim 6.2 [B]**: Gate history comes from prior `opencode-plugin-ci-results` artifacts (== `gate.artifact` default == Action `artifact-name` default) on the same branch via `gh` (needs `actions: read`), never from cache eviction-prone caches; matrix legs use distinct artifact names mirrored in each leg's `gate.artifact` (`src/gate/history.ts`, `README.md` matrix section).
|
||||
* **Claim 6.3 [B][S]**: The composite Action passes `tiers`/`opencode-versions` only through `env` (`IN_TIERS`/`IN_VERSIONS`/`ACTION_PATH`) with zero actions-expression interpolation of inputs inside any action.yml `run:` script, installs with `bun install --frozen-lockfile` in `github.action_path`, and emits `result` (`pass`/`fail`) + `report-path` (`action.yml`).
|
||||
* **Claim 6.4 [B]**: npm packaging ships exactly `bin, src, probe, schema, examples, README.md, LICENSE, NOTICE` with bin `opencode-plugin-ci`, `engines bun>=1.2`, runtime deps only `yaml ^2.9.1` + `zod ^4.1.0`, `@opencode/plugin` 2.0.26 dev-only for types, and committed `schema/*.json` byte-identical to `bun scripts/gen-schema.ts` output (`package.json`, `src/config/schema.ts` via zod `.strict()`).
|
||||
* **Claim 7.1 [B][S]**: Every config/scenario object schema is zod `.strict()` so unknown keys (e.g. `expect: { tool: […] }`) fail validation with file path + line, and `matches`/`notMatches` support a leading `(?i)` case-insensitivity flag (`src/config/schema.ts`).
|
||||
* **Claim 7.2 [B][G]**: CI (`.github/workflows/ci.yml`: `check` + `e2e` with pinned `@opencode/[email protected]` via `OPENCODE_BIN`, Node 22, `oven-sh/setup-bun@v2`, `actions/checkout@v4`) reproduces the laptop tiers, and the M14 tree state (`bun run check` + 24 e2e green) is re-derivable from a clean install (`package.json` scripts).
|
||||
* **Claim 7.3 [G]**: The probe redacts values of env/header keys matching `/KEY|TOKEN|SECRET|PASSWORD|AUTH|COOKIE/i` (fail-closed superset of the spec's four) with truncation (`MAX` 4000, depth-6, 200-item arrays), `noEvents` evidence (`events.jsonl`, `probe.jsonl`, `stub-requests.jsonl`, `stderr.log`) never embeds secrets, and the README forbids `pull_request_target` with PR code + secrets because the live tier executes untrusted plugin code with the real key (`probe/core.js`, `README.md` Security).
|
||||
|
||||
---
|
||||
|
||||
## 4. Reality-Check Recipes
|
||||
|
||||
Run from the repo root. Derive pins from repo files; never hardcode what `rg` can tell you.
|
||||
|
||||
```bash
|
||||
export AUDIT_TMP="$(mktemp -d -t opencode-v2-plugin-teser-audit-XXXXXX)"
|
||||
export REPO="$(git rev-parse --show-toplevel)"
|
||||
echo "HEAD: $(git rev-parse HEAD) $(git log -1 --format='%ci %s') UTC: $(date -u +%FT%TZ)"
|
||||
bun --version; node --version; gh auth status; ${OPENCODE_BIN:-opencode} --version || true
|
||||
|
||||
# Pins straight from repo files:
|
||||
rg -o '"@opencode/plugin": "[^"]+"' package.json
|
||||
rg -o '"(yaml|zod)": "[^"]+"' package.json
|
||||
rg -o 'node-version: [0-9]+' action.yml .github/workflows/ci.yml
|
||||
rg -o '(oven-sh/setup-bun|actions/(setup-node|upload-artifact|checkout))@v[0-9]+' action.yml .github/workflows/ci.yml
|
||||
rg -o '"@opencode/cli@[0-9.]+"' .github/workflows/ci.yml src -g '!review'
|
||||
rg -o 'opencode-plugin-ci-results' src/config/schema.ts action.yml | head -5
|
||||
rg -o 'JUDGE_SENTINEL = "[^"]+"|EXHAUSTED = "[^"]+"' src/stub/stub.ts
|
||||
|
||||
# Known-bad probes (the method must work on garbage):
|
||||
npm view @opencode/cli version 0.0.0-nonexistent 2>&1 | tail -2 # must 404
|
||||
OPENCODE_BIN="${OPENCODE_BIN:-}" bun -e '
|
||||
import { resolveEntries } from "./src/plugin/entry.ts";
|
||||
console.log(JSON.stringify(resolveEntries(await (async()=>{const {mkdtempSync}=await import("node:fs");const {join}=await import("node:path");const {tmpdir}=await import("node:os");return mkdtempSync(join(tmpdir(),"empty-"));})())));'
|
||||
# ^ must yield no server/tui entry (empty-dir => no entry)
|
||||
|
||||
# Fresh pinned binary + full local tiers (never the repo review/oc cache):
|
||||
mkdir -p "$AUDIT_TMP/oc" && npm install --no-save --no-audit --no-fund --prefix "$AUDIT_TMP/oc" "@opencode/cli@$(rg -o '"@opencode/plugin": "[^"]+"' package.json | rg -o '[0-9.]+')"
|
||||
export OPENCODE_BIN="$AUDIT_TMP/oc/node_modules/.bin/opencode" && $OPENCODE_BIN --version
|
||||
bun run check 2>&1 | tail -3
|
||||
OPENCODE_BIN="$AUDIT_TMP/oc/node_modules/.bin/opencode" bun run test:e2e --timeout 600000 2>&1 | tail -5
|
||||
|
||||
# Tier 1 + scripted against a fixture, pass and deliberate-fail:
|
||||
OPENCODE_BIN="$OPENCODE_BIN" bun bin/opencode-plugin-ci.ts run --scenario example --out "$AUDIT_TMP/out-basic" 2>&1 | tail -5 || true
|
||||
# run inside test/fixtures/plugins/tool-basic for the fail.yaml-by-design exit-1 proof
|
||||
|
||||
# Schema strictness + cross-field rules (each must exit 2 with file+field named):
|
||||
bun bin/opencode-plugin-ci.ts validate 2>&1 | tail -2
|
||||
bun scripts/gen-schema.ts && git diff --exit-code -- schema/ && echo SCHEMAS-IN-SYNC
|
||||
|
||||
# Gate + judge unit surface (offline):
|
||||
rg -n 'samples >= 5|passes / samples >= 0.9' src/gate/verdict.ts
|
||||
rg -n 'acknowledgments: \[\]' src/gate/gate.ts
|
||||
rg -n 'pass.*boolean' src/tiers/judge.ts | head -3
|
||||
|
||||
# Supply-chain surface:
|
||||
npm view @opencode/[email protected] version 2>&1 | tail -1
|
||||
npm pack --dry-run 2>&1 | tail -5
|
||||
curl -s -X POST https://api.osv.dev/v1/query -d '{"package":{"name":"zod","ecosystem":"npm"},"version":"4.1.0"}' | head -c 400; echo
|
||||
|
||||
# Sabotage targets (one-line break each in $AUDIT_TMP/wt, full suite, record red/green, restore):
|
||||
# [S] src/stub/stub.ts:256 (auto-wrap findCatalogPath) | probe/core.js:64 (metadata-last spread) | src/tiers/load.ts:89 (plugin order pre/plugin/post) | src/run.ts:136 (tier-1-fail skips tiers 2-3) | src/run.ts:191 (required-vs-allowed versions) | src/gate/gate.ts:92 (absent-acks default) | src/tiers/judge.ts:127 (strict boolean pass) | action.yml:43 (env-passthrough, no interpolation) | src/plugin/shim.ts:22 (file-URL shim bytes) | src/config/schema.ts:23 (zod .strict()) | src/assert/expect.ts:163 (count default gte:1) | src/assert/matchers.ts:11 (truncated got in failures) | src/tiers/live.ts:308 (pre-attempt budget check) | src/assert/expect.ts:381 (noErrors default true) | src/gate/verdict.ts:135 (samples >= 5 -> >= 500; sabotage in disposable copy only — this file is a verbatim port, never edit the real one) | src/probe/read.ts:113 (befores.get(record.callID) -> befores.get(record.tool)) | src/assert/expect.ts:324 (remove .. containment check -> always pass) | src/tiers/load.ts:140 (=== shimServerJs -> !==; inventory identity [1.2], order stays at load.ts:89 [4.3])
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Execution Directives for OpenCode
|
||||
|
||||
1. **Independent exploration.** Spawn Alpha, Beta and Gamma concurrently as three separate subagents. Hand each one this entire file plus its role. They must not read each other's notes until step 4. Each one works through the claims it owns (ownership is marked `[A]`, `[B]`, `[G]` in section 3) and may file findings on any claim.
|
||||
2. **Run things.** Prefer executing to reading. Use the recipes in section 4 and any further commands you need (`git ls-remote`, `curl`, `jq`, `rg`, `bun`, `npm view`, `npm pack`). Save the raw output of every command you cite under `$AUDIT_TMP/logs/`.
|
||||
3. **Empirical Proof Mandate.** For every claim judged `BROKEN`, `PARTIAL`, insecure, or outdated, record:
|
||||
- the exact file path and line number (`path/to/file.ts:123`);
|
||||
- the exact command executed to prove it, runnable as written from the repository root;
|
||||
- the raw output, error message, or registry response (trimmed to the relevant lines, never paraphrased);
|
||||
- for upstream comparisons, the upstream file path and commit SHA you compared against.
|
||||
4. **Cross-examination.** When all three are done, each finding is handed to a different subagent (Alpha's to Beta, Beta's to Gamma, Gamma's to Alpha), whose job is to refute it: re-run the proof, look for a misreading, a wrong upstream version, or an environment artifact. A refuted finding is dropped or downgraded, and the refutation is noted in the finding.
|
||||
5. **Coverage gate.** Before synthesis, check that every claim ID in section 3 has a status. Any claim without one goes back to its owner.
|
||||
6. **Synthesis.** The controller writes `AUDIT_REPORT_FINDINGS.md` in the structure below, then cleans up (`git worktree remove`, stop any containers or processes you started) and confirms the real checkout has no changes apart from the report and this prompt file.
|
||||
|
||||
---
|
||||
|
||||
## 6. Report Structure: `AUDIT_REPORT_FINDINGS.md`
|
||||
|
||||
```markdown
|
||||
# AUDIT_REPORT_FINDINGS: opencode-v2-plugin-teser
|
||||
|
||||
## Baseline
|
||||
Commit, commit date, audit date (UTC), tool versions, host capabilities, and the list of checks that could not run here and why.
|
||||
|
||||
## Verdict
|
||||
Three to six sentences: does `opencode-v2-plugin-teser` do what it is supposed to be, against the real systems it plugs into? Name the single most important thing to fix.
|
||||
|
||||
## Prioritized Defect Matrix
|
||||
| ID | Priority | Claim | Location | Summary |
|
||||
|
||||
## Findings
|
||||
### OCI-<claim-id>-<slug>: <title>
|
||||
- **Priority:** P0 | P1 | P2 | P3
|
||||
- **Claim:** <claim ID and text>
|
||||
- **Location:** `path:line`
|
||||
- **Proof:** the command(s), runnable from the repo root
|
||||
- **Raw output:** fenced block, trimmed, verbatim
|
||||
- **Why it matters:** one or two sentences, in terms of what breaks for a user
|
||||
- **Remediation:** an exact, fail-closed diff or command sequence
|
||||
- **Verify the fix:** the command that goes from failing to passing once fixed
|
||||
- **Cross-examination:** who challenged it and what they found
|
||||
|
||||
## Claim Ledger
|
||||
| Claim | Status | Evidence (command or finding ID) |
|
||||
Every claim ID from section 3, no omissions.
|
||||
|
||||
## Sabotage Results
|
||||
| Invariant | One-line sabotage (file:line, before -> after) | Suite result | Caught? |
|
||||
|
||||
## Regression Ledger
|
||||
| Previous finding ID | Previous priority | Status now: FIXED / STILL OPEN / REGRESSED / NO LONGER APPLICABLE | Evidence |
|
||||
(Write "First run" if there was no previous report.)
|
||||
|
||||
## Not Audited / Unverifiable
|
||||
What could not be checked, why, and what a human needs to provide to check it.
|
||||
```
|
||||
|
||||
**Finding IDs are stable across runs.** Use `OCI-<claim-id>-<short-slug>` (for example `OCI-3.2-hooks-order`) so the next run's Regression Ledger can match findings by ID.
|
||||
|
||||
**Priority definitions:**
|
||||
- **P0 / Critical:** the component does not work against its real host or upstream (tier 2 passes against the stub while the plugin never loads under the real binary; a published npm tarball missing `probe/` or `schema/`; the Action executing untrusted PR plugin code with the user's real API key); a security boundary can be bypassed; a pinned artifact does not exist; a release on a registry is broken.
|
||||
- **P1 / High:** deprecated or end-of-life versions, fail-open error handling, an untested security or correctness invariant (sabotage survived), an integration contract that drifted from the current upstream head, severe scope or architecture drift from what the repo is supposed to be.
|
||||
- **P2 / Medium:** non-fatal version lag, documentation that contradicts behavior, missing CI coverage for a reality check, suboptimal resource or performance choices.
|
||||
- **P3 / Low:** style, dead code, naming, minor cleanups.
|
||||
|
||||
For every finding, provide the exact, fail-closed code diff or command sequence to remediate. Do not apply it to the real checkout.
|
||||
@@ -0,0 +1,598 @@
|
||||
# AUDIT_REPORT_FINDINGS: opencode-v2-plugin-teser
|
||||
|
||||
> Status: describes v1 at `af6675c`; the OCI-6.2 remediation was later reverted — see R1 in `review-main-8aa4d61`; do not follow stale advice.
|
||||
|
||||
## Baseline
|
||||
|
||||
- Commit: `af6675cc17e994faf4b9c8df3f386a65097fb489`
|
||||
- Commit date: `2026-10-17 20:34:47 -0400` (`Merge pull request #4 from Heretek-AI/impl/v1`)
|
||||
- Audit date (UTC): `2026-10-18T02:00:34Z`
|
||||
- Tools: bun 1.4.2, node v26.10.0, npm 11.19.1, git 2.56.0, gh 2.97.0 (authed BillyOutlast), jq-1.8.2, rg 15.1.0, curl 8.22.0, opencode v2.0.32 system; fresh 2.0.26 + head 2.0.32 in $AUDIT_TMP; `npm view @opencode/cli version → 2.0.32`
|
||||
- Host: Linux; loopback-only stub; isolated HOME/XDG per sandbox; `gh auth status` authed
|
||||
- `git status --porcelain` at start: only `?? PROMPT_ADVERSARIAL_FLEET_AUDIT.md`
|
||||
- AUDIT_TMP=/tmp/opencode-v2-plugin-teser-audit-psbwq9; wt=/tmp/opencode-v2-plugin-teser-audit-psbwq9/wt (detached af6675c)
|
||||
- Previous report: none (First run)
|
||||
- Fresh binaries used: `$AUDIT_TMP/oc-2.0.26` (`opencode v2.0.26`), `$AUDIT_TMP/oc-head` (`opencode v2.0.32`); repo `review/oc` (2.0.26) used only for comparison, never as proof
|
||||
- Critical host factor: stray `/tmp/opencode.json` listing `plugins: ["@heretek-ai/opencode-rea","@aerovato/operator-opencode-v2@latest"]`; opencode v2 walks up from cwd and merges ancestor `opencode.json(c)` files. Polluted runs under default TMPDIR failed spuriously; clean runs used `TMPDIR=/dev/shm/ocz` (ancestors `/dev/shm`,`/dev`,`/` contain no `opencode.json`) without touching any user file. See OCI-4.4-ancestor.
|
||||
- Checks that could not run here and why: live tier against a real model with a real key (refusal boundary — only skip path + `OPENCODE_PLUGIN_CI_JUDGE_VIA_STUB=1` stub paths exercised, with throwaway dummy key never sent to a real provider); Windows drive-letter shim paths and `cmd /c` install branch (Linux host only); `pull_request_target` with live secrets (forbidden by hard limits — PoC used throwaway `sk-test-*` dummies, stub model, loopback only); real GitHub Actions runner execution of `working-directory: ../` and same-name matrix artifact collision (no GitHub writes allowed — inferred from runner semantics + code + docs); head-install runtime behavior beyond Alpha's 10-fixture matrix (Beta/Gamma deferred to Alpha); full `bun run test:e2e` green baseline in one shot in Gamma scope (deferred to Alpha; equivalent fixture-by-fixture CLI coverage achieved).
|
||||
|
||||
## Verdict
|
||||
|
||||
`opencode-v2-plugin-teser` does what it is supposed to be: load, scripted (stub, zero tokens), and live-skip tiers reproduce identically at pinned 2.0.26 and head 2.0.32 across all 10 fixtures, the probe inventory/hook-order/request/catalog surfaces behave as claimed, judge isolation and budget skip paths are real, and packaging/verbatim-port/schema-sync claims hold except where noted. The gaps are coverage and containment, not wholesale breakage: 4 untested invariants survived sabotage, ancestor configs leak into the "isolated" sandbox, workspace writes escape, and secrets persist outside env/headers. The single most important fix is ancestor neutralization + allow-list: after each run assert the merged plugin set equals the configured set (`[pre, shimmed plugin, post]` plus shim) and fail tier 1 closed on extras, and contain sandbox roots against walk-up `opencode.json(c)` discovery.
|
||||
|
||||
## Prioritized Defect Matrix
|
||||
|
||||
| ID | Priority | Claim | Location | Summary |
|
||||
|---|---|---|---|---|
|
||||
| OCI-4.4-ancestor | P1 | 4.4 sandbox isolation | `src/sandbox/sandbox.ts:92-95` | Ancestor `opencode.json` merged into sandbox; foreign npm plugins installed/executed; determinism + key-exfil path |
|
||||
| OCI-1.5-skip | P1 | 1.5 tier-1-fail skips tiers 2-3 | `src/run.ts:248-261` | Cascade `skipped: "tier 1 failed"` has zero test coverage; sabotage green |
|
||||
| OCI-6.4-unpublished | P1 | 6.4 npm packaging / bunx path | `package.json:1-2` | Package never published; README `bunx` quickstart and `$schema` URLs 404 |
|
||||
| OCI-6.3-action-guard | P1 | 6.3 zero-interpolation invariant | `action.yml:40-51` | Current file clean but no automated guard; interpolation regression would be silent |
|
||||
| OCI-6.4-files-guard | P1 | 6.4 published `files` list | `package.json:17-26` | Dropping `probe` from `files` ships a tarball that fails every tier; no test pins `files`/`bin` |
|
||||
| OCI-7.3-redact-untested | P1 | 7.3 redact regex | `probe/core.js:3` | SECRET_KEY superset has zero tests; narrowing/breaking it stays green |
|
||||
| OCI-7.3-evidence-leaks | P1 | 7.3 noEvents never embeds secrets | `probe/core.js:69-145` | Only env/headers redacted; prompt/tool I/O/system/stub/events/report embed secrets verbatim |
|
||||
| OCI-4.4-workspace-escape | P1 | sandbox workspace containment | `src/sandbox/sandbox.ts:108-112` | `join(project,rel)` with no check; `../` escapes projectDir, `../../` escapes sandbox root to /tmp |
|
||||
| OCI-3.3-meta | P2 | 3.3 probe metadata-last | `probe/core.js:61-65` | Metadata-last ordering untested; payload-spread-last sabotage green (merged Gamma duplicate) |
|
||||
| OCI-4.3-order | P2 | 4.3 plugin order `[pre,plugin,post]` | `src/tiers/load.ts:88-92` | `load.ts` copy unpinned; swap sabotage green; scripted copy IS pinned |
|
||||
| OCI-2.5-noerrors | P2 | 2.5 `noErrors` defaults true | `src/assert/expect.ts:381` | `?? true` fallback unpinned; prod default is schema-pinned, fallback is dead-layer only |
|
||||
| OCI-1.4-cap | P2 | 1.4 inventory expectations | `probe/core.js:15-17` | 200-item array cap truncates tier-1 inventory; false failures over 200 tools |
|
||||
| OCI-1.4-providers | P2 | 1.4 providers/models | `src/tiers/load.ts:20-26,160-166` | `expect.providers/models` accepted but silently ignored; README contradicts behavior |
|
||||
| OCI-6.1-allowed-wiring | P2 | 6.1 required/allowed wiring | `src/run.ts:191-192` | `allowedVersions` wiring unpinned; only report labels affected (fail-closed), not exit codes |
|
||||
| OCI-5.6-inert-window | P2 | 5.6 inert window/threshold | `src/gate/gate.ts:294-311` | `gate.window/threshold` accepted but never read; README over-promises tuning |
|
||||
| OCI-2.5-files-symlink | P2 | 2.5 files `..` containment | `src/assert/expect.ts:318-370` | Lexical `..`/absolute rejected but symlinks followed out of projectDir |
|
||||
| OCI-6.4-action-pins | P2 | 6.4/7.2 CI pins | `action.yml:30-31,58` | `checkout/setup-node/upload-artifact@v4` float on superseded majors while v7 exists |
|
||||
| OCI-7.1-no-line | P3 | 7.1 strict schemas | `src/config/load.ts:31-39` | Errors name file+field, not line; claim says "file path + line" |
|
||||
| OCI-registry-drift | P3 | registry truth / drift | `src/opencode/binary.ts:79` | Pinned 2.0.26 is 6 releases behind head 2.0.32; drift purely additive; `latest` floats per run |
|
||||
| OCI-6.3-working-dir | P3 | 6.3 working-directory | `action.yml:39,61` | `working-directory`/artifact path unconstrained; `../` escapes workspace (author-error only) |
|
||||
| OCI-7.2-floating-majors | P3 | 7.2/6.3 floating majors hygiene | `action.yml:30-31,58` | Same floating `@v2/@v4` as OCI-6.4-action-pins; GitHub-endorsed pattern, SHA-pinning is hygiene |
|
||||
| OCI-6.2-artifact-poisoning | P3 | 6.2 gate history artifact | `action.yml:17-18` | Default artifact name shared across matrix legs collides unless author overrides per README |
|
||||
|
||||
## Findings
|
||||
|
||||
### OCI-4.4-ancestor: ancestor `opencode.json` is merged into the "isolated" sandbox
|
||||
- **Priority:** P1
|
||||
- **Claim:** 4.4 (binary resolution + sandbox env isolation) + SPEC 3.1 ("Nothing touches the user's real opencode setup" — reverse direction equally unguarded)
|
||||
- **Location:** `src/sandbox/sandbox.ts:92-95` (`root = join(tmpdir(), ...)` — no ancestor neutralization)
|
||||
- **Proof:** from repo root with default TMPDIR (pinned 2.0.26 fresh binary):
|
||||
```bash
|
||||
cd test/fixtures/plugins/tool-basic
|
||||
OPENCODE_BIN=$AUDIT_TMP/oc-2.0.26/bin/opencode bun $REPO/bin/opencode-plugin-ci.ts run --tier load --scenario tool-basic-pass --opencode 2.0.26 --out $AUDIT_TMP/out-basic-pass-2026
|
||||
# control:
|
||||
TMPDIR=/dev/shm/ocz OPENCODE_BIN=$AUDIT_TMP/oc-2.0.26/bin/opencode bun $REPO/bin/opencode-plugin-ci.ts run --tier load --scenario tool-basic-pass --opencode 2.0.26 --out $AUDIT_TMP/out-clean-pass
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
# polluted (default TMPDIR, /tmp/opencode.json present):
|
||||
tier 1 FAILED with `expected tool "echo_tool" not in inventory tools [rea_address_name, ...200 rea_* tools...]`
|
||||
probe.jsonl inventory shows active foreign plugins `rea@... (@heretek-ai/opencode-rea)` and `aerovato.operator-memory (@aerovato/operator-opencode-v2@latest)`
|
||||
stderr.log shows `msg="loading plugin" id=@heretek-ai/opencode-rea entrypoint=file:///tmp/opencode-plugin-ci-*/cache/opencode/npm/...`
|
||||
# clean (TMPDIR=/dev/shm/ocz): same command PASSED
|
||||
# polluted N tools: 200, HAS echo_tool: False; clean N tools: 63, echo_tool present
|
||||
```
|
||||
- **Why it matters:** Identical checkout gives different results depending on stray ancestor configs, breaking "identically on laptop and CI" (observed cost ~1.8s → ~17s plus false tier-1 failures, and the repo's own e2e inherits the fragility via `tmpdir()`). Ancestor-listed npm plugins are installed (`@latest`) and executed with the sandbox env — in the live tier that env carries real API keys, so any local writer of an ancestor dir can inject key-exfiltrating code.
|
||||
- **Remediation:** fail-closed allow-list in `runLoad`/`runScenario`: after each run compare `ctx.plugin.list()` ids in the probe inventory against expected (probes + shim) and fail tier 1 on extras; snapshot `/tmp`-style ancestors and refuse/warn when any `opencode.json(c)` exists between sandbox root and filesystem root; ask upstream for `OPENCODE_NO_PROJECT_CONFIG=1` support. Implement (1) today in `runLoad`.
|
||||
- **Verify the fix:** re-run the Proof command with a planted `$TMPDIR-ancestor/opencode.json` listing a bogus plugin → tier 1 must fail closed (or the file must be provably ignored).
|
||||
- **Cross-examination:** Beta challenged (reproduced polluted-vs-clean pair on same host, verified `/tmp/opencode.json` is a genuine user file not an audit plant, verified upstream walk-up discovery in `review/opencode/.../config/discovery.ts` + `fs.up` walk, verified `opencode run --help` on 2.0.26 and head shows no neutralizing flag). Verdict: STANDS P1 (P0 considered for live-keys + `@latest` auto-install but rejected — exploit needs local ancestor-dir write, outside threat model).
|
||||
|
||||
### OCI-1.5-skip: tier-1-failure cascade skip has zero test coverage
|
||||
- **Priority:** P1
|
||||
- **Claim:** 1.5 — a failing tier 1 skips tiers 2 and 3 for that version (`skipped: "tier 1 failed"`)
|
||||
- **Location:** `src/run.ts:248-261` (`if (!tier1Pass) { ... results.push(skippedForTier1(...)); continue; }`)
|
||||
- **Proof:** in `$AUDIT_TMP/wt`, one-line never-skip sabotage, then from wt root:
|
||||
```bash
|
||||
# sabotage: `if (!tier1Pass) {` → `if (!tier1Pass && tier1Pass) {` (biome+tsc clean)
|
||||
bun run check
|
||||
OPENCODE_BIN=$AUDIT_TMP/oc-2.0.26/bin/opencode TMPDIR=/dev/shm/ocz bun test ./test/e2e/cli.test.ts --timeout 600000
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
Ran 213 tests ... 0 fail (exit 0)
|
||||
cli.test.ts: 4 pass, 0 fail (the `run in throws exits 1` case still exits 1 via required load failure — never inspects results.json for skipped entries)
|
||||
```
|
||||
- **Why it matters:** If the cascade regresses, a tier-1 root cause is buried under N tier-2/3 failures; worse, live scenarios would burn real `live.budgetUSD` on doomed attempts after a tier-1 failure.
|
||||
- **Remediation:** extract the cascade into a pure `skippedAfterTier1(...)` helper asserting entries + `continue` semantics, plus extend `test/e2e/cli.test.ts` "run in throws" to assert `results.json` contains `throws-smoke` with `skipped: "tier 1 failed"`.
|
||||
- **Verify the fix:** re-run the sabotage → `bun run check` must go red.
|
||||
- **Cross-examination:** Beta challenged (re-ran S4c in wt — biome-clean, 213 pass; read `test/e2e/cli.test.ts:52-55` — asserts only exit code; `grep -rn "tier 1 failed" src test` → only definition in `src/run.ts:130-139`). Verdict: STANDS P1.
|
||||
|
||||
### OCI-6.4-unpublished: package never published — README `bunx` path and `$schema` URLs both 404
|
||||
- **Priority:** P1
|
||||
- **Claim:** 6.4 (npm packaging / `bunx @heretek-ai/opencode-plugin-ci` distribution path); also README `$schema` URLs and 7.2 quickstart reproducibility
|
||||
- **Location:** `package.json:1-2` (name/version 0.1.0, unpublished), `README.md:36,48,66,161`
|
||||
- **Proof:** from repo root:
|
||||
```bash
|
||||
npm view @heretek-ai/opencode-plugin-ci versions --json
|
||||
curl -sL -o /dev/null -w "HTTP:%{http_code}\n" https://unpkg.com/@heretek-ai/opencode-plugin-ci/schema/config.json
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
npm error code E404
|
||||
npm error 404 Not Found - GET https://registry.npmjs.org/@heretek-ai%2fopencode-plugin-ci - Not found
|
||||
HTTP:404 ... Package not found: @heretek-ai/opencode-plugin-ci
|
||||
```
|
||||
- **Why it matters:** The documented primary install path (`bunx @heretek-ai/opencode-plugin-ci init --force`, README:36,:48) fails verbatim for every user today; the `$schema` comment README tells users to paste (README:66) resolves to a 404 so editor schema assistance is dead.
|
||||
- **Remediation:** publish 0.1.0 (or prerelease tag) and re-run verify; until then README quickstart must show the source-substitution form (`bun <checkout>/bin/opencode-plugin-ci.ts`) instead of `bunx`.
|
||||
- **Verify the fix:**
|
||||
```bash
|
||||
npm view @heretek-ai/opencode-plugin-ci version | grep -q 0.1.0 && diff <(curl -sL https://unpkg.com/@heretek-ai/opencode-plugin-ci/schema/config.json) schema/config.json && echo PUBLISHED-IN-SYNC
|
||||
```
|
||||
- **Cross-examination:** Gamma challenged (re-ran E404 + unpkg 404; stronger ground: `docs/NOTES.md:270` admits "requires the package to be published, which it is not yet", and `grep -rn "npm publish" .github/ package.json` is empty so no publish automation exists; Beta's E403/E401 disambiguator is wrong — npm returns 404 for inaccessible private scopes too — but immaterial). Verdict: STANDS P1.
|
||||
|
||||
### OCI-6.3-action-guard: zero-interpolation invariant in action.yml has no automated guard
|
||||
- **Priority:** P1
|
||||
- **Claim:** 6.3 ("zero actions-expression interpolation of inputs inside any action.yml `run:` script")
|
||||
- **Location:** `action.yml:40-51` (env block `IN_TIERS`/`IN_VERSIONS`/`ACTION_PATH` + `run:` script using only `$IN_*`/`$ACTION_PATH`/`$GITHUB_OUTPUT`)
|
||||
- **Proof:** in `$AUDIT_TMP/wt`, sabotage `IN_TIERS: ${{ inputs.tiers }}` passthrough dropped + `# ${{ inputs.tiers }}` injected into `run:` script, then from wt root:
|
||||
```bash
|
||||
bun run check
|
||||
rg -ln "action.yml" test/
|
||||
grep -c actionlint .github/workflows/ci.yml || true
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
213 pass / 0 fail (exit 0)
|
||||
rg: no test file references action.yml; ci.yml has no actionlint step
|
||||
```
|
||||
- **Why it matters:** The no-interpolation property is the Action's command-injection boundary; a future edit interpolating `${{ inputs.tiers }}` into the script would be a silent security regression with no red test. The file itself is currently clean — the guard is what's missing.
|
||||
- **Remediation:** add a unit test reading `action.yml`, extracting every `run:` block, asserting it contains no `${{ inputs.` / `${{ github.` expression (allow-listing `${{ github.action_path }}` only in `working-directory:`/`env:` properties, never in `run:`); or add an actionlint + expression-grep step to `ci.yml`.
|
||||
- **Verify the fix:** re-apply the sabotage edit → new test/step must fail; `bun run check` green on fixed tree.
|
||||
- **Cross-examination:** Gamma challenged (re-read action.yml:27-62 — `${{ }}` appears only in `working-directory:`/`env:`/`with:`/`if:` standard safe positions; `run:` body uses only shell vars; sabotage was two edits not one-line and comment-position injection is bash-inert without newline breakout — proves guard-absence not exploitability). Verdict: STANDS P1 (security-boundary invariant with zero guard).
|
||||
|
||||
### OCI-6.4-files-guard: published `files` list (notably `probe/`) has no automated guard
|
||||
- **Priority:** P1
|
||||
- **Claim:** 6.4 ("npm packaging ships exactly `bin, src, probe, schema, examples, README.md, LICENSE, NOTICE`")
|
||||
- **Location:** `package.json:17-26` (`files` array)
|
||||
- **Proof:** in `$AUDIT_TMP/wt`, remove `"probe"` from `files`, then from wt root:
|
||||
```bash
|
||||
bun run check
|
||||
npm pack --dry-run 2>&1 | grep -c "probe/core.js" || true
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
213 pass / 0 fail
|
||||
sabotaged tree ships 0 probe/core.js (vs intact tree ships probe/core.js + probe/pre/server.js)
|
||||
```
|
||||
- **Why it matters:** `probe/` is load-bearing at runtime (sandbox `plugins` array points at it); dropping it from `files` produces a published tarball that fails every tier with a confusing plugin-load error, caught by no test. Same for `bin`/`schema`.
|
||||
- **Remediation:** extend `test/unit/schema-sync.test.ts` (or new `packaging.test.ts`) to assert `package.json` `files` deep-equals the documented set and `bin` maps `opencode-plugin-ci`, and/or run `npm pack --dry-run` in CI asserting the tarball contains `probe/core.js`, `probe/pre/server.js`, `probe/post/server.js`, `schema/config.json`, `bin/opencode-plugin-ci.ts`.
|
||||
- **Verify the fix:** re-drop `"probe"` from `files` → new test must fail; `npm pack --dry-run | grep probe/core.js` must succeed on fixed tree.
|
||||
- **Cross-examination:** Gamma challenged (re-ran drop-`probe` → 0 `probe/core.js` shipped; verified intact tree ships 45 files incl. probe; verified `probePluginDirs()` in `src/probe/read.ts:164-168` feeds sandbox plugins in `src/tiers/load.ts:83-91`; verified no unit asserts `files`/`bin`). Verdict: STANDS P1.
|
||||
|
||||
### OCI-7.3-redact-untested: probe SECRET_KEY regex has zero test coverage
|
||||
- **Priority:** P1 (downgraded from P0 by cross-examination; fact stands, severity corrected)
|
||||
- **Claim:** 7.3 (probe redacts `/KEY|TOKEN|SECRET|PASSWORD|AUTH|COOKIE/i`)
|
||||
- **Location:** `probe/core.js:3`
|
||||
- **Proof:** in `$AUDIT_TMP/wt`:
|
||||
```bash
|
||||
cd /tmp/opencode-v2-plugin-teser-audit-psbwq9/wt
|
||||
# replace SECRET_KEY with /KEY|TOKEN|SECRET|PASSWORD/i, then with /KEY/i
|
||||
bun run check 2>&1 | tail -5
|
||||
bun test ./test/unit 2>&1 | tail -8
|
||||
git checkout -- probe/core.js
|
||||
rg -n 'probe/core|redactEnv|SECRET_KEY' test/unit test/e2e
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
# narrow to spec-four:
|
||||
213 pass, 0 fail, 1 snapshots, 469 expect() calls — suite GREEN (should be red)
|
||||
# break to /KEY/ only:
|
||||
213 pass, 0 fail — suite GREEN (should be red)
|
||||
# coverage check:
|
||||
(no output — zero references to probe/core.js, redactEnv, or SECRET_KEY in test/)
|
||||
```
|
||||
- **Why it matters:** The redact regex is the single security boundary between live keys (shell env + model headers) and persisted/uploaded evidence. An accidental narrowing or typo ships silently; AUTH/COOKIE bearer tokens then persist in `probe.jsonl` artifacts.
|
||||
- **Remediation:** add a unit test importing `probe/core.js:makeProbe`, feeding env/headers with every alternative (`KEY, TOKEN, SECRET, PASSWORD, AUTH, COOKIE`, mixed case) plus canary values, asserting `[redacted]` in output and absence of canaries. Fail-closed: assert each alternative individually, not just one.
|
||||
- **Verify the fix:** re-apply the narrow sabotage; the new test must fail (suite red); restore and suite green.
|
||||
- **Cross-examination:** Alpha challenged (re-ran narrowing to `/KEY/i` → 213 pass; confirmed zero `test/` references; confirmed Gamma's own logs prove the boundary FUNCTIONS including AUTH/COOKIE; priority definitions reserve P0 for a bypassable boundary and assign sabotage-survived to P1; SPEC 3.3 promises only four alternatives so narrowing to spec-four would not even violate SPEC). Verdict: fact STANDS, DOWNGRADED P0→P1.
|
||||
|
||||
### OCI-7.3-evidence-leaks: only env/headers are redacted — prompt/tool I/O/system/stub/events/report embed secrets verbatim
|
||||
- **Priority:** P1
|
||||
- **Claim:** 7.3 ("noEvents evidence (`events.jsonl`, `probe.jsonl`, `stub-requests.jsonl`, `stderr.log`) never embeds secrets")
|
||||
- **Location:** `probe/core.js:69-145` (only `shell.before.env` via `redactEnv` and `session.model.request.headers` via `redactEnv`; no redaction on `tool.before.input`, `tool.after.result`, `session.prompt.prompt`, `session.context.system/tools`), `src/stub/stub.ts:279-282` (recordFile writes full `body`), `src/opencode/events.ts` (no redaction), `src/assert/matchers.ts:show()` (failure `got` includes secret, truncated only), `src/report/markdown.ts` + `appendStepSummary`
|
||||
- **Proof:** from repo root:
|
||||
```bash
|
||||
bun -e 'import { makeProbe } from "./probe/core.js"; /* feed env canaries + tool input/output + prompt; read probe.jsonl */'
|
||||
bun -e 'import { startStub } from "./src/stub/stub.ts"; /* POST body with "system sk-test-SYSTEM-SECRET"; cat recordFile */'
|
||||
rg -n 'redact|SECRET|KEY|TOKEN' src/report/*.ts src/opencode/*.ts src/tiers/*.ts src/assert/*.ts
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
{"env":{"API_KEY":"[redacted]","MY_TOKEN":"[redacted]","MY_SECRET":"[redacted]","MY_PASSWORD":"[redacted]","MY_AUTH":"[redacted]","MY_COOKIE":"[redacted]",...},"phase":"post",...,"kind":"shell.before"}
|
||||
{"headers":{"Authorization":"[redacted]",...},"kind":"session.model.request"}
|
||||
{"tool":"shell","callID":"1","input":{"command":"cat .env sk-test-API_KEY-123"},...,"kind":"tool.before"}
|
||||
{"tool":"shell","callID":"1",...,"result":{"content":"sk-test-output-secret"},...,"kind":"tool.after"}
|
||||
{"prompt":"print sk-test-prompt-secret",...,"kind":"session.prompt"}
|
||||
sk-test-output-secret LEAKED (probe does NOT redact tool I/O)
|
||||
sk-test-prompt-secret LEAKED (probe does NOT redact prompt)
|
||||
{"kind":"primary",...,"body":{"messages":[{"content":"system sk-test-SYSTEM-SECRET"},...]}}
|
||||
sk-test-SYSTEM-SECRET LEAKED in stub-requests (no redaction)
|
||||
```
|
||||
- **Why it matters:** Any scenario whose prompt/workspace/tool output contains a real secret (easy mistake: `cat .env`) persists it into `probe.jsonl`, `stub-requests.jsonl`, `events.jsonl`, and via assertion failures into `report.md` + `$GITHUB_STEP_SUMMARY`, all uploaded as `opencode-plugin-ci-results` artifacts.
|
||||
- **Remediation:** narrow claim 7.3 to "env/header values redacted; all other evidence surfaces MUST NOT contain secrets (author responsibility)" AND either (a) add value-side scrubbing of live key values in `report.md`/failures/`stub-requests` writes, or (b) fail validation when `expect.*` matchers contain `sk-`-like literals and warn when workspace/prompt contains live key values.
|
||||
- **Verify the fix:** re-run corpus with dummy canaries in prompt/tool output; after fix canaries must be absent from `probe.jsonl`/`stub-requests.jsonl`/events tail/`report.md`, while env/header tests still pass.
|
||||
- **Cross-examination:** Alpha challenged (scope overreach — README:328-330 already assigns non-env surfaces to author responsibility; challenge FAILS because audited claim text as written says "never embeds secrets" which is false, and automatic artifact upload converts an author slip into secret distribution). Verdict: STANDS P1.
|
||||
|
||||
### OCI-4.4-workspace-escape: scenario workspace writes escape projectDir AND sandbox root
|
||||
- **Priority:** P1
|
||||
- **Claim:** sandbox isolation (SPEC 3.1, workspace is "files written into the project dir") + 7.3 "treat workspace files as untrusted input"; prompt task 4 (confirm rejection or containment)
|
||||
- **Location:** `src/sandbox/sandbox.ts:108-112`
|
||||
```ts
|
||||
for (const [rel, content] of Object.entries(opts.workspace ?? {})) {
|
||||
const dest = join(this.project, rel);
|
||||
mkdirSync(dirname(dest), { recursive: true });
|
||||
writeFileSync(dest, content);
|
||||
}
|
||||
```
|
||||
- **Proof:** from repo root:
|
||||
```bash
|
||||
bun -e 'import { Sandbox } from "./src/sandbox/sandbox.ts"; const s = new Sandbox({ opencodeBin: "/nonexistent/opencode", workspace: { "../escape.txt": "pwned", "sub/../../escape2.txt": "pwn2" } });'
|
||||
bun -e 'import { Sandbox } from "./src/sandbox/sandbox.ts"; const s = new Sandbox({ opencodeBin: "/nonexistent/opencode", workspace: { ["../../gamma-escape-"+Date.now()+".txt"]: "PWNED-OUTSIDE-SANDBOX" } });'
|
||||
rg -n 'resolve|startsWith|escapes|sep' src/sandbox/sandbox.ts
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
project /tmp/opencode-plugin-ci-.../project
|
||||
project listing: [ ".git", "ok.txt", "tmp", "README.md" ]
|
||||
parent listing has escape.txt? YES ESCAPED
|
||||
root parent: [ "gitconfig", "escape2.txt", "escape.txt", ... ]
|
||||
dest /tmp/gamma-escape-....txt
|
||||
/tmp marker exists? YES ESCAPED SANDBOX ROOT: PWNED-OUTSIDE-SANDBOX
|
||||
rg: no containment check
|
||||
```
|
||||
- **Why it matters:** Workspace comes from `.opencode-ci/scenarios/*` in the PR checkout — attacker-controlled in the fork-PR threat model. A malicious scenario can write outside its project dir into the sandbox root and shared `/tmp`, enabling cross-scenario poisoning and runner pollution. `files:` assertions ARE contained but writes are not — asymmetric.
|
||||
- **Remediation:** mirror `evaluateFiles`: `const dest=resolve(project, rel); if (dest!==project && !dest.startsWith(project+sep)) throw ConfigError(...)` before mkdir/write. Reject absolute paths explicitly.
|
||||
- **Verify the fix:** re-run escape corpus; `../` and `../../` entries must throw/exit 2, `ok.txt` still written, suite green.
|
||||
- **Cross-examination:** Alpha challenged (`join` contains absolutes by accident so no absolute escape; attacker already has code exec via install/build/setup so escape adds nothing; challenge FAILS on `..` — `join(project,"../escape.txt")` genuinely escapes, zero containment checks vs `evaluateFiles` which HAS them, task 4 explicitly in scope, fail-open is P1 per definitions). Verdict: STANDS P1.
|
||||
|
||||
### OCI-3.3-meta: probe metadata-last ordering untested (merged Gamma duplicate)
|
||||
- **Priority:** P2 (downgraded from P1 by cross-examination; Gamma OCI-probe-spoof-untested merged here)
|
||||
- **Claim:** 3.3 — the probe writes metadata LAST (`{ ...clip(snapshot(data), 0, max), phase, seq, t, kind }`) after a deep-copy snapshot, so a payload field can never overwrite `kind`/`phase`
|
||||
- **Location:** `probe/core.js:61-65`
|
||||
- **Proof:** in `$AUDIT_TMP/wt`, swap to payload-wins `{ phase, seq, t, kind, ...clip(snapshot(data)) }`, then from wt root:
|
||||
```bash
|
||||
bun run check
|
||||
bun test ./test/e2e/tier2.test.ts --timeout 600000
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
check 0 fail (213 tests); tier2 e2e green
|
||||
No test crafts a payload with colliding kind/phase keys
|
||||
Gamma re-run: `bun test ./test/unit` → 213 pass, 0 fail (GREEN, should be red)
|
||||
```
|
||||
- **Why it matters:** Today no call site passes colliding top-level keys, so blast radius is defense-in-depth only; but the [S] invariant ("a payload field can never overwrite kind/phase") is asserted by nothing — a future probe record with a `kind` field would silently corrupt the evidence stream.
|
||||
- **Remediation:** unit test importing `probe/core.js` `makeProbe` writing a record whose data contains `{kind:"forged",phase:"forged",seq:999}` and asserting the parsed line keeps `kind`/`phase`/`seq` from metadata.
|
||||
- **Verify the fix:** re-run swap sabotage → new test goes red.
|
||||
- **Cross-examination:** Beta challenged (enumerated all 9 `write()` call sites — data keys never include `kind`/`phase`/`seq`/`t`; collision unreachable by construction; ordering is hardening with zero current blast radius). Alpha confirmed (all 8 `write()` sites use fixed top-level keys; attacker-influenced values are always NESTED; identical top-level under either spread order). Verdict: fact STANDS, DOWNGRADED P1→P2; Gamma duplicate merged here.
|
||||
|
||||
### OCI-4.3-order: `load.ts` plugin order unpinned
|
||||
- **Priority:** P2 (downgraded from P1 by cross-examination)
|
||||
- **Claim:** 4.3 — every sandbox is `[pre, shimmed plugin, post]` and hooks fire in config-`plugins` array order
|
||||
- **Location:** `src/tiers/load.ts:88-92`
|
||||
- **Proof:** in `$AUDIT_TMP/wt`, swap to `[post, shim, pre]`, then from wt root:
|
||||
```bash
|
||||
bun run check
|
||||
bun test ./test/e2e/probe.test.ts --timeout 600000
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
check 0 fail; probe.test.ts green (0 fail, 18 expect() calls) — builds its OWN [pre,plugin,post] sandbox, never exercises runLoad's order
|
||||
```
|
||||
- **Why it matters:** A load-order regression silently inverts pre/post attribution for tier-1 evidence; the scripted copy IS pinned (`hook-mutate-pass` e2e), only the load copy is free-floating — the two can drift apart unnoticed. Tier-1 verdicts consume only inventory+logs, so no verdict changes today.
|
||||
- **Remediation:** extend `test/e2e/tier1.test.ts` (or probe.test.ts) with a `runLoad`-level hook-order assertion, e.g. run tier 1 on `hook-mutate` and assert `probe.jsonl` pre/post `tool.before` inputs differ as expected.
|
||||
- **Verify the fix:** re-run swap sabotage → e2e goes red.
|
||||
- **Cross-examination:** Beta challenged (tier-1 verdicts consume only inventory+logs in `src/tiers/load.ts:119-175`; nothing in tier 1 reads hook pre/post records, so swapped load order changes no verdict; verdict-affecting scripted copy IS pinned). Verdict: STANDS, DOWNGRADED P1→P2.
|
||||
|
||||
### OCI-2.5-noerrors: `noErrors` default-true unpinned
|
||||
- **Priority:** P2 (downgraded from P1 by cross-examination)
|
||||
- **Claim:** 2.5 — `noErrors` defaults to `true` (any `error` event fails the scenario)
|
||||
- **Location:** `src/assert/expect.ts:381` (`expect.noErrors ?? true`)
|
||||
- **Proof:** in `$AUDIT_TMP/wt`, `?? true` → `?? false`, then from wt root:
|
||||
```bash
|
||||
bun run check
|
||||
bun test ./test/e2e/tier2.test.ts --timeout 600000
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
check 0 fail (all unit noErrors cases set it explicitly); tier2 e2e green (no fixture emits error events on a passing path)
|
||||
```
|
||||
- **Why it matters:** If the fallback regresses, scenarios that SHOULD fail on opencode `error` events while `tool_use` assertions still match would silently pass. User-facing default is schema-pinned; only the redundant fallback is unpinned.
|
||||
- **Remediation:** unit test: `evaluate({}, evidence-with-error-event)` must contain `error event`; plus one scripted e2e fixture whose script forces an `error` event with `noErrors` unset.
|
||||
- **Verify the fix:** re-run sabotage → check goes red.
|
||||
- **Cross-examination:** Beta challenged (production `scenario.expect` always passes through `ScenarioSchema` defaulting `noErrors: true` in `src/config/schema.ts:197` — proved via `ScenarioSchema.safeParse` → `expect.noErrors = true`; that default IS pinned by `test/unit/config-schema.test.ts:248`; `?? true` is unreachable dead fallback on prod path). Verdict: STANDS, DOWNGRADED P1→P2.
|
||||
|
||||
### OCI-1.4-cap: probe 200-item array cap truncates the tier-1 inventory
|
||||
- **Priority:** P2
|
||||
- **Claim:** 1.4 — every `expect` entry is found in the probe inventory
|
||||
- **Location:** `probe/core.js:15-17` (`value.slice(0, 200)` inside `clip`, applied to ALL arrays incl. inventory)
|
||||
- **Proof:** polluted-vs-clean pair from OCI-4.4-ancestor (same commands); direct unit confirmation via `clip` on a 201-element array.
|
||||
```bash
|
||||
# polluted run analysis: N tools: 200, HAS echo_tool: False; clean: 63 tools, echo_tool present
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
N tools: 200, HAS echo_tool: False
|
||||
tier 1: expected tool "echo_tool" not in inventory tools [...]
|
||||
```
|
||||
- **Why it matters:** False tier-1 failures (and silently dropped hook records) whenever the tool list exceeds 200 — an artifact of evidence clipping, not of the plugin under test. Any heavy MCP/plugin union over 200 tools hits it even in a clean env.
|
||||
- **Remediation:** bypass array-capping for the `inventory` record (dedicated path that clips strings but not list membership), or raise the cap with a `…[+N more]` marker AND make tier-1 matching cap-aware (prefer the former).
|
||||
- **Verify the fix:** re-run polluted fixture (or a 201-tool synthetic inventory through `readProbe`+`runLoad` matching) → `echo_tool` found.
|
||||
- **Cross-examination:** Beta challenged (re-proven by same polluted-vs-clean pair; code confirmed; adversarial note accepted — in this instance triggered via ancestor pollution but defect is independent; requires unusual tool counts and cap is intentional bounding → P2 proportionate). Verdict: STANDS P2.
|
||||
|
||||
### OCI-1.4-providers: `expect.providers/models` accepted but silently ignored
|
||||
- **Priority:** P2
|
||||
- **Claim:** 1.4 (tier-1 inventory expectations) vs README config surface
|
||||
- **Location:** `src/tiers/load.ts:20-26,160-166` (`LoadTierExpected` has no providers/models; groups loop covers tools/commands/agents/skills/mcp only); `src/config/schema.ts:41-42` (fields exist, defaulted); `README.md:82-83` (`providers: [] # for provider plugins (tier 1)`)
|
||||
- **Proof:** from repo root with scratch copy of `tool-basic` adding `expect: {tools:[echo_tool], providers:[nope-provider], models:[nope-model]}`:
|
||||
```bash
|
||||
bun bin/opencode-plugin-ci.ts run --tier load --scenario provign --opencode 2.0.26 --out $AUDIT_TMP/provign-out
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
[('load', True, None)] for bogus provider+model names (tier 1 PASSED)
|
||||
```
|
||||
- **Why it matters:** Documentation contradicts behavior: a provider-plugin author asserting `providers: [mine]` gets a vacuous green tier 1 — the exact "tests pass while the check is inert" class the audit hunts.
|
||||
- **Remediation:** either implement (`ctx.provider`/`ctx.model` lists + match in `runLoad`, SPEC 6.4) or fail validation when non-empty (fail-closed; document as unsupported). Don't leave accepted-but-inert.
|
||||
- **Verify the fix:** re-run the Proof config → tier 1 fails naming the missing provider (or `validate` rejects the fields).
|
||||
- **Cross-examination:** Beta challenged (re-proved structurally: scratch config parses cleanly while `grep -n "providers\|models" src/tiers/load.ts` returns NOTHING; README advertises with no warning; `docs/NOTES.md:304` admits ignored but NOTES is internal, README is what authors read). Verdict: STANDS P2.
|
||||
|
||||
### OCI-6.1-allowed-wiring: required-vs-allowed version wiring has no automated guard
|
||||
- **Priority:** P2 (downgraded from P1 by cross-examination)
|
||||
- **Claim:** 6.1 ("Results for `opencode.versions[0]` are required while later versions are allowed failures (`allowedVersions` passed to the report renderer — the M10 bug class)")
|
||||
- **Location:** `src/run.ts:191-192` (`const required = versions[0]; const allowedVersions = versions.slice(1);`) wired to `renderMarkdown(resultsFile, { allowedVersions })` at `src/run.ts:443`
|
||||
- **Proof:** in `$AUDIT_TMP/wt`, one-line `const allowedVersions = versions.slice(1);` → `const allowedVersions: string[] = [];`, then from wt root:
|
||||
```bash
|
||||
bun run check
|
||||
rg -ln "allowed|versions\[0\]" test/e2e/
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
sabotage applied: src/run.ts
|
||||
213 pass / 0 fail (exit 0)
|
||||
rg: no e2e file mentions allowed versions
|
||||
```
|
||||
- **Why it matters:** The renderer is tested; the call site is not. If someone drops the `allowedVersions` argument, allowed-leg labels render wrong. Impact is report headers only — fail-closed direction (renders allowed leg as required = false alarm, never silent pass).
|
||||
- **Remediation:** extract `partitionVersions(versions)` (`{required, allowedVersions}`) into a pure function, assert `partitionVersions(["2.0.26","latest"]) === {required:"2.0.26", allowedVersions:["latest"]}`, and call it from `runCi`.
|
||||
- **Verify the fix:** re-run the sabotage → new test must fail; `bun run check` green on fixed tree.
|
||||
- **Cross-examination:** Gamma challenged (re-ran sabotage → 213 pass; BUT exit-code logic in `src/run.ts:468-492` filters exclusively on `r.opencode === required` and never reads `allowedVersions` — only `renderMarkdown` consumes it, i.e. headers only; sabotage direction renders false alarm not silent pass; M10 re-render path IS guarded by `test/unit/report.test.ts:131` `deriveAllowedVersions mirrors run.ts versions.slice(1)`). Verdict: fact STANDS, DOWNGRADED P1→P2.
|
||||
|
||||
### OCI-5.6-inert-window: `gate.window`/`gate.threshold` are accepted but inert
|
||||
- **Priority:** P2
|
||||
- **Claim:** 5.6 (absent-acks default + custom window/threshold)
|
||||
- **Location:** `src/config/schema.ts:60-61` (accepted, defaults 5/0.9), `src/gate/gate.ts:294-311` (tally re-implements `samples >= 5 && passes/samples >= 0.9` literally), `src/gate/verdict.ts:135` (verbatim upstream)
|
||||
- **Proof:** from repo root:
|
||||
```bash
|
||||
rg -n 'window|threshold' src/gate/gate.ts src/config/schema.ts
|
||||
rg -n '\.window|\.threshold|gate\.window' src/ | grep -v test || true
|
||||
# executed: scratch tool-basic copy with gate: {window: 50, threshold: 0.5, bootstrap: allow} → validate exit 0, no warning
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
const stable = [...tally]
|
||||
.filter(([, cell]) => cell.samples >= 5 && cell.passes / cell.samples >= 0.9)
|
||||
# zero reads of .window/.threshold outside schema defaults
|
||||
```
|
||||
- **Why it matters:** A user who sets `gate: {window: 10, threshold: 0.5}` per README gets silently different-from-expected gating — the "tests pass, behavior wrong" class. NOTES documents it, README does not.
|
||||
- **Remediation:** README + schema descriptions gain `deprecated — inert, upstream verdict hard-codes 5/0.9` notes and `validate` warns when non-default values are set.
|
||||
- **Verify the fix:** set non-default window/threshold → `validate`/run emits `gate.window is accepted but inert (upstream verdict uses >= 5 samples, >= 90%)` warning naming the config file.
|
||||
- **Cross-examination:** Gamma challenged (executed live `validate` with absurd window/threshold accepted silently; README:92-93 documents both with no inert warning; schema descriptions carry no inert note). Verdict: STANDS P2.
|
||||
|
||||
### OCI-2.5-files-symlink: `files:` assertion follows symlinks outside projectDir
|
||||
- **Priority:** P2
|
||||
- **Claim:** 2.5 (`files:` rejects any `..` escaping the project dir)
|
||||
- **Location:** `src/assert/expect.ts:318-370` (resolve+startsWith on lexical path only; `existsSync`/`readFileSync` follow symlinks)
|
||||
- **Proof:** from repo root:
|
||||
```bash
|
||||
bun -e 'import { evaluate } from "./src/assert/expect.ts"; /* symlink /etc -> $proj/linketc; evaluate({files:[{path:"linketc/passwd",exists:true}]}) */'
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
"linketc/passwd" => PASS (allowed) while "../evil.txt", "sub/../../evil.txt", "/etc/passwd" all correctly FAIL: path escapes the project dir.
|
||||
```
|
||||
- **Why it matters:** Plugin under test runs inside the sandbox and can create symlinks (`linketc -> /etc`). A scenario asserting on that path then reads outside content into `got` failure strings → `report.md`/SUMMARY exfiltration channel for host files. Requires plugin+scenario collusion but breaks stated containment.
|
||||
- **Remediation:** after lexical check, `realpath` the parent (or file if present) and re-check containment; or `lstat` + reject symlinks. Fail closed on `realpath` error.
|
||||
- **Verify the fix:** `linketc/passwd` must yield `path escapes the project dir` (or symlink rejection), legit `sub/../ok.txt` still passes.
|
||||
- **Cross-examination:** Alpha challenged (scope overreach — claim promises only `..` rejection which WORKS; challenge FAILS to move severity because lexical-only containment with a real collusion-gated host-file channel fits P2). Verdict: STANDS P2.
|
||||
|
||||
### OCI-6.4-action-pins: three GitHub Actions float on superseded v4 majors while v6/v7 exist
|
||||
- **Priority:** P2
|
||||
- **Claim:** 6.4 (packaging/CI pins) and 7.2 (CI definition); audit task "pins vs head"
|
||||
- **Location:** `action.yml:30-31,58`, `.github/workflows/ci.yml:9-10,16-18`
|
||||
- **Proof:** from repo root:
|
||||
```bash
|
||||
rg -n '(oven-sh/setup-bun|actions/(setup-node|upload-artifact|checkout))@v[0-9]+' action.yml .github/workflows/ci.yml
|
||||
git ls-remote https://github.com/actions/checkout.git HEAD
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
checkout: df4cb1c... v6.0.3 / d23441a... v6.1.0 / 3d3c42e... v7–v7.0.1; setup-node: ... v7–v7.1.0; upload-artifact: ... v7–v7.0.2; setup-bun: ... v2.2.0 == HEAD
|
||||
v4 tags still published, no deprecation notice; setup-bun@v2 == HEAD (no lag)
|
||||
```
|
||||
- **Why it matters:** Non-fatal lag (v4 still maintained, CI green), but floating majors mean CI semantics can shift without a repo diff; checkout v4 lacks v5+ hardening notes users may assume.
|
||||
- **Remediation:** bump `actions/checkout@v4`→`@v7` (or pinned SHA), `actions/setup-node@v4`→`@v7`, `actions/upload-artifact@v4`→`@v7` in both files after one green CI run; or document why v4 is held with a review date.
|
||||
- **Verify the fix:** `rg '(checkout|setup-node|upload-artifact)@v[0-9]' action.yml .github/workflows/ci.yml` shows only v7; `bun run check` + CI e2e green.
|
||||
- **Cross-examination:** Gamma challenged (verified v7 tags exist, v4 not deprecated/archived, v7.0.x brand-new so holding v4 short-term is defensible — exactly why P2 not P1 is correct). Verdict: STANDS P2.
|
||||
|
||||
### OCI-7.1-no-line: validation errors name file + field, not line
|
||||
- **Priority:** P3
|
||||
- **Claim:** 7.1 (fail validation with file path + line)
|
||||
- **Location:** `src/config/load.ts:31-39` (`formatIssues`: `${file}: validation failed:\n - at "${path}": ${message}`)
|
||||
- **Proof:** from repo root with six scratch configs:
|
||||
```bash
|
||||
bun bin/opencode-plugin-ci.ts validate --config $AUDIT_TMP/cfg-tests/case-strict/.opencode-ci/config.yaml
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
Configuration error: /tmp/.../case-strict/.opencode-ci/scenarios/bad.yaml: validation failed:
|
||||
- at "expect": Unrecognized key: "tool"
|
||||
(exit 2 in all six cases; file + dotted field path, no line number; yamlLineHint serves parse errors only)
|
||||
```
|
||||
- **Why it matters:** Minor — editors/CI must map field path back to a line by hand; the claim text should say "file path + field path".
|
||||
- **Remediation:** one-word doc fix in audit claim + README ("unknown keys fail validation naming the file and field"); optionally attach `linePos` via YAML AST mapping (not required).
|
||||
- **Verify the fix:** re-run the six scratch validates → messages unchanged (file+field); claim text corrected.
|
||||
- **Cross-examination:** Gamma challenged (executed typo'd `expect.tool` → file+dotted field, no line; trivially re-runnable in `$AUDIT_TMP/cross-f7`). Verdict: STANDS P3.
|
||||
|
||||
### OCI-registry-drift: pinned 2.0.26 is 6 releases behind head 2.0.32; plugin-API drift purely additive
|
||||
- **Priority:** P3
|
||||
- **Claim:** Beta task 1/2 (registry truth, upstream drift); touches 6.1's `latest` leg and 4.4's binary resolution
|
||||
- **Location:** `src/opencode/binary.ts:79` (`version === "latest" ? await npmViewVersion() : version` — resolved live per run)
|
||||
- **Proof:** from repo root:
|
||||
```bash
|
||||
npm view @opencode/cli versions --json | jq 'length'
|
||||
npm view @opencode/[email protected] version
|
||||
npm view @opencode/cli version
|
||||
curl -s -X POST https://api.osv.dev/v1/query -d '{"package":{"name":"zod","ecosystem":"npm"},"version":"4.1.0"}'
|
||||
git -C review/opencode show v2.0.26:packages/core/src/codemode/catalog.ts | diff - logs/catalog-head.ts || true
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
1141 versions, contains 2.0.26, head 2.0.32 (also @opencode/plugin: 1109 versions, contains 2.0.26, head 2.0.32; neither deprecated/yanked)
|
||||
yaml: 105 versions, 2.9.1 == dist-tag latest; zod: 1013 versions, 4.1.0 present, head 4.6.7
|
||||
OSV [email protected], [email protected], @opencode/[email protected] + latest counterparts: 0 vulns each
|
||||
API diffs pinned→head: tool.ts IDENTICAL, shell.ts IDENTICAL, catalog.ts IDENTICAL, hooks.ts IDENTICAL, policy.ts IDENTICAL, agent.ts cosmetic-only, session.ts +1 union member, permission.ts +assert method
|
||||
```
|
||||
- **Why it matters:** The default `opencode.versions: ["2.0.26","latest"]` leg floats to a different binary on every run (today 2.0.32); drift so far is additive-only, so the allowed-failure leg is low-risk — but nothing re-verifies this automatically.
|
||||
- **Remediation:** none required now; add a scheduled/CI head-leg e2e and consider a policy note on how far `latest` may drift before re-pinning.
|
||||
- **Verify the fix:** re-run the four `diff pinned vs HEAD` comparisons after each re-pin; `npm view @opencode/cli version` for the float value.
|
||||
- **Cross-examination:** Gamma challenged (re-verified head 2.0.32 vs pinned, no deprecated/yanked, OSV `{}` for all six package-versions, spot-check catalog diff identical, pinned tag SHA `9b4ec571` matches). Verdict: STANDS P3 (observation).
|
||||
|
||||
### OCI-6.3-working-dir: working-directory/artifact path unconstrained
|
||||
- **Priority:** P3 (downgraded from P2 by cross-examination)
|
||||
- **Claim:** 6.3 (Action inputs travel via env, working-directory safe)
|
||||
- **Location:** `action.yml:39`, `action.yml:61`
|
||||
- **Proof:** from repo root:
|
||||
```bash
|
||||
rg -n '\$\{\{' action.yml
|
||||
sed -n '37,56p' action.yml
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
run: scripts use only $IN_TIERS/$IN_VERSIONS/$ACTION_PATH (quoted) — no shell injection
|
||||
But working-directory: ${{ inputs.working-directory }} and path: ${{ inputs.working-directory }}/.opencode-ci/out are runner-evaluated paths with no validation; ../ resolves outside GITHUB_WORKSPACE
|
||||
```
|
||||
- **Why it matters:** Low exploitability today (inputs are workflow-author controlled, not PR metadata). A reusable action documenting `working-directory` as "plugin repo directory" should not silently honor `/tmp` or `../../` — hardening, not a live vuln.
|
||||
- **Remediation:** docs: "working-directory must be inside GITHUB_WORKSPACE; `..` and absolute paths are unsupported" + optional guard step via env passthrough (keep zero-interpolation pattern): `case "$IN_WD" in /*|*..*) echo "working-directory must be inside the workspace" >&2; exit 2;; esac`.
|
||||
- **Verify the fix:** `working-directory: ../` fails exit 2 with message; `.` still works.
|
||||
- **Cross-examination:** Alpha challenged (traversal requires WORKFLOW AUTHOR to pass `../`; author already has arbitrary code exec; PR metadata cannot reach field unless author wires it; runner behavior never executed live). Verdict: DOWNGRADED P2→P3.
|
||||
|
||||
### OCI-7.2-floating-majors: unpinned @v2/@v4 actions float (supply-chain drift hygiene)
|
||||
- **Priority:** P3 (downgraded from P2 by cross-examination; overlaps OCI-6.4-action-pins — retained as hygiene entry)
|
||||
- **Claim:** 7.2 / 6.3 (modern, supported, production-grade pins)
|
||||
- **Location:** `action.yml:30-31,58`, `.github/workflows/ci.yml:9-10,16-18` (`oven-sh/setup-bun@v2`, `actions/checkout@v4`, `actions/setup-node@v4`, `actions/upload-artifact@v4`)
|
||||
- **Proof:** from repo root:
|
||||
```bash
|
||||
rg -n '(oven-sh/setup-bun|actions/(setup-node|upload-artifact|checkout))@v[0-9]+' action.yml .github/workflows/ci.yml
|
||||
git ls-remote https://github.com/oven-sh/setup-bun HEAD
|
||||
git ls-remote https://github.com/actions/checkout HEAD
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
oven-sh/setup-bun@v2 (HEAD 0c5077e... == v2.2.0, no lag); actions/*@v4 (heads move; majors float)
|
||||
```
|
||||
- **Why it matters:** Floating majors can silently change runner behavior between audits; `@v2`/`@v4` are GitHub-endorsed consumption (official READMEs advertise major tags) and satisfy "modern, supported" — SHA-pinning is hygiene, not a medium defect. Overlaps OCI-6.4-action-pins version-lag entry.
|
||||
- **Remediation:** pin each `uses:` to `major-tag@sha` + Dependabot; or record upstream SHAs in NOTES.
|
||||
- **Verify the fix:** `rg` shows 40-char SHAs; `git ls-remote` matches pinned SHAs at audit time.
|
||||
- **Cross-examination:** Alpha challenged (floating majors are endorsed norm, no CVE/drift demonstrated, `bun.lock` + `--frozen-lockfile` cover npm side; elevating ecosystem default to P2 misprices it). Verdict: DOWNGRADED P2→P3.
|
||||
|
||||
### OCI-6.2-artifact-poisoning: default artifact name shared across matrix legs
|
||||
- **Priority:** P3 (downgraded from P2 by cross-examination)
|
||||
- **Claim:** 6.2 (matrix legs use distinct artifact names mirrored in gate.artifact)
|
||||
- **Location:** `action.yml:17-18` (default `opencode-plugin-ci-results`), `src/config/schema.ts:67` (default match), `src/gate/history.ts:104-206` (filters by artifact+workflow+branch+requiredVersion+live-only, newest-first)
|
||||
- **Proof:** from repo root:
|
||||
```bash
|
||||
rg -n 'artifact' action.yml src/config/schema.ts src/gate/history.ts README.md
|
||||
```
|
||||
- **Raw output:**
|
||||
```
|
||||
defaults match (opencode-plugin-ci-results both sides — good); history fetch is --workflow <file> --branch <branch> --name <artifact> then keeps only tier==="live" && opencode===requiredVersion && skipped===undefined rows
|
||||
Two matrix legs (2.0.26 required + latest allowed) uploading the same default name in the same workflow run collide (upload-artifact@v4 merges/overwrites by name)
|
||||
```
|
||||
- **Why it matters:** Misconfigured matrix (the default!) yields flaky-gate history poisoning: allowed `latest` results can mask required-leg verdicts. Docs warn, but the default is the trap. No live workflow dispatched (no GitHub writes allowed) — inferred hazard.
|
||||
- **Remediation:** fail validation when `gate.artifact` equals the default AND more than one opencode version is configured, with message pointing at README matrix section; or namespace downloads by version explicitly.
|
||||
- **Verify the fix:** matrix config with `versions: ["2.0.26","latest"]` + default artifact → validation error naming file+field; per-leg distinct artifacts pass.
|
||||
- **Cross-examination:** Alpha challenged (repo ALREADY documents requirement twice — README:275-292 with YAML example + action.yml:18 input description; default single-leg use safe; failure requires ignoring docs; inferred user-misconfiguration hazard is P3 enhancement). Verdict: DOWNGRADED P2→P3.
|
||||
|
||||
## Claim Ledger
|
||||
|
||||
| Claim | Status | Evidence |
|
||||
|---|---|---|
|
||||
| 1.1 tier-1 boots real binary, asserts code/timeout/no-errors | VERIFIED | `src/sandbox/sandbox.ts:159-174`, `src/tiers/load.ts:119-132`; clean fixture runs code 0 + durationMs; fake-argv log |
|
||||
| 1.2 plugin identity via probe inventory | VERIFIED | `src/tiers/load.ts:139-151`; throws `failed` path names `fixture boom`; S13 sabotage caught by e2e |
|
||||
| 1.3 logfmt plugin-log problems | VERIFIED | `src/opencode/logs.ts:6-76,87-113`; throws second failure verbatim |
|
||||
| 1.4 expect tools/commands/agents/skills/mcp + TUI static | VERIFIED+findings | tier-1 pass + negatives (`nope-cmd` fails naming inventory); `tui-only` pass / `tui-broken` fail; gaps OCI-1.4-cap, OCI-1.4-providers |
|
||||
| 1.5 tier-1-fail skips tiers 2-3 | VERIFIED+gap | throws/tui-broken full runs show `skipped: "tier 1 failed"`; skip unpinned → OCI-1.5-skip |
|
||||
| 2.1 tools before/after pairing, count default | VERIFIED | `src/probe/read.ts:106-135`, `src/assert/expect.ts:154-191`; tool-basic/direct pass; S7+S11 sabotages caught |
|
||||
| 2.2 output text + truncated got | VERIFIED | `src/assert/expect.ts:28-78`, `src/assert/matchers.ts:87-205`; S8b caught |
|
||||
| 2.3 hooks pre/post + phase default post | VERIFIED | hook-mutate/shell-env/promptfix e2e + swap experiments both directions @2.0.26+2.0.32 |
|
||||
| 2.4 request nth/system/tools catalog paths | VERIFIED | system-inject + tool-direct + catreq e2e; negatives name got values |
|
||||
| 2.5 noErrors default, exitCode default, files `..` | PARTIAL | exitCode + files `..` VERIFIED; `noErrors` default fallback unpinned → OCI-2.5-noerrors (P2); symlink bypass → OCI-2.5-files-symlink |
|
||||
| 3.1 stub auto-wrap via findCatalogPath | VERIFIED | `src/stub/stub.ts:243-268`; tool-basic vs tool-direct e2e; S1 caught |
|
||||
| 3.2 request classification + exhausted script | VERIFIED | `src/stub/stub.ts:224-233`; pure + live-stub exhausted proof |
|
||||
| 3.3 probe metadata-last after snapshot | PARTIAL | behavior implication proven via hook-order divergence; ordering untested → OCI-3.3-meta (P2, merged) |
|
||||
| 3.4 system parts clipped at SYSTEM_MAX | VERIFIED | `probe/core.js:120-136`; system-inject survives @both versions |
|
||||
| 4.1 shim file-URL re-export | VERIFIED | `src/plugin/shim.ts:19-24`; spaces e2e `%20`; S6 caught |
|
||||
| 4.2 resolveEntries order + no-entry exit 2 | VERIFIED | `src/plugin/entry.ts:164-220`; 8/8 pure order checks; no-entry exit 2 |
|
||||
| 4.3 plugin order [pre,plugin,post] | VERIFIED+gap | default + swapped proofs @2.0.26+2.0.32; load copy unpinned → OCI-4.3-order (P2) |
|
||||
| 4.4 binary resolution + sandbox env | VERIFIED+findings | OPENCODE_BIN + latest-install + 7 env assertions green; ancestor leak → OCI-4.4-ancestor (P1); cap → OCI-1.4-cap |
|
||||
| 4.5 install/build once, fail exit 2 | VERIFIED | `src/plugin/install.ts:12-58`; install exit-3 → exit 2, no tier ran |
|
||||
| 5.1 live k-of-n + budget accounting | PARTIAL | defaults + cross-field + usage/budget with fake binary VERIFIED; real-model counting UNVERIFIABLE (no key, refusal boundary) |
|
||||
| 5.2 pre-attempt budget skip + no-key skip | VERIFIED | no-key exit 0 + spent>=limit zero-spawn; S9b caught |
|
||||
| 5.3 judge isolation + strict boolean + sentinel | VERIFIED | argv + tool-less @both versions + parseVerdict matrix; S5 caught |
|
||||
| 5.4 judge tokens count + cross-field rules | VERIFIED | costUSD 0.3 = 0.25+0.05 accounting; 6 cross-field shapes exit 2 |
|
||||
| 5.5 flake gate verbatim constants | VERIFIED | byte-diff vs IUMBTEMS (5-line header only); sabotage `>=500` caught |
|
||||
| 5.6 absent-acks default + inert window/threshold | PARTIAL | absent default VERIFIED + sabotage caught; window/threshold inert → OCI-5.6-inert-window (P2) |
|
||||
| 6.1 required[0]/allowed-rest + exit codes | PARTIAL | structure VERIFIED; wiring unpinned → OCI-6.1-allowed-wiring (P2) |
|
||||
| 6.2 gate history via gh artifacts | VERIFIED | `src/gate/history.ts` gh list/download; defaults match; zero cache usage |
|
||||
| 6.3 Action env-passthrough + lockfile + outputs | PARTIAL | passthrough VERIFIED by inspection; invariant unguarded → OCI-6.3-action-guard (P1); working-dir → OCI-6.3-working-dir (P3) |
|
||||
| 6.4 npm packaging + engines + schema sync | PARTIAL | tarball + probe hygiene + schemas-in-sync VERIFIED; unpublished → OCI-6.4-unpublished (P1); files unguarded → OCI-6.4-files-guard (P1); pins → OCI-6.4-action-pins (P2) |
|
||||
| 7.1 strict schemas + (?i) + cross-field | PARTIAL | 19/19 `.strict()` + 6 cross-field + `(?i)` executed VERIFIED; sabotage caught; line-number overpromise → OCI-7.1-no-line (P3) |
|
||||
| 7.2 CI reproduces tiers | PARTIAL | `bun install` + `bun run check` 213 pass + pins VERIFIED; full e2e deferred to Alpha; floating majors → OCI-7.2-floating-majors (P3); drift → OCI-registry-drift (P3) |
|
||||
| 7.3 probe redaction + noEvents + PRT prohibition | PARTIAL | env/header redaction VERIFIED incl. AUTH/COOKIE; redact untested → OCI-7.3-redact-untested (P1); other surfaces leak → OCI-7.3-evidence-leaks (P1) |
|
||||
|
||||
## Sabotage Results
|
||||
|
||||
Suite = `bun run check` (typecheck + biome + 213 unit tests) unless e2e noted. All in `$AUDIT_TMP/wt`, restored after each. Duplicates noted.
|
||||
|
||||
| Invariant | One-line sabotage (file:line, before → after) | Suite result | Caught? |
|
||||
|---|---|---|---|
|
||||
| [3.1] auto-wrap `src/stub/stub.ts:256` `findCatalogPath(...)` → `undefined` | Alpha S1 | check: 2 unit FAIL + tier2 e2e FAIL | YES |
|
||||
| [3.3] metadata-last `probe/core.js:64` metadata spread first, payload spread last | Alpha S2 = Gamma metadata-last (DUPLICATE) | check 0 fail (213); e2e tier2 green; `bun test ./test/unit` 213 pass | NO → OCI-3.3-meta |
|
||||
| [4.3] load order `src/tiers/load.ts:89` `[pre,shim,post]` → `[post,shim,pre]` | Alpha S3b | check 0 fail; `probe.test.ts` green (own order) | NO → OCI-4.3-order |
|
||||
| [1.2] identity `src/tiers/load.ts:140` `=== shimServerJs` → `!==` | Alpha S13 | check green; e2e tier1 FAIL (throws flips to pass) | YES (e2e) |
|
||||
| [2.1] count default `src/assert/expect.ts:163` `?? {gte:1}` → `?? {gte:0}` | Alpha S7 | check: unit FAIL | YES |
|
||||
| [2.1] callID pairing `src/probe/read.ts:113` `befores.get(record.callID)` → `befores.get(record.tool)` | Alpha S11 | check: unit FAILs + e2e tier2 FAIL | YES |
|
||||
| [2.2] truncate `src/assert/matchers.ts:11` drop `truncate()` in `show()` | Alpha S8b | check: unit FAIL | YES |
|
||||
| [2.5] noErrors `src/assert/expect.ts:381` `?? true` → `?? false` | Alpha S10b | check 0 fail; e2e tier2 green | NO → OCI-2.5-noerrors |
|
||||
| [2.5] files `..` `src/assert/expect.ts:324` neuter rejection | Alpha S12c2 = Gamma files `..` (DUPLICATE) | check: unit FAIL `escaping paths are rejected` | YES |
|
||||
| [5.2] budget `src/tiers/live.ts:308` `>=` → `>` / `if (false)` | Alpha S9b = Gamma budget (DUPLICATE) | check: unit FAIL `exhausted budget skips without spawning` | YES |
|
||||
| [5.3] judge `src/tiers/judge.ts:127` `=== "boolean"` → truthy | Alpha S5 = Gamma judge (DUPLICATE) | check: unit FAIL `pass as a string is rejected` | YES |
|
||||
| [4.1] shim `src/plugin/shim.ts:22` file-URL → raw path | Alpha S6 | check: unit FAIL `exact re-export` | YES |
|
||||
| [1.5] cascade `src/run.ts:248` `if (!tier1Pass)` → `if (!tier1Pass && tier1Pass)` | Alpha S4c | check 0 fail (213); `cli.test.ts` green | NO → OCI-1.5-skip |
|
||||
| [5.5] verbatim `src/gate/verdict.ts:135` `samples >= 5` → `samples >= 500` | Beta 5.5 | red (gate-verdict sticky-red tests fail) | YES |
|
||||
| [5.6] absent-acks `src/gate/gate.ts:92` default → `throw` | Beta 5.6 | red (`readAcknowledgments > absent defaults` + 4 runGate fails) | YES |
|
||||
| [6.1] allowed wiring `src/run.ts:192` `slice(1)` → `[]` | Beta 6.1 | GREEN (213 pass) | NO → OCI-6.1-allowed-wiring |
|
||||
| [6.3] no-interpolation `action.yml:43,48` drop passthrough + inject `# ${{ inputs.tiers }}` | Beta 6.3 | GREEN (213 pass) | NO → OCI-6.3-action-guard |
|
||||
| [6.4] files `package.json:17-26` remove `"probe"` | Beta 6.4 | GREEN (213 pass) | NO → OCI-6.4-files-guard |
|
||||
| [7.1] strictness naive `src/config/schema.ts` delete `.strict()` | Beta 7.1-naive | red via biome FORMAT (incidental, not semantic) | INCIDENTAL |
|
||||
| [7.1] strictness stable `src/config/schema.ts` `.strict()` → `.strip()` | Beta 7.1-stable = Gamma strict (DUPLICATE) | red (`expect.tool typo fails` + `loadConfig names file+field` fail) | YES |
|
||||
| [7.3] redact superset `probe/core.js:3` drop AUTH/COOKIE | Gamma redact-superset | `bun run check`: 213 pass GREEN | NO → OCI-7.3-redact-untested |
|
||||
| [7.3] redact break `probe/core.js:3` → `/KEY/i` | Gamma redact-break | `bun test ./test/unit`: 213 pass GREEN | NO → OCI-7.3-redact-untested |
|
||||
|
||||
Caught 15/20 unique invariants (counting 5 duplicates once). Uncaught 5 unique → 4 Alpha P1→(1 P1 + 3 P2 after cross) + Beta 3 (1 P2 + 2 P1) + Gamma 2 unique (redact P1, metadata duplicate). Final findings 22 (8 P1, 9 P2, 5 P3) after severity corrections and merges.
|
||||
|
||||
## Regression Ledger
|
||||
|
||||
| Previous finding ID | Previous priority | Status now: FIXED / STILL OPEN / REGRESSED / NO LONGER APPLICABLE | Evidence |
|
||||
|---|---|---|---|
|
||||
| (none — First run) | n/a | n/a | No previous `AUDIT_REPORT_FINDINGS.md` existed at audit start; nothing to re-verify. Next run must list all 22 OCI-* IDs above with re-verification from scratch. |
|
||||
|
||||
## Not Audited / Unverifiable
|
||||
|
||||
- Live tier against a REAL model (5.1 k-of-n counting, judge quality, real spend): no API key in this environment; refusal boundary respected (only skip-path + stub-judge paths with throwaway dummy key, loopback stub only). Needs a human-supplied key in an isolated account with a tight budget to check.
|
||||
- `opencode models` bare-id/default resolution against a live key-widened list: unit-covered + passthrough proven; live-list behavior not observable here.
|
||||
- Windows/macOS specifics (`pathToFileURL` shim with `C:\` paths, `cmd /c` install path, drive-letter handling): Linux host; space handling proven (`space%20dir` e2e), `C:\`-style untestable here. Needs a Windows runner to check.
|
||||
- `assistant`/`event-file`-style timing races (double `inventory` records observed — hook-mutate clean run 2 identical active inventories; polluted run failed-then-active; `readProbe` last-wins): no flake proven, noted as robustness observation. Needs timing-instrumented runs to check.
|
||||
- Full `bun run test:e2e --timeout 600000` (24 tests) in one shot in Gamma/Beta scope: not run there by design (Alpha's runtime domain); equivalent fixture-by-fixture CLI coverage achieved via fresh binaries. Next run should still attempt the full suite once in a clean-TMPDIR host.
|
||||
- Gate real history path (`gh run list`/`run download`, denial/fallback on `gh` failure, token scope beyond code-read): covered only by injected-`fetchHistoryFn` unit tests, never against real `gh`/API here. Needs a scratch repo with staged artifacts and a scoped token to check.
|
||||
- `bunx` resolution post-publish: untestable until OCI-6.4-unpublished is remediated (package does not exist on the registry).
|
||||
- Action injection detonation in a real runner (malicious `tiers`/`versions` payloads, `working-directory: ../` runner-side, same-name matrix artifact overwrite): static verification only (zero `${{ }}` in `run:` scripts, quoted env, `takeValue` rejection); no GitHub writes allowed. Needs a disposable test repo + workflow dispatch to check.
|
||||
- `pull_request_target` with live secrets: deliberately never tested with a real key. PoC with throwaway `sk-test-*` dummy proves the exfil channel (evil plugin `setup` read env + wrote `/tmp/gamma-evil-exfil.txt`); live-key impact is inferred. Do not test with a real key in an audit.
|
||||
- Upstream head drift beyond static API diffs (Beta) + Alpha's 10-fixture matrix (runtime): local `review/opencode` clone (2026-10-10) is 8 days behind remote HEAD `c38d83c0`; remote-HEAD API diffs rely on Beta's logs. Needs a fresh clone at each re-run to check. Note: prompt's "fork, not sst/opencode" premise is outdated — `sst/opencode` 301-redirects to `anomalyco/opencode`, both remotes return identical SHA `c38d83c0c4f92466ff9c8e12b8a741698e811012`; anomalyco IS canonical.
|
||||
- Symlink/absolute workspace edge cases on non-Linux filesystems and concurrent-leg `/tmp` cross-talk under load: inferred from `join`/`resolve` proofs, not executed under contention. Needs a loaded multi-leg run to check.
|
||||
@@ -0,0 +1,127 @@
|
||||
# Review: opencode-plugin-ci v1 (PR #4, merged at `af6675c`)
|
||||
|
||||
Reviewed on 2026-10-18 from a fresh clone of `main`. The review was read-only: nothing was committed and the tree stayed clean (`git status --short` showed nothing). The baseline is `docs/SPEC.md` and `docs/IMPLEMENTATION_PROMPT.md` (the brief).
|
||||
|
||||
**Verdict: NEEDS-WORK.** The components are well built and well tested. The orchestration layer, which nothing tests, has four blockers. I reproduced each blocker with a throwaway fixture.
|
||||
|
||||
## 0. What I re-ran
|
||||
|
||||
| Claim | Result |
|
||||
|---|---|
|
||||
| `bun run check` with 213 unit tests passing | Confirmed: 213 pass, 0 fail (typecheck and biome clean) |
|
||||
| `bun run test:e2e` with 24 passing | Confirmed: 24 pass, 0 fail, 32.8s, using `@opencode/[email protected]` |
|
||||
| Schemas are in sync | Confirmed: `bun scripts/gen-schema.ts` left `schema/` unchanged |
|
||||
| GitHub CI | Green: `check` and `e2e` passed on the PR head and on the merge commit |
|
||||
| `verdict.ts` is a verbatim copy | Confirmed. With the 5-line header removed, it diffs empty against IUMBTEMS `packages/core/src/evals/gate.ts` (c6455bf). The same holds for `exit.ts`. The test copy differs only in its 2 import lines |
|
||||
| No `${{ inputs.* }}` inside `run:` blocks | Confirmed. Inputs reach the script only through `env:`, and `bun install --frozen-lockfile` runs in `github.action_path` |
|
||||
|
||||
## 1. The 9 known gaps
|
||||
|
||||
| # | Gap | Verdict | Evidence |
|
||||
|---|---|---|---|
|
||||
| 1 | Tier 1 ignores `expect.providers` and `expect.models` | **Understated** | The schema accepts both (`src/config/schema.ts:41-42`), and README.md:82-83 advertises them "for provider plugins (tier 1)". A provider plugin's expectation therefore passes silently. The fields should be rejected until they are implemented |
|
||||
| 2 | No `workspace.tar` and no per-scenario `result.json` | Accurate | Evidence dirs contain only events, probe, stub-requests and stderr |
|
||||
| 3 | No flag for `webfetch` or `websearch` use in scripted runs | Accurate | Nothing in `src/report/markdown.ts` inspects tool names |
|
||||
| 4 | The SPEC 5.3 API names were illustrative | Accurate | The exports in `src/index.ts` match the README. Gap list misses that the `bun:test` escape hatch it points to is broken (blocker B3 below) |
|
||||
| 5 | `ctx.storage` is not snapshotted | Accurate | Minor |
|
||||
| 6 | No dogfood workflow | Accurate | This is my brief's fault: M13 never asked for it, although SPEC 10.8 does |
|
||||
| 7 | The Action has no `config` input and never outputs `skipped` | Accurate | My brief's `action.yml` template had the same omissions (`action.yml:20-23`, `:55`) |
|
||||
| 8 | The redaction regex is wider than the SPEC's | Accurate and benign | The brief asked for the wider regex. SPEC 3.3 should be updated to match |
|
||||
| 9 | Custom `gate.window` and `gate.threshold` have no effect | **Understated, and the reason is half wrong** | The fields are still accepted (`schema.ts:60-61`) and documented (README:92-93), so a custom value is silently ignored. Only the sticky-red helper inside the verbatim file hard-codes 5 and 0.9 (`verdict.ts:135`). The main `stable` list is built in the non-verbatim wrapper (`src/gate/gate.ts:309`). Best fix: remove both fields from the schema, README and SPEC, so a custom value fails validation instead of doing nothing |
|
||||
|
||||
## 2. New gaps
|
||||
|
||||
### Blockers
|
||||
|
||||
**B1. Entries are resolved before install and build run, so compiled plugins can never run in CI.** `src/run.ts:171` resolves the entry before running the build at `:178-185`. In a fresh checkout, `dist/` does not exist yet, so a plugin whose `main` or `exports` points at `dist/server.js` exits 2 with "No server entry" before its build ever runs. Reproduced: a plugin with `"main": "./dist/server.js"` and `build: mkdir -p dist && cp src/server.js dist/server.js` exits 2, and `dist/` is never created. This is the most common real plugin layout, and the README's own example (`entry: dist/server.js`). **This one is my fault:** step 2 of the brief's M10 `run.ts` order put resolution before the build. SPEC 6.1 has the order right.
|
||||
|
||||
**B2. Tier 1 ignores `plugin.entry`.** `src/tiers/load.ts:62` calls `resolveEntries(pluginPath)` without the explicit entry, and `RunLoadOptions` has no field to pass it in. Reproduced: with `plugin.entry: src/plugin.ts` and an empty `package.json`, `validate` exits 0 but `run --tier load` exits 2. When `package.json` points at a different file, tier 1 silently tests that file while tiers 2 and 3 test the configured one.
|
||||
|
||||
**B3. The `bun-test` escape hatch can never pass.** `src/run.ts:391` runs `bun test .opencode-ci`. Bun 1.4.2 skips dot-directories when it discovers tests, so this always fails with "The following filters did not match any test files … .opencode-ci", even when every test passes. Any repo that uses the documented escape hatch is permanently red. Verified fix: pass each discovered file as an explicit `./`-relative path. `bun test ./.opencode-ci/x.test.ts` runs the file and passes or fails correctly.
|
||||
|
||||
**B4. Running out of budget hides a failed live attempt.** `src/tiers/live.ts:308-323` returns `pass: true, skipped: "budget exhausted"` even after attempts have already failed. Reproduced with a fake opencode binary that reports a cost of $1.50 against a $1 budget: attempt 1 failed (`expected count >= 1, got 0`), the case was reported as skipped, and the run exited 0. The gate never saw the failure, because `gateApplies` ignores skipped rows. A CI tool must never turn a recorded failure green.
|
||||
|
||||
### Should-fix
|
||||
|
||||
- **S1. The live budget resets for each opencode version.** `src/run.ts:300` creates the budget inside the version loop. SPEC 5.1 calls it a "hard stop for the whole live tier" and the brief calls it a "run-wide counter". With `["2.0.26", "latest"]`, spend can reach twice the budget, partly on a version whose failures are allowed anyway.
|
||||
- **S2. A typo in `plugin.entry` silently falls through to auto-detection** (`src/plugin/entry.ts:169-177`). If `index.ts` exists, the wrong file is tested and nobody is told. This breaks the "typos never silently pass" rule. An explicit entry that does not exist after the build should be a configuration error.
|
||||
- **S3. Some config fields are accepted but never read.** Besides gaps 1 and 9 above: `live.timeoutMs` (`schema.ts:53`) is never used, because live attempts use `scenario.timeoutMs`. `judgeReply` on scripted scenarios (`schema.ts:229`) is passed to the stub, but scripted mode never sends a judge request, so it has no effect. My brief was inconsistent on `judgeReply`. Reject each of these fields or wire it up.
|
||||
- **S4. Negative matchers pass when the value is missing.** `src/assert/matchers.ts:110-118` and `:133-144` convert `undefined` to the string `"undefined"`. So `input: { comand: { notContains: secret } }` (note the typo) passes. In the spec's flagship "redacts secrets" example, that is a silent pass on the one assertion that matters. A missing value should fail every matcher except `exists: false`.
|
||||
- **S5. Namespaced tool names are not normalized in assertions.** `src/assert/expect.ts:159-161` compares `call.tool === tool.name` exactly. The M2 notes show the hook sees `ns_spike_echo`, while the script auto-wrap and `request.tools` accept `spike_echo`. So `expect.tools: [{ name: spike_echo }]` fails with a count of 0 for a namespaced tool, and the README does not mention this. Match the way `findCatalogPath` does: exact name, last segment, or `_`-joined form.
|
||||
- **S6. The gate finds no history on pull requests** (inferred from GitHub's docs, not run). `src/gate/gate.ts:204` filters history with `GITHUB_REF_NAME`. On `pull_request` events that value is `<n>/merge`, but `gh run list --branch` matches head branch names. So a live failure on a same-repo PR always exits 2. With `bootstrap: allow`, every PR live failure passes forever. The upstream gate has the same behavior, but it only runs on `main`. My brief copied it. Use base-branch history (`GITHUB_BASE_REF`) on PRs.
|
||||
- **S7. Gate history cannot be wired up for matrix builds.** The `artifact-name` input (`action.yml:17-19`) is never passed to the CLI, and `gate.artifact` lives in the one config file that every matrix leg shares. Distinct names per leg therefore cannot match, and README.md:274-292 admits as much. This design came from my brief. Fix: the Action passes its artifact name to the CLI (a new `--gate-artifact` flag, or an env var), and that value overrides the config.
|
||||
- **S8. The orchestration has no tests** (details in section 3).
|
||||
|
||||
### Nits
|
||||
|
||||
- `pluginLogProblems` matches only the shim path (`src/tiers/load.ts:152-155`). An `ERROR` line that names the real `src/server.ts` but not the plugin id is missed.
|
||||
- Every evidence dir holds two copies of the probe log, one named `probe-live.jsonl` even in tiers 1 and 2 (`scripted.ts:50`, `load.ts:84`).
|
||||
- `workspace` keys are not checked for `..` (`src/sandbox/sandbox.ts:108-112`), though `files` paths are (`expect.ts:324`).
|
||||
- With `OPENCODE_PLUGIN_CI_JUDGE_VIA_STUB` set, the stub judge defaults to `{"pass":true}` (`judge.ts:193`). The default should be a fail.
|
||||
- An install or build failure is reported as "Configuration error" with exit 2, and no report is written (`src/plugin/install.ts:55`).
|
||||
- `run --scenario <typo>` runs nothing and exits 0 (`run.ts:202-205`).
|
||||
- Results record `opencode` as the requested string (`latest`), not the resolved version the brief asked for (`run.ts:212`).
|
||||
- The "using defaults" warning prints twice (`config/load.ts:101` and `run.ts:165`).
|
||||
- The comment at `codemode.ts:11-17` says the brief's regex captured the `(`. It didn't. The code is still correct.
|
||||
- The `tool-direct` fixture doesn't prove the tool is direct: `request.tools` includes catalog names (`expect.ts:130-144`). Adding `noTools: [execute]` would prove it.
|
||||
- Gate temp dirs are never cleaned up, and each of the 15 history runs downloads the full evidence artifact (`history.ts:155-156`).
|
||||
|
||||
## 3. Test quality
|
||||
|
||||
The component suite is good:
|
||||
- The gate tests are a verbatim port.
|
||||
- Schema, matchers, entry resolution, events, logs and the report snapshot have real branch coverage.
|
||||
- The parsers are tested against captured opencode output.
|
||||
- The e2e suite runs every fixture's happy path and the tier 1 failure modes against the real binary.
|
||||
|
||||
The orchestrator is not covered:
|
||||
- `run.ts` has no tests for its step order, exit codes, required versus allowed versions, or the skip after a failed tier 1.
|
||||
- The live attempt loop has no tests. `live.test.ts` covers only the two skip paths, and the loop is where B4 sits.
|
||||
- The `bun-test` case has no tests.
|
||||
- No run-level test uses `plugin.entry` or a build step.
|
||||
- `action.yml` is never exercised; no CI job runs `uses: ./`.
|
||||
|
||||
All four blockers sit in this uncovered layer, which is why "213 + 24 green" coexists with them. Negative coverage is also thin. SPEC 9 asks for passing and deliberately failing scenarios per fixture, but only `tool-basic` has a failing scenario. The other pass scenarios do mix positive and negative matchers, so they are not vacuous.
|
||||
|
||||
**Verdict:** adequate for components, inadequate for the orchestrator.
|
||||
|
||||
## 4. Security
|
||||
|
||||
No blockers.
|
||||
|
||||
- The sandbox env is an allowlist: PATH, the XDG dirs and the configured live keys only (`sandbox.ts:134-146`). I ran a canary key through a sandbox at `--log-level info` and `all`. It never appeared in stdout, stderr or the recorded model requests.
|
||||
- The probe redacts env values and headers whose names look secret. Tool inputs and outputs are recorded as they are, which is what SPEC promises. A scenario that runs `env` would therefore put the key into `events.jsonl` and `probe.jsonl`, which the Action uploads. The README tells authors never to print secrets; consider redacting known live-key values from evidence before upload.
|
||||
- The README warns against `pull_request_target` (README:323), and the example workflow sets `actions: read` and nothing broader.
|
||||
- The judge is plugin-free and tool-less (a deny-all agent), runs only after the deterministic checks pass, and treats an invalid reply as a fail.
|
||||
- Install, build, the static TUI import and `bun test` inherit the full environment, including `GH_TOKEN` and the live keys. They run the PR's own code, so this is not an escalation. Scoping `GH_TOKEN` to the gate would still be tidier.
|
||||
|
||||
## 5. Overall: NEEDS-WORK
|
||||
|
||||
Top 3 follow-ups, written as junior-agent tasks:
|
||||
|
||||
1. **Fix the run order and entry plumbing (B1, B2, S2).**
|
||||
- In `src/run.ts`, run `plugin.install` and `plugin.build` *before* `ensureEntriesOrThrow`.
|
||||
- Add `serverEntry: string` to `RunLoadOptions` in `src/tiers/load.ts`, delete its internal `resolveEntries` call, and pass `entries.server` from `run.ts`.
|
||||
- In `resolveEntries`, throw `ConfigError("plugin.entry <path> does not exist")` when an explicit entry is not a file.
|
||||
- In `cmdValidate`, when no entry resolves and `plugin.build` is set, print a warning ("entry may be created by the build") and exit 0.
|
||||
- Add two e2e fixtures:
|
||||
- `built-entry`: `"main": "./dist/server.js"`, a build that copies `src/server.js` into `dist/`, and `dist/` git-ignored.
|
||||
- `explicit-entry`: `plugin.entry: src/plugin.ts` and no `main`.
|
||||
- Assert that `run --tier load` exits 0 in a clean temp copy of each fixture.
|
||||
2. **Fix `bun-test` discovery (B3).**
|
||||
- In `src/run.ts`, replace `["bun", "test", ".opencode-ci"]` with `["bun", "test", ...testFiles.map((f) => "./" + relative(cwd, f))]`.
|
||||
- Add e2e tests on a temp copy of `tool-basic`:
|
||||
- a passing `.opencode-ci/ok.test.ts` makes the `bun-test` case pass;
|
||||
- a failing one makes it fail, with the failing test's name in `failures`.
|
||||
3. **Fix the live budget and test the attempt loop (B4, S1).**
|
||||
- In `runLiveScenario`, when the budget is exhausted and `attempts.length > 0`, return `pass: false` with "budget exhausted after N attempt(s): k passed, needed p". Return `skipped` only when no attempt ran.
|
||||
- In `run.ts`, create the budget once, above the version loop.
|
||||
- Add unit tests that drive `runLiveScenario` with a fake opencode shell script, as in this review's reproduction. The script prints a `text` event and a `step_finish` event with a chosen `cost`. Cover:
|
||||
- 2 of 3 attempts passing;
|
||||
- early stop after 2 passes;
|
||||
- early stop once passing is impossible;
|
||||
- the judge not running when the deterministic checks fail;
|
||||
- the budget running out mid-scenario, which must fail;
|
||||
- one budget shared across two versions.
|
||||
|
||||
After these three, I'd handle the rest as one batch: reject the ignored config fields (S3, gaps 1 and 9), fail matchers on missing values (S4), normalize namespaced names (S5), and fix gate history on PRs and in matrix builds (S6, S7).
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@heretek-ai/opencode-plugin-ci",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.1",
|
||||
"description": "Verify opencode v2 plugins actually work, locally and in GitHub Actions",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
|
||||
+24
-7
@@ -4,7 +4,7 @@ const SECRET_KEY = /KEY|TOKEN|SECRET|PASSWORD|AUTH|COOKIE/i;
|
||||
const MAX = 4000;
|
||||
const SYSTEM_MAX = 50000; // system parts are long; plugin additions often land at the end
|
||||
|
||||
function clip(value, depth = 0, max = MAX) {
|
||||
function clip(value, depth = 0, max = MAX, capArrays = true) {
|
||||
if (typeof value === "string") {
|
||||
return value.length > max
|
||||
? `${value.slice(0, max)}…[+${value.length - max}]`
|
||||
@@ -13,11 +13,16 @@ function clip(value, depth = 0, max = MAX) {
|
||||
if (value === null || typeof value !== "object") return value;
|
||||
if (depth > 6) return "[depth]";
|
||||
if (Array.isArray(value)) {
|
||||
return value.slice(0, 200).map((item) => clip(item, depth + 1, max));
|
||||
// OCI-1.4-cap: the 200-item cap truncated tier-1 inventory lists, so a
|
||||
// plugin union over 200 tools failed tier 1 on clipping, not on the
|
||||
// plugin. Inventory membership must never be truncated; only strings
|
||||
// are clipped there. Every other record keeps the array cap.
|
||||
const items = capArrays ? value.slice(0, 200) : value;
|
||||
return items.map((item) => clip(item, depth + 1, max, capArrays));
|
||||
}
|
||||
const out = {};
|
||||
for (const [key, item] of Object.entries(value)) {
|
||||
out[key] = clip(item, depth + 1, max);
|
||||
out[key] = clip(item, depth + 1, max, capArrays);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -49,6 +54,19 @@ const safe = async (fn) => {
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Append one probe record. Metadata (`phase`, `seq`, `t`, `kind`) spreads
|
||||
* LAST so a payload field can never overwrite it (OCI-3.3-meta). Exported so
|
||||
* unit tests can pin the ordering against a payload-wins sabotage; `write`
|
||||
* inside `makeProbe` is the only other spread site and delegates here.
|
||||
*/
|
||||
export function writeRecord(out, kind, data, phase, seq, max = MAX) {
|
||||
appendFileSync(
|
||||
out,
|
||||
`${JSON.stringify({ ...clip(snapshot(data), 0, max, kind !== "inventory"), phase, seq, t: Date.now(), kind })}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
export function makeProbe(phase) {
|
||||
let seq = 0;
|
||||
let inventoryDone = false;
|
||||
@@ -58,11 +76,10 @@ export function makeProbe(phase) {
|
||||
const out = ctx.options?.out;
|
||||
if (!out) return;
|
||||
// Metadata goes LAST so a payload field can never overwrite `kind`/`phase`.
|
||||
// Single spread site: `writeRecord` below. Inventory records bypass
|
||||
// the 200-item array cap (OCI-1.4-cap).
|
||||
const write = (kind, data, max = MAX) =>
|
||||
appendFileSync(
|
||||
out,
|
||||
`${JSON.stringify({ ...clip(snapshot(data), 0, max), phase, seq: seq++, t: Date.now(), kind })}\n`,
|
||||
);
|
||||
writeRecord(out, kind, data, phase, seq++, max);
|
||||
|
||||
write("setup", { ctxKeys: Object.keys(ctx) });
|
||||
|
||||
|
||||
+1
-31
@@ -105,20 +105,6 @@
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"providers": {
|
||||
"default": [],
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"models": {
|
||||
"default": [],
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -126,9 +112,7 @@
|
||||
"commands",
|
||||
"agents",
|
||||
"skills",
|
||||
"mcp",
|
||||
"providers",
|
||||
"models"
|
||||
"mcp"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
@@ -190,18 +174,6 @@
|
||||
"default": true,
|
||||
"type": "boolean"
|
||||
},
|
||||
"window": {
|
||||
"default": 5,
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 9007199254740991
|
||||
},
|
||||
"threshold": {
|
||||
"default": 0.9,
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1
|
||||
},
|
||||
"bootstrap": {
|
||||
"default": "fail",
|
||||
"type": "string",
|
||||
@@ -222,8 +194,6 @@
|
||||
},
|
||||
"required": [
|
||||
"enabled",
|
||||
"window",
|
||||
"threshold",
|
||||
"bootstrap",
|
||||
"artifact"
|
||||
],
|
||||
|
||||
@@ -82,11 +82,7 @@
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"judgeReply": {
|
||||
"type": "string"
|
||||
},
|
||||
"timeoutMs": {
|
||||
"default": 120000,
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 9007199254740991
|
||||
@@ -653,7 +649,6 @@
|
||||
"id",
|
||||
"mode",
|
||||
"prompt",
|
||||
"timeoutMs",
|
||||
"expect"
|
||||
],
|
||||
"additionalProperties": false
|
||||
|
||||
+80
-7
@@ -2,8 +2,8 @@
|
||||
// Every failed assertion becomes a human-readable string ([] = pass).
|
||||
// Fail closed: missing records, missing files, and unparseable bodies are
|
||||
// failures, never silent passes.
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { resolve, sep } from "node:path";
|
||||
import { existsSync, readFileSync, realpathSync } from "node:fs";
|
||||
import { basename, dirname, join, resolve, sep } from "node:path";
|
||||
import type { Matcher, ScenarioExpect } from "../config/schema.ts";
|
||||
import { errors, textOf, toolUses } from "../opencode/events.ts";
|
||||
import type { ProbeLog, ProbeRecord } from "../probe/read.ts";
|
||||
@@ -143,6 +143,27 @@ function requestToolNames(body: unknown): string[] {
|
||||
return [...names];
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical tool name for comparisons. Hooks record a namespaced tool under
|
||||
* its effective name `<namespace>_<name>` (dots in the namespace become
|
||||
* underscores), while catalogs and `tool_use` inner calls use dots
|
||||
* (`tools.ns.my_tool`, `ns.my_tool`). Joining the segments with `_` maps all
|
||||
* of those to the effective name.
|
||||
*/
|
||||
export function normalizeToolName(name: string): string {
|
||||
const segments =
|
||||
name.startsWith("tools.") || name.startsWith("tools[")
|
||||
? pathSegments(name)
|
||||
: name.split(".");
|
||||
return segments.join("_");
|
||||
}
|
||||
|
||||
/** Recorded names that end in `_<wanted>`: likely the namespaced form the author meant. */
|
||||
function namespacedHints(recorded: string[], wanted: string): string[] {
|
||||
const suffix = `_${normalizeToolName(wanted)}`;
|
||||
return [...new Set(recorded.filter((name) => name.endsWith(suffix)))];
|
||||
}
|
||||
|
||||
const HOOK_KINDS: Record<string, string> = {
|
||||
"tool.execute.before": "tool.before",
|
||||
"tool.execute.after": "tool.after",
|
||||
@@ -156,13 +177,23 @@ function evaluateTools(expect: ScenarioExpect, ev: Evidence): string[] {
|
||||
const wanted = expect.tools ?? [];
|
||||
wanted.forEach((tool, index) => {
|
||||
const label = `tools[${index}].${tool.name}`;
|
||||
const calls = ev.probe
|
||||
.toolCalls("post")
|
||||
.filter((call) => call.tool === tool.name);
|
||||
const recorded = ev.probe.toolCalls("post");
|
||||
const calls = recorded.filter(
|
||||
(call) => normalizeToolName(call.tool) === normalizeToolName(tool.name),
|
||||
);
|
||||
failures.push(
|
||||
...check(calls.length, { count: tool.count ?? { gte: 1 } }, label),
|
||||
);
|
||||
if (calls.length === 0) {
|
||||
const hints = namespacedHints(
|
||||
recorded.map((call) => call.tool),
|
||||
tool.name,
|
||||
);
|
||||
if (hints.length > 0) {
|
||||
failures.push(
|
||||
`${label}: namespaced tools are recorded as <namespace>_<name>; did you mean ${hints.join(" or ")}?`,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const perCall = calls.map((call) => {
|
||||
@@ -196,7 +227,12 @@ function evaluateTools(expect: ScenarioExpect, ev: Evidence): string[] {
|
||||
use.tool,
|
||||
...use.innerCalls.map((inner) => inner.tool),
|
||||
]);
|
||||
if (probeNames.includes(name) || eventNames.includes(name)) {
|
||||
const wanted = normalizeToolName(name);
|
||||
if (
|
||||
[...probeNames, ...eventNames].some(
|
||||
(called) => normalizeToolName(called) === wanted,
|
||||
)
|
||||
) {
|
||||
failures.push(
|
||||
`noTools: expected tool ${JSON.stringify(name)} not to be called, but it was.`,
|
||||
);
|
||||
@@ -212,8 +248,9 @@ function evaluateHooks(expect: ScenarioExpect, ev: Evidence): string[] {
|
||||
const label = `hooks[${index}].${hook.hook}@${hook.phase}`;
|
||||
let candidates = ev.probe.byKind(kind, hook.phase);
|
||||
if (hook.tool !== undefined) {
|
||||
const wanted = normalizeToolName(hook.tool);
|
||||
candidates = candidates.filter(
|
||||
(record) => String(record.tool ?? "") === hook.tool,
|
||||
(record) => normalizeToolName(String(record.tool ?? "")) === wanted,
|
||||
);
|
||||
}
|
||||
if (candidates.length === 0) {
|
||||
@@ -325,6 +362,42 @@ function evaluateFiles(expect: ScenarioExpect, ev: Evidence): string[] {
|
||||
failures.push(`${label}: path escapes the project dir.`);
|
||||
return;
|
||||
}
|
||||
// OCI-2.5-files-symlink: the lexical check above is blind to symlinks
|
||||
// (e.g. `linketc -> /etc` makes `linketc/passwd` read host files into
|
||||
// `got` failure strings). Resolve the real path — the file itself when
|
||||
// present, else its parent plus the final segment — and re-check
|
||||
// containment. Fail closed on any realpath error.
|
||||
try {
|
||||
const baseReal = realpathSync(base);
|
||||
// Nearest existing ancestor (the file itself when present), so
|
||||
// assertions on not-yet-created files still resolve; then re-append
|
||||
// the missing tail segments.
|
||||
let cur = absolute;
|
||||
const tail: string[] = [];
|
||||
for (;;) {
|
||||
if (existsSync(cur)) {
|
||||
break;
|
||||
}
|
||||
const parent = dirname(cur);
|
||||
if (parent === cur) {
|
||||
break;
|
||||
}
|
||||
tail.unshift(basename(cur));
|
||||
cur = parent;
|
||||
}
|
||||
const resolved = join(realpathSync(cur), ...tail);
|
||||
if (resolved !== baseReal && !resolved.startsWith(baseReal + sep)) {
|
||||
failures.push(
|
||||
`${label}: path escapes the project dir (symlink target outside).`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
failures.push(
|
||||
`${label}: path escapes the project dir (could not resolve real path).`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const present = existsSync(absolute);
|
||||
if (file.exists !== undefined) {
|
||||
if (file.exists && !present) {
|
||||
|
||||
@@ -89,6 +89,15 @@ export function check(
|
||||
matcher: Matcher,
|
||||
label: string,
|
||||
): string[] {
|
||||
// A field that was never recorded fails every matcher except
|
||||
// `exists: false`. Otherwise a typo in a field name (`comand`) would let
|
||||
// `notContains`/`notMatches` pass vacuously.
|
||||
if (value === undefined && matcher.exists !== true) {
|
||||
return matcher.exists === false
|
||||
? []
|
||||
: [`${label}: no value was recorded (check the field name).`];
|
||||
}
|
||||
|
||||
const failures: string[] = [];
|
||||
|
||||
if (matcher.equals !== undefined && !deepEqual(value, matcher.equals)) {
|
||||
|
||||
+64
-9
@@ -4,6 +4,7 @@
|
||||
// opencode-plugin-ci validate
|
||||
// opencode-plugin-ci run [--tier load|scripted|live]... [--scenario id]...
|
||||
// [--opencode v]... [--out dir] [--keep] [--gate]
|
||||
// [--gate-artifact name]
|
||||
// [--verbose] [--quiet]
|
||||
// opencode-plugin-ci report [<results.json>]
|
||||
// opencode-plugin-ci --version | --help
|
||||
@@ -19,7 +20,8 @@ import {
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { ConfigError, loadAll } from "./config/load.ts";
|
||||
import { ensureEntriesOrThrow } from "./plugin/entry.ts";
|
||||
import { versionSchema } from "./config/schema.ts";
|
||||
import { type Entries, ensureEntriesOrThrow } from "./plugin/entry.ts";
|
||||
import {
|
||||
appendStepSummary,
|
||||
deriveAllowedVersions,
|
||||
@@ -35,7 +37,7 @@ const USAGE = `opencode-plugin-ci
|
||||
Usage:
|
||||
opencode-plugin-ci init [--force]
|
||||
opencode-plugin-ci validate
|
||||
opencode-plugin-ci run [--tier load|scripted|live] [--scenario <id>] [--opencode <v>] [--out <dir>] [--keep] [--gate] [--verbose] [--quiet]
|
||||
opencode-plugin-ci run [--tier load|scripted|live] [--scenario <id>] [--opencode <v>] [--out <dir>] [--keep] [--gate] [--gate-artifact <name>] [--verbose] [--quiet]
|
||||
opencode-plugin-ci report [<results.json>]
|
||||
opencode-plugin-ci --version
|
||||
opencode-plugin-ci --help
|
||||
@@ -81,22 +83,24 @@ function collectValues(raw: string[]): string[] {
|
||||
return out;
|
||||
}
|
||||
|
||||
interface RunFlags {
|
||||
export interface RunFlags {
|
||||
tiers: TierName[];
|
||||
scenarios: string[];
|
||||
opencodes: string[];
|
||||
out?: string;
|
||||
keep: boolean;
|
||||
gate: boolean;
|
||||
gateArtifact?: string;
|
||||
verbose: boolean;
|
||||
quiet: boolean;
|
||||
}
|
||||
|
||||
function parseRunArgs(args: string[]): RunFlags {
|
||||
export function parseRunArgs(args: string[]): RunFlags {
|
||||
const tiers: TierName[] = [];
|
||||
const scenarios: string[] = [];
|
||||
const opencodes: string[] = [];
|
||||
let out: string | undefined;
|
||||
let gateArtifact: string | undefined;
|
||||
let keep = false;
|
||||
let gate = false;
|
||||
let verbose = false;
|
||||
@@ -136,7 +140,16 @@ function parseRunArgs(args: string[]): RunFlags {
|
||||
if (arg === "--opencode") {
|
||||
i++;
|
||||
}
|
||||
opencodes.push(...collectValues([raw]));
|
||||
for (const version of collectValues([raw])) {
|
||||
// The version becomes an evidence path segment, so it must
|
||||
// never contain path syntax: same rule as opencode.versions.
|
||||
if (!versionSchema.safeParse(version).success) {
|
||||
throw new UsageError(
|
||||
`Invalid --opencode ${JSON.stringify(version)}: expected x.y.z or "latest".`,
|
||||
);
|
||||
}
|
||||
opencodes.push(version);
|
||||
}
|
||||
} else if (arg === "--out" || arg.startsWith("--out=")) {
|
||||
out =
|
||||
arg === "--out"
|
||||
@@ -145,6 +158,17 @@ function parseRunArgs(args: string[]): RunFlags {
|
||||
if (arg === "--out") {
|
||||
i++;
|
||||
}
|
||||
} else if (
|
||||
arg === "--gate-artifact" ||
|
||||
arg.startsWith("--gate-artifact=")
|
||||
) {
|
||||
gateArtifact =
|
||||
arg === "--gate-artifact"
|
||||
? takeValue(args, i, "--gate-artifact")
|
||||
: arg.slice("--gate-artifact=".length);
|
||||
if (arg === "--gate-artifact") {
|
||||
i++;
|
||||
}
|
||||
} else if (arg === "--keep") {
|
||||
keep = true;
|
||||
} else if (arg === "--gate") {
|
||||
@@ -160,7 +184,17 @@ function parseRunArgs(args: string[]): RunFlags {
|
||||
throw new UsageError(`Unknown argument for run: ${arg}`);
|
||||
}
|
||||
}
|
||||
return { tiers, scenarios, opencodes, out, keep, gate, verbose, quiet };
|
||||
return {
|
||||
tiers,
|
||||
scenarios,
|
||||
opencodes,
|
||||
out,
|
||||
keep,
|
||||
gate,
|
||||
gateArtifact,
|
||||
verbose,
|
||||
quiet,
|
||||
};
|
||||
}
|
||||
|
||||
function exampleSourceDir(): string {
|
||||
@@ -237,15 +271,32 @@ async function cmdValidate(): Promise<number> {
|
||||
logger.warn(warning);
|
||||
}
|
||||
const pluginPath = resolve(cwd, config.plugin.path);
|
||||
const entries = ensureEntriesOrThrow(pluginPath, config.plugin.entry);
|
||||
const tuiOnly = isTuiOnly(config, entries);
|
||||
let entries: Entries | undefined;
|
||||
try {
|
||||
entries = ensureEntriesOrThrow(pluginPath, config.plugin.entry);
|
||||
} catch (error) {
|
||||
// The entry may be a build output that `validate` never builds;
|
||||
// `run` resolves entries again after plugin.build.
|
||||
if (
|
||||
!(error instanceof ConfigError) ||
|
||||
config.plugin.build === undefined
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
logger.warn(
|
||||
`${error.message} (plugin.build is set, so run checks again after building).`,
|
||||
);
|
||||
}
|
||||
const tuiOnly = entries !== undefined && isTuiOnly(config, entries);
|
||||
const byMode = new Map<string, number>();
|
||||
for (const scenario of scenarios) {
|
||||
byMode.set(scenario.mode, (byMode.get(scenario.mode) ?? 0) + 1);
|
||||
}
|
||||
console.log(`plugin: ${pluginPath}${tuiOnly ? " (TUI-only)" : ""}`);
|
||||
console.log(
|
||||
`entries: server=${entries.server ?? "-"} tui=${entries.tui ?? "-"}`,
|
||||
entries === undefined
|
||||
? "entries: not found yet (created by plugin.build?)"
|
||||
: `entries: server=${entries.server ?? "-"} tui=${entries.tui ?? "-"}`,
|
||||
);
|
||||
console.log(`opencode versions: ${config.opencode.versions.join(", ")}`);
|
||||
console.log(
|
||||
@@ -284,6 +335,7 @@ async function cmdRun(args: string[]): Promise<number> {
|
||||
out: flags.out,
|
||||
keep: flags.keep,
|
||||
gate: flags.gate,
|
||||
gateArtifact: flags.gateArtifact,
|
||||
verbose: flags.verbose,
|
||||
quiet: flags.quiet,
|
||||
});
|
||||
@@ -347,6 +399,9 @@ async function cmdReport(args: string[]): Promise<number> {
|
||||
);
|
||||
try {
|
||||
const results = readResultsFile(resultsPath);
|
||||
// `report` re-renders saved results and never loads live keys, so no
|
||||
// scrubbing happens here: values were already scrubbed at `run` time
|
||||
// before `results.json` was written (see scrubRow in run.ts).
|
||||
const markdown = renderMarkdown(results, {
|
||||
allowedVersions: deriveAllowedVersions(results),
|
||||
});
|
||||
|
||||
@@ -98,9 +98,6 @@ export function loadConfig(cwd: string = process.cwd()): {
|
||||
const warnings = [
|
||||
`No ${CONFIG_YAML} or ${CONFIG_JSON} found under ${cwd}: using defaults.`,
|
||||
];
|
||||
for (const w of warnings) {
|
||||
console.warn(w);
|
||||
}
|
||||
return { config: ConfigSchema.parse({}), warnings };
|
||||
}
|
||||
const configPath = hasYaml ? yamlPath : jsonPath;
|
||||
|
||||
+28
-18
@@ -1,11 +1,15 @@
|
||||
// Config + scenario schemas (zod v4). Every object is `.strict()` so typos
|
||||
// fail validation instead of silently passing (SPEC 5.1, 5.2).
|
||||
import { isAbsolute } from "node:path";
|
||||
import { z } from "zod";
|
||||
import { DEFAULT_RESULTS_ARTIFACT } from "../gate/history.ts";
|
||||
|
||||
const VERSION_RE = /^\d+\.\d+\.\d+$/;
|
||||
const SCENARIO_ID_RE = /^[a-z0-9][a-z0-9-]*$/;
|
||||
// Case ids the runner itself uses: tier 1 ("load") and the escape hatch.
|
||||
const RESERVED_IDS = ["load", "bun-test"];
|
||||
|
||||
const versionSchema = z.union([
|
||||
export const versionSchema = z.union([
|
||||
z.literal("latest"),
|
||||
z.string().regex(VERSION_RE, 'must be semver (x.y.z) or "latest"'),
|
||||
]);
|
||||
@@ -38,8 +42,6 @@ const configExpectSchema = z
|
||||
agents: stringArray(),
|
||||
skills: stringArray(),
|
||||
mcp: stringArray(),
|
||||
providers: stringArray(),
|
||||
models: stringArray(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
@@ -57,14 +59,12 @@ const liveConfigSchema = z
|
||||
const gateConfigSchema = z
|
||||
.object({
|
||||
enabled: z.boolean().default(true),
|
||||
window: z.number().int().positive().default(5),
|
||||
threshold: z.number().min(0).max(1).default(0.9),
|
||||
bootstrap: z.enum(["fail", "allow"]).default("fail"),
|
||||
workflow: z.string().min(1).optional(),
|
||||
// Artifact name the gate downloads history from. Must match the
|
||||
// `artifact-name` input of the GitHub Action (M13); matrix users
|
||||
// set a distinct name per leg here and there.
|
||||
artifact: z.string().min(1).default("opencode-plugin-ci-results"),
|
||||
// Artifact name the gate downloads history from. `run --gate-artifact`
|
||||
// overrides it; the GitHub Action passes its `artifact-name` input
|
||||
// that way, so matrix legs never need per-leg config.
|
||||
artifact: z.string().min(1).default(DEFAULT_RESULTS_ARTIFACT),
|
||||
})
|
||||
.strict();
|
||||
|
||||
@@ -226,14 +226,31 @@ export const ScenarioSchema = z
|
||||
workspace: z.record(z.string(), z.string()).optional(),
|
||||
prompt: z.string().min(1),
|
||||
script: z.array(stepSchema).optional(),
|
||||
judgeReply: z.string().optional(),
|
||||
timeoutMs: z.number().int().positive().default(120_000),
|
||||
// No default here: scripted runs fall back to 120000 and live runs to
|
||||
// config.live.timeoutMs, so an unset value must stay distinguishable.
|
||||
timeoutMs: z.number().int().positive().optional(),
|
||||
expect: scenarioExpectSchema.prefault({}),
|
||||
judge: judgeSchema.optional(),
|
||||
})
|
||||
.strict()
|
||||
.superRefine((val, ctx) => {
|
||||
const tag = `scenario "${val.id}"`;
|
||||
if (RESERVED_IDS.includes(val.id)) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
path: ["id"],
|
||||
message: `${tag}: id "${val.id}" is reserved (the tool uses it for its own case and evidence dir)`,
|
||||
});
|
||||
}
|
||||
for (const key of Object.keys(val.workspace ?? {})) {
|
||||
if (isAbsolute(key) || key.split(/[\\/]/).includes("..")) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
path: ["workspace", key],
|
||||
message: `${tag}: workspace path "${key}" must be relative and stay inside the project dir`,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (val.mode === "scripted" && val.script === undefined) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
@@ -262,13 +279,6 @@ export const ScenarioSchema = z
|
||||
message: `${tag}: field "judge" is live-only and is not allowed when mode is "scripted"`,
|
||||
});
|
||||
}
|
||||
if (val.mode === "live" && val.judgeReply !== undefined) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
path: ["judgeReply"],
|
||||
message: `${tag}: field "judgeReply" is scripted-only and is not allowed when mode is "live"`,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
export type Scenario = z.infer<typeof ScenarioSchema>;
|
||||
+2
-1
@@ -11,6 +11,7 @@ import { createLogger } from "../util/log.ts";
|
||||
import {
|
||||
fetchHistory,
|
||||
gatePrerequisites,
|
||||
historyBranch,
|
||||
resolveWorkflowName,
|
||||
} from "./history.ts";
|
||||
import {
|
||||
@@ -201,7 +202,7 @@ export async function runGate(ctx: GateContext): Promise<GateOutcome> {
|
||||
}
|
||||
|
||||
const repo = env.GITHUB_REPOSITORY;
|
||||
const branch = env.GITHUB_REF_NAME;
|
||||
const branch = historyBranch(env);
|
||||
if (
|
||||
repo === undefined ||
|
||||
repo.length === 0 ||
|
||||
|
||||
+32
-1
@@ -5,7 +5,7 @@
|
||||
// entry per sampled run as a `GateHistoryRun` (the verbatim verdict input)
|
||||
// instead of an inline tally. Every external call goes through the runner
|
||||
// so unit tests never touch the network.
|
||||
import { mkdtempSync, readFileSync } from "node:fs";
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { GateHistoryRun } from "./verdict.ts";
|
||||
@@ -76,6 +76,22 @@ export function parseWorkflowName(ref: string | undefined): string | undefined {
|
||||
return file;
|
||||
}
|
||||
|
||||
/**
|
||||
* Branch whose runs form the gate history. On pull_request runs
|
||||
* GITHUB_REF_NAME is "<n>/merge", which no run is ever filed under (gh lists
|
||||
* runs by head branch), so PR runs read the target branch's history from
|
||||
* GITHUB_BASE_REF, which GitHub sets only on pull request events.
|
||||
*/
|
||||
export function historyBranch(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): string | undefined {
|
||||
const base = env.GITHUB_BASE_REF;
|
||||
if (base !== undefined && base.length > 0) {
|
||||
return base;
|
||||
}
|
||||
return env.GITHUB_REF_NAME;
|
||||
}
|
||||
|
||||
/** Resolve the workflow file name: explicit config wins, else GITHUB_WORKFLOW_REF. */
|
||||
export function resolveWorkflowName(
|
||||
configured: string | undefined,
|
||||
@@ -154,6 +170,21 @@ export async function fetchHistory(
|
||||
|
||||
const tmpRoot =
|
||||
opts.tmpDir ?? mkdtempSync(join(tmpdir(), "opencode-ci-gate-"));
|
||||
try {
|
||||
return await downloadHistory(opts, run, runs, tmpRoot);
|
||||
} finally {
|
||||
if (opts.tmpDir === undefined) {
|
||||
rmSync(tmpRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadHistory(
|
||||
opts: FetchHistoryOptions,
|
||||
run: CommandRunner,
|
||||
runs: { id: number; createdAt: string }[],
|
||||
tmpRoot: string,
|
||||
): Promise<GateHistoryRun[]> {
|
||||
const history: GateHistoryRun[] = [];
|
||||
for (const entry of runs) {
|
||||
const dir = join(tmpRoot, String(entry.id));
|
||||
|
||||
+12
-9
@@ -157,9 +157,10 @@ const TUI_FILES = ["tui.ts", "tui.js", "tui.mjs"];
|
||||
|
||||
/**
|
||||
* Resolve absolute server/tui entry paths. An explicit entry is joined to
|
||||
* pluginPath; when it names an existing file it wins, otherwise resolution
|
||||
* falls through to the default order (so a typo surfaces as "no entry"
|
||||
* instead of a crash). Paths that resolve to nothing are left undefined.
|
||||
* pluginPath and must name an existing file: a missing explicit entry throws
|
||||
* ConfigError instead of silently falling back to auto-detection (a typo
|
||||
* must never test a different file). Other paths that resolve to nothing are
|
||||
* left undefined.
|
||||
*/
|
||||
export function resolveEntries(pluginPath: string, explicit?: string): Entries {
|
||||
const entries: Entries = {};
|
||||
@@ -168,17 +169,19 @@ export function resolveEntries(pluginPath: string, explicit?: string): Entries {
|
||||
|
||||
if (explicit !== undefined && explicit.length > 0) {
|
||||
const candidate = abs(explicit);
|
||||
if (isFile(candidate)) {
|
||||
entries.server = candidate;
|
||||
return entries;
|
||||
if (!isFile(candidate)) {
|
||||
throw new ConfigError(
|
||||
`plugin.entry ${JSON.stringify(explicit)} does not exist (resolved to ${candidate}). ` +
|
||||
"Fix the path, or check that plugin.build creates it.",
|
||||
);
|
||||
}
|
||||
// Fall through: the explicit path names nothing, so keep looking
|
||||
// and let the caller report "no server entry" with full context.
|
||||
entries.server = candidate;
|
||||
}
|
||||
|
||||
const pkg = readPackageJson(pluginPath);
|
||||
|
||||
const serverTarget = serverFromExports(pkg);
|
||||
const serverTarget =
|
||||
entries.server === undefined ? serverFromExports(pkg) : undefined;
|
||||
if (serverTarget !== undefined) {
|
||||
const candidate = joinTarget(pluginPath, serverTarget);
|
||||
if (isFile(candidate)) {
|
||||
|
||||
+12
-5
@@ -1,12 +1,19 @@
|
||||
// Run a plugin's install/build shell step before any tier runs.
|
||||
import { type ChildProcess, spawn } from "node:child_process";
|
||||
import { ConfigError } from "../config/load.ts";
|
||||
|
||||
/** A plugin install/build command failed: the plugin's failure (exit 1), not a config error. */
|
||||
export class StepError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "StepError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `command` in `cwd` via the platform shell (bash -lc on POSIX,
|
||||
* cmd /c on Windows) with async spawn, streaming the child's output to
|
||||
* our own stdout/stderr. Never sync-spawn: the stub model server lives in
|
||||
* this process during tier runs. Throws ConfigError (exit 2 upstream) when
|
||||
* this process during tier runs. Throws StepError (exit 1 upstream) when
|
||||
* the command fails.
|
||||
*/
|
||||
export function runStep(
|
||||
@@ -26,7 +33,7 @@ export function runStep(
|
||||
});
|
||||
} catch (error) {
|
||||
reject(
|
||||
new ConfigError(
|
||||
new StepError(
|
||||
`${name} step failed to start (${command}): ${String(error)}`,
|
||||
),
|
||||
);
|
||||
@@ -40,7 +47,7 @@ export function runStep(
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
reject(
|
||||
new ConfigError(`${name} step failed to start (${command}): ${error}`),
|
||||
new StepError(`${name} step failed to start (${command}): ${error}`),
|
||||
);
|
||||
});
|
||||
child.on("close", (code, signal) => {
|
||||
@@ -52,7 +59,7 @@ export function runStep(
|
||||
signal !== null && signal !== undefined
|
||||
? `signal ${signal}`
|
||||
: `exit code ${code ?? "unknown"}`;
|
||||
reject(new ConfigError(`${name} step failed (${command}): ${detail}.`));
|
||||
reject(new StepError(`${name} step failed (${command}): ${detail}.`));
|
||||
});
|
||||
});
|
||||
}
|
||||
+104
-28
@@ -1,7 +1,8 @@
|
||||
// Orchestrator for `opencode-plugin-ci run` (M10 order):
|
||||
// 1. Load + validate config/scenarios.
|
||||
// 2. Resolve entries (exit 2 when neither server nor TUI exists).
|
||||
// 3. Run plugin.install / plugin.build once.
|
||||
// Orchestrator for `opencode-plugin-ci run`:
|
||||
// 1. Load + validate config/scenarios (and every --scenario id).
|
||||
// 2. Run plugin.install / plugin.build once (a failure is exit 1).
|
||||
// 3. Resolve entries (exit 2 when neither server nor TUI exists). This must
|
||||
// follow the build: the entry is often a build output such as dist/.
|
||||
// 4. For each opencode version: resolve binary, tier 1 (or static tier 1
|
||||
// for TUI-only plugins), then tier 2 (scripted), then tier 3 (live,
|
||||
// sequential with a shared run-wide budget).
|
||||
@@ -14,7 +15,7 @@
|
||||
// failure. Results on non-first versions are allowed failures.
|
||||
import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join, relative, resolve } from "node:path";
|
||||
import { loadAll } from "./config/load.ts";
|
||||
import { ConfigError, loadAll } from "./config/load.ts";
|
||||
import type { Config, Scenario } from "./config/schema.ts";
|
||||
import { runGate } from "./gate/gate.ts";
|
||||
import { resolveOpencode } from "./opencode/binary.ts";
|
||||
@@ -35,9 +36,27 @@ import { concurrencyLimit, pool, runScenario } from "./tiers/scripted.ts";
|
||||
import { runUiStatic } from "./tiers/ui-static.ts";
|
||||
import { appendFile, ensureDir } from "./util/fs.ts";
|
||||
import { createLogger, type Logger } from "./util/log.ts";
|
||||
import { collectSecretValues, redactSecretValues } from "./util/redact.ts";
|
||||
|
||||
export type TierName = "load" | "scripted" | "live";
|
||||
|
||||
/**
|
||||
* Split the opencode version matrix into the required first version and the
|
||||
* allowed-to-fail rest (OCI-6.1-allowed-wiring). Extracted pure so the wiring
|
||||
* at the `runCi` call site is pinned: dropping the `allowedVersions`
|
||||
* argument to the report renderer must go red here instead of silently
|
||||
* rendering allowed legs as required.
|
||||
*/
|
||||
export function partitionVersions(versions: string[]): {
|
||||
required: string;
|
||||
allowedVersions: string[];
|
||||
} {
|
||||
return {
|
||||
required: versions[0] as string,
|
||||
allowedVersions: versions.slice(1),
|
||||
};
|
||||
}
|
||||
|
||||
export interface RunOptions {
|
||||
cwd?: string;
|
||||
tiers?: TierName[];
|
||||
@@ -46,6 +65,8 @@ export interface RunOptions {
|
||||
out?: string;
|
||||
keep?: boolean;
|
||||
gate?: boolean;
|
||||
/** Overrides gate.artifact; the Action passes its artifact-name input. */
|
||||
gateArtifact?: string;
|
||||
verbose?: boolean;
|
||||
quiet?: boolean;
|
||||
}
|
||||
@@ -151,6 +172,32 @@ function skippedTuiOnly(scenario: Scenario, version: string): CaseResult {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace known live secret values in display strings before persisting
|
||||
* (OCI-7.3-evidence-leaks). Outcomes (`pass`, `skipped`, counts) and the
|
||||
* evidence dirs are untouched, so the gate and exit codes behave identically.
|
||||
*/
|
||||
function scrubRow(row: CaseResult, secrets: string[]): CaseResult {
|
||||
if (secrets.length === 0) {
|
||||
return row;
|
||||
}
|
||||
const scrub = (text: string): string => redactSecretValues(text, secrets);
|
||||
return {
|
||||
...row,
|
||||
failures: row.failures?.map(scrub),
|
||||
textExcerpt:
|
||||
row.textExcerpt === undefined ? undefined : scrub(row.textExcerpt),
|
||||
attempts: row.attempts?.map((attempt) => ({
|
||||
...attempt,
|
||||
failures: attempt.failures.map(scrub),
|
||||
})),
|
||||
judge:
|
||||
row.judge === undefined
|
||||
? undefined
|
||||
: { ...row.judge, reason: scrub(row.judge.reason) },
|
||||
};
|
||||
}
|
||||
|
||||
export async function runCi(opts?: RunOptions): Promise<RunOutcome> {
|
||||
const cwd = resolve(opts?.cwd ?? process.cwd());
|
||||
const logger: Logger = createLogger({
|
||||
@@ -166,15 +213,18 @@ export async function runCi(opts?: RunOptions): Promise<RunOutcome> {
|
||||
logger.warn(warning);
|
||||
}
|
||||
|
||||
const pluginPath = resolve(cwd, config.plugin.path);
|
||||
// 2. Resolve entries (throws ConfigError when nothing resolves).
|
||||
const entries = ensureEntriesOrThrow(pluginPath, config.plugin.entry);
|
||||
const tuiOnly = isTuiOnly(config, entries);
|
||||
if (tuiOnly) {
|
||||
logger.info("TUI-only plugin: tier 1 is static; tiers 2/3 are skipped.");
|
||||
const unknown = (wantedScenarios ?? []).filter(
|
||||
(id) => !scenarios.some((scenario) => scenario.id === id),
|
||||
);
|
||||
if (unknown.length > 0) {
|
||||
throw new ConfigError(
|
||||
`unknown --scenario id(s): ${unknown.join(", ")} (known: ${scenarios.map((s) => s.id).join(", ") || "none"}).`,
|
||||
);
|
||||
}
|
||||
|
||||
// 3. Install + build once, before any tier.
|
||||
const pluginPath = resolve(cwd, config.plugin.path);
|
||||
// 2. Install + build once, before resolving entries: the entry is often
|
||||
// a build output (dist/server.js) that does not exist in a fresh checkout.
|
||||
if (config.plugin.install !== undefined) {
|
||||
logger.info(`Running install step: ${config.plugin.install}`);
|
||||
await runStep("install", config.plugin.install, pluginPath);
|
||||
@@ -184,12 +234,18 @@ export async function runCi(opts?: RunOptions): Promise<RunOutcome> {
|
||||
await runStep("build", config.plugin.build, pluginPath);
|
||||
}
|
||||
|
||||
// 3. Resolve entries (throws ConfigError when nothing resolves).
|
||||
const entries = ensureEntriesOrThrow(pluginPath, config.plugin.entry);
|
||||
const tuiOnly = isTuiOnly(config, entries);
|
||||
if (tuiOnly) {
|
||||
logger.info("TUI-only plugin: tier 1 is static; tiers 2/3 are skipped.");
|
||||
}
|
||||
|
||||
const versions =
|
||||
opts?.opencodes !== undefined && opts.opencodes.length > 0
|
||||
? opts.opencodes
|
||||
: config.opencode.versions;
|
||||
const required = versions[0] as string;
|
||||
const allowedVersions = versions.slice(1);
|
||||
const { required, allowedVersions } = partitionVersions(versions);
|
||||
const outDir = resolve(opts?.out ?? join(cwd, ".opencode-ci/out"));
|
||||
|
||||
if (opts?.keep === true) {
|
||||
@@ -205,6 +261,9 @@ export async function runCi(opts?: RunOptions): Promise<RunOutcome> {
|
||||
wantedScenarios.includes(scenario.id);
|
||||
|
||||
const results: CaseResult[] = [];
|
||||
// One budget for the whole live tier across every opencode version
|
||||
// (SPEC 5.1: "hard stop for the whole live tier").
|
||||
const budget: LiveBudget = { spent: 0, limit: config.live.budgetUSD };
|
||||
|
||||
// 4. One pass per opencode version.
|
||||
for (const version of versions) {
|
||||
@@ -222,6 +281,7 @@ export async function runCi(opts?: RunOptions): Promise<RunOutcome> {
|
||||
? await runUiStatic({ pluginPath, outDir, version })
|
||||
: await runLoad({
|
||||
pluginPath,
|
||||
serverEntry: entries.server as string,
|
||||
pluginOptions: config.plugin.options,
|
||||
opencodeBin: bin,
|
||||
outDir,
|
||||
@@ -297,7 +357,6 @@ export async function runCi(opts?: RunOptions): Promise<RunOutcome> {
|
||||
// counter, so parallel runs would race the accounting.
|
||||
const pending = live;
|
||||
if (pending.length > 0) {
|
||||
const budget: LiveBudget = { spent: 0, limit: config.live.budgetUSD };
|
||||
const keys = collectLiveKeys(config);
|
||||
if (Object.keys(keys).length === 0) {
|
||||
const reason = noKeySkipReason(config);
|
||||
@@ -388,13 +447,18 @@ export async function runCi(opts?: RunOptions): Promise<RunOutcome> {
|
||||
logger.info(`Running programmatic tests: ${testFiles.length} file(s)...`);
|
||||
const { bin } = await resolveOpencode(required);
|
||||
const started = Date.now();
|
||||
const proc = Bun.spawn(["bun", "test", ".opencode-ci"], {
|
||||
cwd,
|
||||
env: { ...process.env, OPENCODE_BIN: bin },
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
stdin: "ignore",
|
||||
});
|
||||
// Explicit "./" paths: `bun test <dir>` skips dot-directories, so a
|
||||
// ".opencode-ci" filter never matches any file (bun 1.4.2).
|
||||
const proc = Bun.spawn(
|
||||
["bun", "test", ...testFiles.map((file) => `./${relative(cwd, file)}`)],
|
||||
{
|
||||
cwd,
|
||||
env: { ...process.env, OPENCODE_BIN: bin },
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
stdin: "ignore",
|
||||
},
|
||||
);
|
||||
const [stdout, stderr] = await Promise.all([
|
||||
new Response(proc.stdout as ReadableStream).text(),
|
||||
new Response(proc.stderr as ReadableStream).text(),
|
||||
@@ -424,13 +488,17 @@ export async function runCi(opts?: RunOptions): Promise<RunOutcome> {
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Write results.json + report.md (+ step summary).
|
||||
// 6. Write results.json + report.md (+ step summary). Known live secret
|
||||
// values are scrubbed from display strings before anything is persisted
|
||||
// or summarized; in-memory verdicts were already computed on raw text.
|
||||
const secrets = collectSecretValues(collectLiveKeys(config));
|
||||
const scrubbed = results.map((row) => scrubRow(row, secrets));
|
||||
const resultsFile: ResultsFile = {
|
||||
version: 1,
|
||||
at: new Date().toISOString(),
|
||||
tool: toolVersion(),
|
||||
plugin: pluginDisplayName(pluginPath),
|
||||
results,
|
||||
results: scrubbed,
|
||||
};
|
||||
const resultsPath = join(outDir, "results.json");
|
||||
const reportPath = join(outDir, "report.md");
|
||||
@@ -440,7 +508,10 @@ export async function runCi(opts?: RunOptions): Promise<RunOutcome> {
|
||||
`${JSON.stringify(resultsFile, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
const markdown = renderMarkdown(resultsFile, { allowedVersions });
|
||||
const markdown = redactSecretValues(
|
||||
renderMarkdown(resultsFile, { allowedVersions }),
|
||||
secrets,
|
||||
);
|
||||
writeFileSync(reportPath, markdown, "utf8");
|
||||
appendStepSummary(markdown);
|
||||
logger.info(
|
||||
@@ -449,17 +520,22 @@ export async function runCi(opts?: RunOptions): Promise<RunOutcome> {
|
||||
|
||||
// 7. Gate: when it applies, required-version live failures are decided
|
||||
// by the gate instead of failing directly. Tiers 1 and 2 always fail.
|
||||
const gateConfig: Config =
|
||||
opts?.gateArtifact === undefined
|
||||
? config
|
||||
: { ...config, gate: { ...config.gate, artifact: opts.gateArtifact } };
|
||||
const gateOutcome = await runGate({
|
||||
cwd,
|
||||
config,
|
||||
config: gateConfig,
|
||||
results,
|
||||
requiredVersion: required,
|
||||
gateFlag: opts?.gate === true,
|
||||
logger,
|
||||
});
|
||||
if (gateOutcome.ran) {
|
||||
appendFile(reportPath, gateOutcome.section);
|
||||
appendStepSummary(gateOutcome.section);
|
||||
const section = redactSecretValues(gateOutcome.section, secrets);
|
||||
appendFile(reportPath, section);
|
||||
appendStepSummary(section);
|
||||
logger.info(
|
||||
`Flake gate: exit ${gateOutcome.verdict?.exit ?? "unknown"} (exit ${gateOutcome.exitCode}).`,
|
||||
);
|
||||
|
||||
+31
-2
@@ -5,7 +5,8 @@ import { type ChildProcess, spawn } from "node:child_process";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { dirname, isAbsolute, join, resolve, sep } from "node:path";
|
||||
import { ConfigError } from "../config/load.ts";
|
||||
import { parseEvents, type RunEvent } from "../opencode/events.ts";
|
||||
|
||||
export interface SandboxPlugin {
|
||||
@@ -57,6 +58,29 @@ function stubProvider(baseURL: string): Record<string, unknown> {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a scenario `workspace` key to an absolute path inside `project`.
|
||||
* Mirrors `evaluateFiles` containment in `src/assert/expect.ts`: absolute
|
||||
* paths and any `..` escape of the project dir are a configuration error
|
||||
* (exit 2 upstream), never a silent write outside the sandbox. `Sandbox`
|
||||
* is public API, so this holds even when the schema already rejected the
|
||||
* key: defense in depth at the write site.
|
||||
*/
|
||||
export function safeWorkspaceDest(project: string, rel: string): string {
|
||||
if (isAbsolute(rel)) {
|
||||
throw new ConfigError(
|
||||
`workspace path "${rel}" must be relative and stay inside the project dir.`,
|
||||
);
|
||||
}
|
||||
const dest = resolve(project, rel);
|
||||
if (dest !== project && !dest.startsWith(project + sep)) {
|
||||
throw new ConfigError(
|
||||
`workspace path "${rel}" must be relative and stay inside the project dir.`,
|
||||
);
|
||||
}
|
||||
return dest;
|
||||
}
|
||||
|
||||
function runGitInit(
|
||||
project: string,
|
||||
env: Record<string, string>,
|
||||
@@ -106,7 +130,7 @@ export class Sandbox {
|
||||
|
||||
writeFileSync(join(this.project, "README.md"), "# Sandbox project\n");
|
||||
for (const [rel, content] of Object.entries(opts.workspace ?? {})) {
|
||||
const dest = join(this.project, rel);
|
||||
const dest = safeWorkspaceDest(this.project, rel);
|
||||
mkdirSync(dirname(dest), { recursive: true });
|
||||
writeFileSync(dest, content);
|
||||
}
|
||||
@@ -151,6 +175,11 @@ export class Sandbox {
|
||||
this.probeFile = join(this.root, "probe.jsonl");
|
||||
|
||||
this.ready = runGitInit(this.project, this.env);
|
||||
// The constructor can throw after this (workspace containment),
|
||||
// leaving no Sandbox behind to await `ready`: observe it here so a
|
||||
// late git failure never surfaces as an unhandled rejection. run()
|
||||
// still awaits the original promise and reports the failure.
|
||||
void this.ready.catch(() => undefined);
|
||||
}
|
||||
|
||||
async run(prompt: string, opts?: SandboxRunOptions): Promise<RunOutput> {
|
||||
|
||||
@@ -8,9 +8,8 @@
|
||||
|
||||
/** Every `tools.…(` call path listed in a Code Mode catalog. */
|
||||
export function catalogPaths(text: string): string[] {
|
||||
// NOTE: the capture group below includes the trailing "(" (the `\(` sits
|
||||
// inside the group in the spec regex). Strip it so the result is a usable
|
||||
// call path such as `tools.spike_echo` rather than `tools.spike_echo(`.
|
||||
// The capture group includes the trailing "(", which is stripped so the
|
||||
// result is a usable call path such as `tools.spike_echo`.
|
||||
const re =
|
||||
/^\s*-\s+(tools(?:\.[A-Za-z_$][\w$]*|\[(?:"[^"]+"|'[^']+')\])+\()/gm;
|
||||
return [...text.matchAll(re)].flatMap((m) =>
|
||||
|
||||
+15
-1
@@ -15,6 +15,7 @@ import {
|
||||
type Server,
|
||||
type ServerResponse,
|
||||
} from "node:http";
|
||||
import { redactSecretValues } from "../util/redact.ts";
|
||||
import { allMessageText, findCatalogPath } from "./codemode.ts";
|
||||
|
||||
export interface ToolCall {
|
||||
@@ -44,6 +45,13 @@ export interface StubOptions {
|
||||
script: Step[];
|
||||
judgeReply?: string;
|
||||
recordFile?: string;
|
||||
/**
|
||||
* Live secret values to scrub from the `recordFile` JSONL (defense in
|
||||
* depth: a prompt that echoes a key would otherwise persist it into the
|
||||
* uploaded evidence). In-memory `requests` stay raw so assertions still
|
||||
* see — and fail on — leaked values.
|
||||
*/
|
||||
redactValues?: string[];
|
||||
}
|
||||
|
||||
export interface Stub {
|
||||
@@ -278,7 +286,13 @@ export async function startStub(opts: StubOptions): Promise<Stub> {
|
||||
requests.push(full);
|
||||
if (opts.recordFile) {
|
||||
try {
|
||||
appendFileSync(opts.recordFile, `${JSON.stringify(full)}\n`);
|
||||
const line = JSON.stringify(full);
|
||||
appendFileSync(
|
||||
opts.recordFile,
|
||||
opts.redactValues === undefined
|
||||
? `${line}\n`
|
||||
: `${redactSecretValues(line, opts.redactValues)}\n`,
|
||||
);
|
||||
} catch {
|
||||
/* ignore record errors: stub replies must never fail */
|
||||
}
|
||||
|
||||
+2
-1
@@ -190,7 +190,8 @@ export async function runJudge(opts: RunJudgeOptions): Promise<JudgeOutcome> {
|
||||
if (process.env.OPENCODE_PLUGIN_CI_JUDGE_VIA_STUB === "1") {
|
||||
const stub = await startStub({
|
||||
script: [],
|
||||
judgeReply: opts.stubReply ?? '{"pass":true,"reason":"stubbed"}',
|
||||
judgeReply:
|
||||
opts.stubReply ?? '{"pass":false,"reason":"no stubReply given"}',
|
||||
});
|
||||
const sandbox = new Sandbox({
|
||||
opencodeBin: opts.opencodeBin,
|
||||
|
||||
+30
-10
@@ -14,6 +14,8 @@ import { writeShim } from "../plugin/shim.ts";
|
||||
import { type ProbeLog, probePluginDirs, readProbe } from "../probe/read.ts";
|
||||
import type { CaseResult } from "../report/results.ts";
|
||||
import { type RunOutput, Sandbox } from "../sandbox/sandbox.ts";
|
||||
import { resetDir } from "../util/fs.ts";
|
||||
import { collectSecretValues, redactSecretValues } from "../util/redact.ts";
|
||||
import { runJudge } from "./judge.ts";
|
||||
|
||||
/** Run-wide budget counter, shared by every live scenario in a run. */
|
||||
@@ -248,7 +250,7 @@ export async function runLiveScenario(
|
||||
): Promise<CaseResult> {
|
||||
const scenario = opts.scenario;
|
||||
const evidenceDir = resolve(opts.outDir, opts.version, scenario.id);
|
||||
mkdirSync(evidenceDir, { recursive: true });
|
||||
resetDir(evidenceDir, opts.outDir);
|
||||
|
||||
const keys = opts.keys ?? collectLiveKeys(opts.config);
|
||||
if (Object.keys(keys).length === 0) {
|
||||
@@ -285,6 +287,9 @@ export async function runLiveScenario(
|
||||
}
|
||||
const resolvedModel = model;
|
||||
|
||||
// Live models are slower than the stub: an unset scenario timeout uses
|
||||
// config.live.timeoutMs [240000], not the scripted default.
|
||||
const timeoutMs = scenario.timeoutMs ?? opts.config.live.timeoutMs;
|
||||
const need = opts.config.live.pass;
|
||||
const maxAttempts = opts.config.live.repeat;
|
||||
const attempts: { pass: boolean; failures: string[] }[] = [];
|
||||
@@ -297,6 +302,7 @@ export async function runLiveScenario(
|
||||
let durationMs = 0;
|
||||
let textExcerpt: string | undefined;
|
||||
let judge: { pass: boolean; reason: string } | undefined;
|
||||
let budgetStop: string | undefined;
|
||||
|
||||
for (let n = 0; n < maxAttempts; n++) {
|
||||
if (passed >= need) {
|
||||
@@ -306,6 +312,12 @@ export async function runLiveScenario(
|
||||
break;
|
||||
}
|
||||
if (opts.budget.spent >= opts.budget.limit) {
|
||||
if (attempts.length > 0) {
|
||||
// Attempts already ran and fewer than `pass` passed: stopping
|
||||
// here must fail the case, never hide those attempts as a skip.
|
||||
budgetStop = budgetExhaustedReason(opts.budget);
|
||||
break;
|
||||
}
|
||||
return {
|
||||
id: scenario.id,
|
||||
tier: "live",
|
||||
@@ -349,10 +361,12 @@ export async function runLiveScenario(
|
||||
|
||||
let run: RunOutput | undefined;
|
||||
try {
|
||||
run = await sandbox.run(scenario.prompt, {
|
||||
timeoutMs: scenario.timeoutMs,
|
||||
});
|
||||
run = await sandbox.run(scenario.prompt, { timeoutMs });
|
||||
} finally {
|
||||
// Value-side scrubbing (OCI-7.3-evidence-leaks): the live tier
|
||||
// carries the real key, so persisted evidence is scrubbed while
|
||||
// in-memory evidence stays raw for assertions.
|
||||
const secrets = collectSecretValues(keys);
|
||||
try {
|
||||
copyFileSync(probeOut, join(attemptDir, "probe.jsonl"));
|
||||
} catch {
|
||||
@@ -360,10 +374,16 @@ export async function runLiveScenario(
|
||||
}
|
||||
writeFileSync(
|
||||
join(attemptDir, "events.jsonl"),
|
||||
(run?.events ?? []).map((e) => JSON.stringify(e)).join("\n") +
|
||||
(run?.events?.length ? "\n" : ""),
|
||||
redactSecretValues(
|
||||
(run?.events ?? []).map((e) => JSON.stringify(e)).join("\n") +
|
||||
(run?.events?.length ? "\n" : ""),
|
||||
secrets,
|
||||
),
|
||||
);
|
||||
writeFileSync(
|
||||
join(attemptDir, "stderr.log"),
|
||||
redactSecretValues(run?.stderr ?? "", secrets),
|
||||
);
|
||||
writeFileSync(join(attemptDir, "stderr.log"), run?.stderr ?? "");
|
||||
}
|
||||
if (run === undefined) {
|
||||
await sandbox.dispose();
|
||||
@@ -377,7 +397,7 @@ export async function runLiveScenario(
|
||||
let judgeFiles: string | undefined;
|
||||
try {
|
||||
if (run.timedOut) {
|
||||
failures.push(`opencode run timed out after ${scenario.timeoutMs}ms.`);
|
||||
failures.push(`opencode run timed out after ${timeoutMs}ms.`);
|
||||
}
|
||||
failures.push(
|
||||
...evaluate(scenario.expect, {
|
||||
@@ -420,7 +440,7 @@ export async function runLiveScenario(
|
||||
env: keys,
|
||||
evidenceDir: attemptDir,
|
||||
logLevel: opts.logLevel ?? opts.config.logLevel,
|
||||
timeoutMs: scenario.timeoutMs,
|
||||
timeoutMs,
|
||||
text: textOf(run.events),
|
||||
toolCalls: summarizeToolCalls(probe),
|
||||
hookSummary: summarizeHooks(probe),
|
||||
@@ -449,7 +469,7 @@ export async function runLiveScenario(
|
||||
overallPass || attempts.length === 0
|
||||
? undefined
|
||||
: [
|
||||
`live scenario ${scenario.id}: ${passed} of ${attempts.length} attempt(s) passed, needed ${need} of ${maxAttempts}.`,
|
||||
`live scenario ${scenario.id}: ${passed} of ${attempts.length} attempt(s) passed, needed ${need} of ${maxAttempts}${budgetStop === undefined ? "" : ` (${budgetStop})`}.`,
|
||||
...attempts.flatMap((attempt, index) =>
|
||||
attempt.pass
|
||||
? []
|
||||
|
||||
+150
-24
@@ -2,17 +2,21 @@
|
||||
// assert it registers (inventory), stays silent in the logs, and provides
|
||||
// every expected tool/command/agent/skill/mcp name.
|
||||
//
|
||||
import { copyFileSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { copyFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { ConfigError } from "../config/load.ts";
|
||||
import { errors, textOf, usage } from "../opencode/events.ts";
|
||||
import { parseLogs, pluginLogProblems } from "../opencode/logs.ts";
|
||||
import { resolveEntries } from "../plugin/entry.ts";
|
||||
import { writeShim } from "../plugin/shim.ts";
|
||||
import { probePluginDirs, readProbe } from "../probe/read.ts";
|
||||
import {
|
||||
type InventoryPlugin,
|
||||
probePluginDirs,
|
||||
readProbe,
|
||||
} from "../probe/read.ts";
|
||||
import type { CaseResult } from "../report/results.ts";
|
||||
import { type RunOutput, Sandbox } from "../sandbox/sandbox.ts";
|
||||
import { startStub } from "../stub/stub.ts";
|
||||
import { isFile, resetDir } from "../util/fs.ts";
|
||||
|
||||
// Re-exported so existing importers (`./load.ts`) keep working.
|
||||
export type { CaseResult } from "../report/results.ts";
|
||||
@@ -27,6 +31,8 @@ export interface LoadTierExpected {
|
||||
|
||||
export interface RunLoadOptions {
|
||||
pluginPath: string;
|
||||
/** Absolute server entry, resolved once by the caller (honors plugin.entry). */
|
||||
serverEntry: string;
|
||||
pluginOptions?: Record<string, unknown>;
|
||||
opencodeBin: string;
|
||||
outDir: string;
|
||||
@@ -40,6 +46,114 @@ function firstLines(text: string, count = 3): string {
|
||||
return text.split("\n").slice(0, count).join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* The tier-1 sandbox plugin order (OCI-4.3-order): the pre-probe records what
|
||||
* the plugin under test receives, the post-probe records what it produced, so
|
||||
* hooks fire in config `plugins` array order as `[pre, plugin, post]`.
|
||||
* Extracted pure so a load-order regression (e.g. `[post, shim, pre]`,
|
||||
* which silently inverts pre/post attribution) goes red in
|
||||
* `test/unit/load-order.test.ts` instead of passing tier 1 unnoticed.
|
||||
*/
|
||||
export function buildLoadPlugins(
|
||||
pre: string,
|
||||
shimDir: string,
|
||||
post: string,
|
||||
pluginOptions: Record<string, unknown> | undefined,
|
||||
probeOut: string,
|
||||
): Array<{ package: string; options: Record<string, unknown> }> {
|
||||
return [
|
||||
{ package: pre, options: { out: probeOut, phase: "pre" } },
|
||||
{ package: shimDir, options: pluginOptions ?? {} },
|
||||
{ package: post, options: { out: probeOut, phase: "post" } },
|
||||
];
|
||||
}
|
||||
/**
|
||||
* Plugin ids this tool installs itself: the pre/post probes. Anything else
|
||||
* in the tier-1 inventory was merged from outside the sandbox config (for
|
||||
* example an ancestor `opencode.json(c)` opencode walks up to), so it must
|
||||
* fail tier 1 closed instead of silently joining the run.
|
||||
*/
|
||||
export const PROBE_PLUGIN_IDS = [
|
||||
"opencode-plugin-ci.probe.pre",
|
||||
"opencode-plugin-ci.probe.post",
|
||||
] as const;
|
||||
|
||||
/** Allowed inventory ids: the probes plus the plugin under test, when known. */
|
||||
export function expectedInventoryIds(pluginId?: string): string[] {
|
||||
return pluginId === undefined
|
||||
? [...PROBE_PLUGIN_IDS]
|
||||
: [...PROBE_PLUGIN_IDS, pluginId];
|
||||
}
|
||||
|
||||
/**
|
||||
* Inventory plugin ids that are not in `expectedIds` (deduped, in first-seen
|
||||
* order). Binary builtins (`source.type === "builtin"`, shipped with the
|
||||
* opencode binary itself — 86 of them in a clean 2.0.26 inventory) are always
|
||||
* allowed. Every other plugin must be one of the configured local packages,
|
||||
* matched by id AND by resolved local path: an ancestor `opencode.json(c)`
|
||||
* can list foreign npm/local plugins (even under a colliding id), and those
|
||||
* have a non-builtin source outside the configured dirs. Anything without a
|
||||
* verifiable origin fails closed. An empty result means the sandbox saw
|
||||
* exactly the configured set.
|
||||
*/
|
||||
export function findUnexpectedPlugins(
|
||||
plugins: InventoryPlugin[],
|
||||
expectedIds: string[],
|
||||
allowedLocalDirs: string[],
|
||||
): string[] {
|
||||
const allowedIds = new Set(expectedIds);
|
||||
const extras: string[] = [];
|
||||
for (const plugin of plugins) {
|
||||
if (isBuiltinPlugin(plugin.source)) {
|
||||
continue;
|
||||
}
|
||||
const path = sourcePathOf(plugin.source);
|
||||
const dirOk =
|
||||
path !== undefined &&
|
||||
allowedLocalDirs.some((dir) => path === dir || dirname(path) === dir);
|
||||
if (dirOk && plugin.id !== undefined && allowedIds.has(plugin.id)) {
|
||||
continue;
|
||||
}
|
||||
const id = plugin.id ?? path ?? "(unknown plugin)";
|
||||
if (!extras.includes(id)) {
|
||||
extras.push(id);
|
||||
}
|
||||
}
|
||||
return extras;
|
||||
}
|
||||
|
||||
function isBuiltinPlugin(source: unknown): boolean {
|
||||
return (
|
||||
source !== null &&
|
||||
typeof source === "object" &&
|
||||
(source as { type?: unknown }).type === "builtin"
|
||||
);
|
||||
}
|
||||
|
||||
const ANCESTOR_CONFIG_NAMES = ["opencode.json", "opencode.jsonc"] as const;
|
||||
|
||||
/**
|
||||
* Walk from `startDir` up to the filesystem root, returning the first
|
||||
* ancestor `opencode.json(c)` found. opencode merges ancestor configs into
|
||||
* every run, so a hit means the sandbox is not isolated. Pure and sync so
|
||||
* tests can plant a fake ancestor tree.
|
||||
*/
|
||||
export function detectAncestorConfig(startDir: string): string | undefined {
|
||||
let dir = resolve(startDir);
|
||||
for (;;) {
|
||||
for (const name of ANCESTOR_CONFIG_NAMES) {
|
||||
if (isFile(join(dir, name))) {
|
||||
return join(dir, name);
|
||||
}
|
||||
}
|
||||
const parent = dirname(dir);
|
||||
if (parent === dir) {
|
||||
return undefined;
|
||||
}
|
||||
dir = parent;
|
||||
}
|
||||
}
|
||||
|
||||
function sourcePathOf(source: unknown): string | undefined {
|
||||
if (typeof source === "string") {
|
||||
return source;
|
||||
@@ -58,23 +172,16 @@ function sourcePathOf(source: unknown): string | undefined {
|
||||
* only for setup bugs (missing entry, sandbox crash).
|
||||
*/
|
||||
export async function runLoad(opts: RunLoadOptions): Promise<CaseResult> {
|
||||
const pluginPath = resolve(opts.pluginPath);
|
||||
const entries = resolveEntries(pluginPath);
|
||||
if (entries.server === undefined) {
|
||||
throw new ConfigError(
|
||||
`No server entry found under ${pluginPath}: ` +
|
||||
`looked for package.json exports["./server"]/["."], main, ` +
|
||||
`then server.{ts,js,mjs} and index.{ts,js,mjs}.`,
|
||||
);
|
||||
}
|
||||
const serverEntry = entries.server;
|
||||
const serverEntry = opts.serverEntry;
|
||||
|
||||
const evidenceDir = resolve(opts.outDir, opts.version, "load");
|
||||
mkdirSync(evidenceDir, { recursive: true });
|
||||
resetDir(evidenceDir, opts.outDir);
|
||||
|
||||
const stub = await startStub({
|
||||
script: [{ text: "ok" }],
|
||||
recordFile: join(evidenceDir, "stub-requests.jsonl"),
|
||||
// Tier 1 never gets live keys, so no value-side scrubbing is needed
|
||||
// here: the stub record, events and logs are written unscrubbed.
|
||||
});
|
||||
// The shim lives inside the evidence dir (which is kept), not inside
|
||||
// the sandbox root (which is deleted on dispose).
|
||||
@@ -85,11 +192,7 @@ export async function runLoad(opts: RunLoadOptions): Promise<CaseResult> {
|
||||
const sandbox = new Sandbox({
|
||||
opencodeBin: opts.opencodeBin,
|
||||
stubUrl: stub.url,
|
||||
plugins: [
|
||||
{ package: pre, options: { out: probeOut, phase: "pre" } },
|
||||
{ package: shimDir, options: opts.pluginOptions ?? {} },
|
||||
{ package: post, options: { out: probeOut, phase: "post" } },
|
||||
],
|
||||
plugins: buildLoadPlugins(pre, shimDir, post, opts.pluginOptions, probeOut),
|
||||
logLevel: opts.logLevel ?? "info",
|
||||
});
|
||||
|
||||
@@ -98,6 +201,10 @@ export async function runLoad(opts: RunLoadOptions): Promise<CaseResult> {
|
||||
try {
|
||||
run = await sandbox.run("Reply with ok.");
|
||||
} finally {
|
||||
// Tier 1 never receives live keys, so no value-side scrubbing is
|
||||
// needed here: evidence is written directly. `probe.jsonl` stays raw
|
||||
// by design, and `results.json` scrubbing still happens centrally in
|
||||
// `run.ts` (scrubRow) for tiers that do carry live keys.
|
||||
try {
|
||||
copyFileSync(probeOut, join(evidenceDir, "probe.jsonl"));
|
||||
} catch {
|
||||
@@ -149,10 +256,12 @@ export async function runLoad(opts: RunLoadOptions): Promise<CaseResult> {
|
||||
`plugin setup failed: ${firstLines(String(plugin.error ?? "unknown error"))}`,
|
||||
);
|
||||
}
|
||||
const problems = pluginLogProblems(parseLogs(run.stderr), {
|
||||
id: plugin?.id,
|
||||
path: shimServerJs,
|
||||
});
|
||||
const lines = parseLogs(run.stderr);
|
||||
// Stack traces name the real entry, not the shim: check both paths.
|
||||
const problems = [
|
||||
...pluginLogProblems(lines, { id: plugin?.id, path: shimServerJs }),
|
||||
...pluginLogProblems(lines, { path: serverEntry }),
|
||||
].filter((line, index, all) => all.indexOf(line) === index);
|
||||
for (const line of problems) {
|
||||
failures.push(`plugin log problem: ${line._raw}`);
|
||||
}
|
||||
@@ -173,6 +282,23 @@ export async function runLoad(opts: RunLoadOptions): Promise<CaseResult> {
|
||||
}
|
||||
}
|
||||
}
|
||||
const unexpected = findUnexpectedPlugins(
|
||||
inventory.plugins,
|
||||
expectedInventoryIds(plugin?.id),
|
||||
[pre, shimDir, post],
|
||||
);
|
||||
if (unexpected.length > 0) {
|
||||
failures.push(
|
||||
`unexpected plugins in inventory from ancestor config: [${unexpected.join(", ")}].`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const ancestorConfig = detectAncestorConfig(sandbox.root);
|
||||
if (ancestorConfig !== undefined) {
|
||||
failures.push(
|
||||
`ancestor opencode config detected at ${ancestorConfig}: sandbox isolation compromised.`,
|
||||
);
|
||||
}
|
||||
|
||||
const used = usage(run.events);
|
||||
|
||||
+26
-10
@@ -3,7 +3,7 @@
|
||||
// Each scenario gets its own stub, sandbox, and evidence directory, so
|
||||
// scenarios can run concurrently through the `pool` helper below.
|
||||
|
||||
import { copyFileSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { copyFileSync, existsSync, writeFileSync } from "node:fs";
|
||||
import { availableParallelism } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { evaluate } from "../assert/expect.ts";
|
||||
@@ -14,6 +14,9 @@ import { probePluginDirs, readProbe } from "../probe/read.ts";
|
||||
import type { CaseResult } from "../report/results.ts";
|
||||
import { type RunOutput, Sandbox } from "../sandbox/sandbox.ts";
|
||||
import { startStub } from "../stub/stub.ts";
|
||||
import { resetDir } from "../util/fs.ts";
|
||||
import { collectSecretValues, redactSecretValues } from "../util/redact.ts";
|
||||
import { collectLiveKeys } from "./live.ts";
|
||||
|
||||
export interface RunScenarioOptions {
|
||||
scenario: Scenario;
|
||||
@@ -36,12 +39,15 @@ export async function runScenario(
|
||||
): Promise<CaseResult> {
|
||||
const scenario = opts.scenario;
|
||||
const evidenceDir = resolve(opts.outDir, opts.version, scenario.id);
|
||||
mkdirSync(evidenceDir, { recursive: true });
|
||||
resetDir(evidenceDir, opts.outDir);
|
||||
|
||||
const stub = await startStub({
|
||||
script: scenario.script ?? [],
|
||||
judgeReply: scenario.judgeReply,
|
||||
recordFile: join(evidenceDir, "stub-requests.jsonl"),
|
||||
// Configured live-key values may have innocuous names: pass them
|
||||
// explicitly. Host env is never scanned — only these live-key values
|
||||
// are scrubbed.
|
||||
redactValues: collectSecretValues(collectLiveKeys(opts.config)),
|
||||
});
|
||||
// The shim lives inside the evidence dir (which is kept), not inside
|
||||
// the sandbox root (which is deleted on dispose).
|
||||
@@ -65,12 +71,16 @@ export async function runScenario(
|
||||
logLevel: opts.logLevel ?? opts.config.logLevel,
|
||||
});
|
||||
|
||||
const timeoutMs = scenario.timeoutMs ?? 120_000;
|
||||
let run: RunOutput | undefined;
|
||||
try {
|
||||
run = await sandbox.run(scenario.prompt, {
|
||||
timeoutMs: scenario.timeoutMs,
|
||||
});
|
||||
run = await sandbox.run(scenario.prompt, { timeoutMs });
|
||||
} finally {
|
||||
// Value-side scrubbing (OCI-7.3-evidence-leaks): persisted evidence
|
||||
// must not carry live secret values; in-memory evidence stays raw so
|
||||
// assertions still fail on leaks, and probe.jsonl stays raw so
|
||||
// `notContains`-style assertions keep seeing the leak.
|
||||
const secrets = collectSecretValues(collectLiveKeys(opts.config));
|
||||
try {
|
||||
if (existsSync(probeOut)) {
|
||||
copyFileSync(probeOut, join(evidenceDir, "probe.jsonl"));
|
||||
@@ -82,10 +92,16 @@ export async function runScenario(
|
||||
}
|
||||
writeFileSync(
|
||||
join(evidenceDir, "events.jsonl"),
|
||||
(run?.events ?? []).map((e) => JSON.stringify(e)).join("\n") +
|
||||
(run?.events?.length ? "\n" : ""),
|
||||
redactSecretValues(
|
||||
(run?.events ?? []).map((e) => JSON.stringify(e)).join("\n") +
|
||||
(run?.events?.length ? "\n" : ""),
|
||||
secrets,
|
||||
),
|
||||
);
|
||||
writeFileSync(
|
||||
join(evidenceDir, "stderr.log"),
|
||||
redactSecretValues(run?.stderr ?? "", secrets),
|
||||
);
|
||||
writeFileSync(join(evidenceDir, "stderr.log"), run?.stderr ?? "");
|
||||
await stub.stop();
|
||||
}
|
||||
if (run === undefined) {
|
||||
@@ -98,7 +114,7 @@ export async function runScenario(
|
||||
const probe = readProbe(join(evidenceDir, "probe.jsonl"));
|
||||
try {
|
||||
if (run.timedOut) {
|
||||
failures.push(`opencode run timed out after ${scenario.timeoutMs}ms.`);
|
||||
failures.push(`opencode run timed out after ${timeoutMs}ms.`);
|
||||
}
|
||||
failures.push(
|
||||
...evaluate(scenario.expect, {
|
||||
|
||||
@@ -2,12 +2,13 @@
|
||||
// process and check the default export looks like a plugin
|
||||
// ({ id: string, setup: function }). No opencode run is involved.
|
||||
import { type ChildProcess, spawn } from "node:child_process";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { ConfigError } from "../config/load.ts";
|
||||
import { resolveEntries } from "../plugin/entry.ts";
|
||||
import type { CaseResult } from "../report/results.ts";
|
||||
import { resetDir } from "../util/fs.ts";
|
||||
|
||||
export interface RunUiStaticOptions {
|
||||
pluginPath: string;
|
||||
@@ -48,7 +49,7 @@ export async function runUiStatic(
|
||||
].join("\n");
|
||||
|
||||
const evidenceDir = resolve(opts.outDir, opts.version, "load");
|
||||
mkdirSync(evidenceDir, { recursive: true });
|
||||
resetDir(evidenceDir, opts.outDir);
|
||||
|
||||
const child: ChildProcess = spawn("bun", ["-e", code], {
|
||||
cwd: pluginPath,
|
||||
|
||||
@@ -3,15 +3,33 @@ import {
|
||||
appendFileSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { isAbsolute, relative } from "node:path";
|
||||
|
||||
/** mkdir -p. */
|
||||
export function ensureDir(dir: string): void {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Empty `dir` (rm -rf, then mkdir -p). Evidence dirs are reset before each
|
||||
* case because the probe appends: records left by an earlier run in the same
|
||||
* out dir would otherwise be evaluated as if this run produced them. Refuses
|
||||
* any `dir` that is not strictly inside `root`, so a bad path segment can
|
||||
* never delete something outside the out dir.
|
||||
*/
|
||||
export function resetDir(dir: string, root: string): void {
|
||||
const rel = relative(root, dir);
|
||||
if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) {
|
||||
throw new Error(`refusing to reset ${dir}: it is not inside ${root}`);
|
||||
}
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
/** True when path exists and is a regular file. Never throws. */
|
||||
export function isFile(path: string): boolean {
|
||||
try {
|
||||
|
||||
@@ -22,3 +22,71 @@ export function excerpt(value: unknown, max: number = TRUNCATE_MAX): string {
|
||||
return truncate(String(value), max);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimum secret length worth scrubbing. Shorter values (a one-char flag, a
|
||||
* port number, or common words like "false") would nuke unrelated evidence
|
||||
* text.
|
||||
*/
|
||||
const MIN_SECRET_LENGTH = 8;
|
||||
|
||||
/**
|
||||
* Replace every occurrence of each known secret value with `[redacted]`.
|
||||
* Longer secrets go first so overlapping values scrub completely, and each
|
||||
* value's JSON-escaped form (`JSON.stringify(value).slice(1, -1)`) is
|
||||
* scrubbed too, so secrets containing `"` or `\` are still caught inside
|
||||
* JSONL lines. Empty, whitespace-only and short values are skipped. Never
|
||||
* throws: returns the text unchanged on malformed input. Split/join (not
|
||||
* regex) so values with regex metacharacters are matched literally.
|
||||
*/
|
||||
export function redactSecretValues(text: string, secrets: string[]): string {
|
||||
try {
|
||||
if (typeof text !== "string") {
|
||||
return text;
|
||||
}
|
||||
if (!Array.isArray(secrets)) {
|
||||
return text;
|
||||
}
|
||||
const variants = new Set<string>();
|
||||
for (const secret of new Set(secrets)) {
|
||||
if (typeof secret !== "string") {
|
||||
continue;
|
||||
}
|
||||
if (secret.trim().length < MIN_SECRET_LENGTH) {
|
||||
continue;
|
||||
}
|
||||
variants.add(secret);
|
||||
const escaped = JSON.stringify(secret).slice(1, -1);
|
||||
if (escaped !== secret) {
|
||||
variants.add(escaped);
|
||||
}
|
||||
}
|
||||
const values = [...variants].sort((a, b) => b.length - a.length);
|
||||
let out = text;
|
||||
for (const value of values) {
|
||||
out = out.split(value).join("[redacted]");
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect live secret values: the values of explicitly configured key names
|
||||
* (`extra`, e.g. `collectLiveKeys(config)` — a key var may have an innocuous
|
||||
* name like `MY_API`). Host environment variables are deliberately NOT
|
||||
* scanned: the sandbox env is an allowlist, so host vars never reach tier 1
|
||||
* or scripted runs, and scanning them scrubbed innocuous values (e.g.
|
||||
* `TOKENIZERS_PARALLELISM=false`) out of persisted evidence. Values travel
|
||||
* only into scrubbing, never into logs.
|
||||
*/
|
||||
export function collectSecretValues(extra?: Record<string, string>): string[] {
|
||||
const values: string[] = [];
|
||||
for (const value of Object.values(extra ?? {})) {
|
||||
if (typeof value === "string" && value.length > 0) {
|
||||
values.push(value);
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
+14
-1
@@ -2,7 +2,7 @@
|
||||
// review/oc): `run` in tool-basic passes with --scenario filtering and
|
||||
// writes report.md, `run` in throws exits 1, `validate` in no-entry exits 2.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { existsSync } from "node:fs";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
const CLI = resolve(import.meta.dir, "../../bin/opencode-plugin-ci.ts");
|
||||
@@ -52,6 +52,19 @@ describe("cli e2e", () => {
|
||||
test("run in throws exits 1", async () => {
|
||||
const { code } = await runCli(["run"], join(FIXTURES, "throws"));
|
||||
expect(code).toBe(1);
|
||||
// OCI-1.5: a failing tier 1 must skip the later tiers for that
|
||||
// version instead of running them (which would also burn live
|
||||
// budget on doomed attempts).
|
||||
const results = JSON.parse(
|
||||
readFileSync(
|
||||
join(FIXTURES, "throws", ".opencode-ci/out/results.json"),
|
||||
"utf8",
|
||||
),
|
||||
) as {
|
||||
results: { id: string; skipped?: string }[];
|
||||
};
|
||||
const smoke = results.results.find((r) => r.id === "throws-smoke");
|
||||
expect(smoke?.skipped).toBe("tier 1 failed");
|
||||
}, 600_000);
|
||||
|
||||
test("validate in no-entry exits 2", async () => {
|
||||
|
||||
@@ -25,6 +25,15 @@ function opencodeBin(): string {
|
||||
);
|
||||
}
|
||||
|
||||
/** The fixture's server entry, resolved the way `run` resolves it. */
|
||||
function serverEntryOf(name: string): string {
|
||||
const entry = resolveEntries(join(FIXTURES, name)).server;
|
||||
if (entry === undefined) {
|
||||
throw new Error(`fixture ${name} has no server entry`);
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
function outDir(name: string): string {
|
||||
return mkdtempSync(join(tmpdir(), `tier1-${name}-`));
|
||||
}
|
||||
@@ -33,6 +42,7 @@ describe("tier 1 (load)", () => {
|
||||
test("tool-basic passes with the expected tool listed", async () => {
|
||||
const result = await runLoad({
|
||||
pluginPath: join(FIXTURES, "tool-basic"),
|
||||
serverEntry: serverEntryOf("tool-basic"),
|
||||
opencodeBin: opencodeBin(),
|
||||
outDir: outDir("tool-basic"),
|
||||
version: "2.0.26",
|
||||
@@ -47,6 +57,7 @@ describe("tier 1 (load)", () => {
|
||||
test("throws fails with fixture boom", async () => {
|
||||
const result = await runLoad({
|
||||
pluginPath: join(FIXTURES, "throws"),
|
||||
serverEntry: serverEntryOf("throws"),
|
||||
opencodeBin: opencodeBin(),
|
||||
outDir: outDir("throws"),
|
||||
version: "2.0.26",
|
||||
@@ -69,6 +80,7 @@ describe("tier 1 (load)", () => {
|
||||
test("missing expected tool is reported", async () => {
|
||||
const result = await runLoad({
|
||||
pluginPath: join(FIXTURES, "tool-basic"),
|
||||
serverEntry: serverEntryOf("tool-basic"),
|
||||
opencodeBin: opencodeBin(),
|
||||
outDir: outDir("missing-tool"),
|
||||
version: "2.0.26",
|
||||
|
||||
@@ -36,9 +36,36 @@ const CASES: Case[] = [
|
||||
},
|
||||
{ fixture: "tool-direct", file: "pass.yaml", pass: true },
|
||||
{ fixture: "hook-mutate", file: "pass.yaml", pass: true },
|
||||
{
|
||||
fixture: "hook-mutate",
|
||||
file: "fail.yaml",
|
||||
pass: false,
|
||||
contains:
|
||||
'hooks[0][email protected]: expected "echo secret", got "echo [x]".',
|
||||
},
|
||||
{ fixture: "shell-env", file: "pass.yaml", pass: true },
|
||||
{
|
||||
fixture: "shell-env",
|
||||
file: "fail.yaml",
|
||||
pass: false,
|
||||
contains:
|
||||
'hooks[0][email protected]_ON: expected "2", got "1".',
|
||||
},
|
||||
{ fixture: "system-inject", file: "pass.yaml", pass: true },
|
||||
{
|
||||
fixture: "system-inject",
|
||||
file: "fail.yaml",
|
||||
pass: false,
|
||||
contains: 'request.system: expected to contain "FIXTURE-ABSENT"',
|
||||
},
|
||||
{ fixture: "event-file", file: "pass.yaml", pass: true },
|
||||
{
|
||||
fixture: "event-file",
|
||||
file: "fail.yaml",
|
||||
pass: false,
|
||||
contains:
|
||||
"files[0]..fixture/events.log: expected file to exist, but it does not.",
|
||||
},
|
||||
];
|
||||
|
||||
describe("tier 2 (scripted)", () => {
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
id: event-file-fail
|
||||
description: Deliberately failing scenario (no tool call, so the event never fires and the file is never written)
|
||||
mode: scripted
|
||||
prompt: Just answer.
|
||||
script:
|
||||
- text: done
|
||||
expect:
|
||||
exitCode: 0
|
||||
files:
|
||||
- path: .fixture/events.log
|
||||
exists: true
|
||||
@@ -0,0 +1,16 @@
|
||||
id: hook-mutate-fail
|
||||
description: Deliberately failing scenario (expects the raw command after the hook, which redacts it)
|
||||
mode: scripted
|
||||
prompt: Print the word secret.
|
||||
script:
|
||||
- toolCalls:
|
||||
- name: shell
|
||||
arguments: { command: "echo secret", description: "print secret" }
|
||||
- text: done
|
||||
expect:
|
||||
exitCode: 0
|
||||
hooks:
|
||||
- hook: tool.execute.before
|
||||
phase: post
|
||||
tool: shell
|
||||
input: { command: { equals: "echo secret" } }
|
||||
@@ -0,0 +1,16 @@
|
||||
id: shell-env-fail
|
||||
description: Deliberately failing scenario (expects FIXTURE_ON=2, the hook injects 1)
|
||||
mode: scripted
|
||||
prompt: Show the fixture env var.
|
||||
script:
|
||||
- toolCalls:
|
||||
- name: shell
|
||||
arguments:
|
||||
command: "echo FIXTURE_ON=$FIXTURE_ON"
|
||||
description: "print fixture env"
|
||||
- text: done
|
||||
expect:
|
||||
exitCode: 0
|
||||
hooks:
|
||||
- hook: shell.create.before
|
||||
env: { FIXTURE_ON: { equals: "2" } }
|
||||
@@ -0,0 +1,11 @@
|
||||
id: system-inject-fail
|
||||
description: Deliberately failing scenario (expects text the hook never adds)
|
||||
mode: scripted
|
||||
prompt: Hello.
|
||||
script:
|
||||
- text: done
|
||||
expect:
|
||||
exitCode: 0
|
||||
request:
|
||||
nth: 1
|
||||
system: { contains: FIXTURE-ABSENT }
|
||||
@@ -13,6 +13,8 @@ expect:
|
||||
- name: echo_tool
|
||||
status: completed
|
||||
output: { contains: "echo:hi" }
|
||||
# codemode is false, so the call must not go through Code Mode's execute tool.
|
||||
noTools: [execute]
|
||||
request:
|
||||
nth: 1
|
||||
tools: { includes: [echo_tool] }
|
||||
@@ -0,0 +1,210 @@
|
||||
// A stand-in for the `opencode` binary, for orchestration tests that must be
|
||||
// fast and deterministic. `makeFakeOpencode(dir, plan)` writes the plan and a
|
||||
// bash wrapper; point `opencodeBin` / OPENCODE_BIN at the wrapper.
|
||||
//
|
||||
// The sandbox passes only an allowlisted env to opencode, so the plan path
|
||||
// travels as an argument baked into the wrapper, never as an env var.
|
||||
//
|
||||
// Each `run` invocation consumes the next entry of `plan.runs` (the last
|
||||
// entry repeats). The number of `run` invocations so far is kept in
|
||||
// `<plan>.count`, so tests can assert how many runs happened.
|
||||
import {
|
||||
appendFileSync,
|
||||
chmodSync,
|
||||
existsSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
export interface FakeToolCall {
|
||||
tool: string;
|
||||
input?: unknown;
|
||||
result?: unknown;
|
||||
status?: "completed" | "error";
|
||||
}
|
||||
|
||||
export interface FakeRun {
|
||||
/** Process exit code [0]. */
|
||||
exitCode?: number;
|
||||
/** Emitted as one `text` event when set. */
|
||||
text?: string;
|
||||
/** Emitted in one `step_finish` event [0]. */
|
||||
cost?: number;
|
||||
/** Emits one `error` event with this message when set. */
|
||||
error?: string;
|
||||
/** Written to stderr verbatim. */
|
||||
stderr?: string;
|
||||
/** Inventory status of the plugin under test ["active"]. */
|
||||
pluginStatus?: "active" | "failed" | "missing";
|
||||
/** Inventory tools [[]]. */
|
||||
inventoryTools?: string[];
|
||||
/** Written as tool.before/tool.after probe records, pre and post. */
|
||||
toolCalls?: FakeToolCall[];
|
||||
}
|
||||
|
||||
export interface FakePlan {
|
||||
/** Reported by `--version` ["2.0.26"]. */
|
||||
version?: string;
|
||||
/** Printed one per line by `models` [["opencode/fake-model"]]. */
|
||||
models?: string[];
|
||||
runs: FakeRun[];
|
||||
}
|
||||
|
||||
/** Write `<dir>/plan.json` and an executable `<dir>/opencode` wrapper; return the wrapper path. */
|
||||
export function makeFakeOpencode(dir: string, plan: FakePlan): string {
|
||||
const planPath = join(dir, "plan.json");
|
||||
writeFileSync(planPath, JSON.stringify(plan));
|
||||
const wrapper = join(dir, "opencode");
|
||||
writeFileSync(
|
||||
wrapper,
|
||||
`#!/usr/bin/env bash\nexec bun '${import.meta.path}' '${planPath}' "$@"\n`,
|
||||
);
|
||||
chmodSync(wrapper, 0o755);
|
||||
return wrapper;
|
||||
}
|
||||
|
||||
/** How many `run` invocations the fake has served for the plan in `dir`. */
|
||||
export function fakeRunCount(dir: string): number {
|
||||
const file = join(dir, "plan.json.count");
|
||||
return existsSync(file) ? Number(readFileSync(file, "utf8")) : 0;
|
||||
}
|
||||
|
||||
function main(): number {
|
||||
const [planPath, command, ...rest] = process.argv.slice(2);
|
||||
if (planPath === undefined) {
|
||||
return 2;
|
||||
}
|
||||
const plan = JSON.parse(readFileSync(planPath, "utf8")) as FakePlan;
|
||||
if (command === "--version") {
|
||||
console.log(`opencode v${plan.version ?? "2.0.26"}`);
|
||||
return 0;
|
||||
}
|
||||
if (command === "models") {
|
||||
console.log((plan.models ?? ["opencode/fake-model"]).join("\n"));
|
||||
return 0;
|
||||
}
|
||||
if (command !== "run") {
|
||||
console.error(
|
||||
`fake opencode: unsupported command ${command} ${rest.join(" ")}`,
|
||||
);
|
||||
return 2;
|
||||
}
|
||||
|
||||
const countFile = `${planPath}.count`;
|
||||
const n = existsSync(countFile) ? Number(readFileSync(countFile, "utf8")) : 0;
|
||||
writeFileSync(countFile, String(n + 1));
|
||||
const run = plan.runs[Math.min(n, plan.runs.length - 1)] ?? {};
|
||||
|
||||
// The sandbox writes JSON (not JSONC) to this path, so JSON.parse works.
|
||||
const configFile = join(
|
||||
process.env.XDG_CONFIG_HOME ?? "",
|
||||
"opencode",
|
||||
"opencode.jsonc",
|
||||
);
|
||||
const config = JSON.parse(readFileSync(configFile, "utf8")) as {
|
||||
plugins?: { package: string; options?: Record<string, unknown> }[];
|
||||
};
|
||||
const plugins = config.plugins ?? [];
|
||||
const probeOut = plugins.find((p) => typeof p.options?.out === "string")
|
||||
?.options?.out as string | undefined;
|
||||
const underTest = plugins.find((p) =>
|
||||
p.package.endsWith("/plugin-under-test"),
|
||||
);
|
||||
|
||||
if (probeOut !== undefined) {
|
||||
let seq = 0;
|
||||
const write = (record: Record<string, unknown>): void => {
|
||||
appendFileSync(
|
||||
probeOut,
|
||||
`${JSON.stringify({ ...record, seq: seq++, t: Date.now() })}\n`,
|
||||
);
|
||||
};
|
||||
|
||||
(run.toolCalls ?? []).forEach((call, index) => {
|
||||
for (const phase of ["pre", "post"]) {
|
||||
write({
|
||||
kind: "tool.before",
|
||||
phase,
|
||||
tool: call.tool,
|
||||
callID: `call-${index}`,
|
||||
input: call.input ?? {},
|
||||
});
|
||||
}
|
||||
for (const phase of ["pre", "post"]) {
|
||||
write({
|
||||
kind: "tool.after",
|
||||
phase,
|
||||
tool: call.tool,
|
||||
callID: `call-${index}`,
|
||||
status: call.status ?? "completed",
|
||||
input: call.input ?? {},
|
||||
result: call.result ?? { content: "" },
|
||||
});
|
||||
}
|
||||
});
|
||||
const status = run.pluginStatus ?? "active";
|
||||
write({
|
||||
kind: "inventory",
|
||||
phase: "post",
|
||||
plugins:
|
||||
underTest === undefined || status === "missing"
|
||||
? []
|
||||
: [
|
||||
{
|
||||
id: "fixture-under-test",
|
||||
source: {
|
||||
type: "local",
|
||||
path: `${underTest.package}/server.js`,
|
||||
},
|
||||
status,
|
||||
error:
|
||||
status === "failed" ? "Error: fake setup failure" : undefined,
|
||||
},
|
||||
],
|
||||
tools: run.inventoryTools ?? [],
|
||||
commands: [],
|
||||
agents: [],
|
||||
skills: [],
|
||||
mcp: [],
|
||||
});
|
||||
}
|
||||
|
||||
if (run.text !== undefined) {
|
||||
console.log(
|
||||
JSON.stringify({ type: "text", part: { type: "text", text: run.text } }),
|
||||
);
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
type: "step_finish",
|
||||
part: {
|
||||
type: "step-finish",
|
||||
reason: "stop",
|
||||
cost: run.cost ?? 0,
|
||||
tokens: {
|
||||
input: 10,
|
||||
output: 5,
|
||||
reasoning: 0,
|
||||
cache: { read: 0, write: 0 },
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
if (run.error !== undefined) {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
type: "error",
|
||||
error: { type: "fake", message: run.error },
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (run.stderr !== undefined) {
|
||||
process.stderr.write(run.stderr);
|
||||
}
|
||||
return run.exitCode ?? 0;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
process.exit(main());
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// Shared setup for tests that drive `runCi` with the fake opencode binary.
|
||||
// Call isolateCiEnv() once at the top of a test file.
|
||||
import { afterAll, afterEach, beforeEach } from "bun:test";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { type FakeRun, makeFakeOpencode } from "./fake-opencode.ts";
|
||||
import { cleanupTempDirs, tempDir } from "./tmp.ts";
|
||||
|
||||
/** The smallest plugin opencode will activate. */
|
||||
export const PLUGIN_JS =
|
||||
'export default { id: "fixture-run", setup: async () => {} };\n';
|
||||
|
||||
/** Write a throwaway plugin repo; `files` maps relative paths to contents. */
|
||||
export function makePlugin(files: Record<string, string>): string {
|
||||
const dir = tempDir("run-plugin-");
|
||||
for (const [rel, content] of Object.entries(files)) {
|
||||
mkdirSync(dirname(join(dir, rel)), { recursive: true });
|
||||
writeFileSync(join(dir, rel), content);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** Point OPENCODE_BIN at a fresh fake; returns the fake's directory. */
|
||||
export function useFake(runs: FakeRun[]): string {
|
||||
const dir = tempDir("run-fake-");
|
||||
process.env.OPENCODE_BIN = makeFakeOpencode(dir, { runs });
|
||||
return dir;
|
||||
}
|
||||
|
||||
const SAVED = [
|
||||
"OPENCODE_BIN",
|
||||
"OPENCODE_API_KEY",
|
||||
"GITHUB_ACTIONS",
|
||||
"GITHUB_STEP_SUMMARY",
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Save and restore the env vars runCi reads, and clear the CI ones for each
|
||||
* test: with GITHUB_ACTIONS set the gate could run, and with
|
||||
* GITHUB_STEP_SUMMARY set the report would be appended to the real summary.
|
||||
*/
|
||||
export function isolateCiEnv(): void {
|
||||
let saved: Record<string, string | undefined> = {};
|
||||
beforeEach(() => {
|
||||
saved = Object.fromEntries(SAVED.map((name) => [name, process.env[name]]));
|
||||
delete process.env.GITHUB_ACTIONS;
|
||||
delete process.env.GITHUB_STEP_SUMMARY;
|
||||
delete process.env.OPENCODE_API_KEY;
|
||||
});
|
||||
afterEach(() => {
|
||||
for (const name of SAVED) {
|
||||
if (saved[name] === undefined) {
|
||||
delete process.env[name];
|
||||
} else {
|
||||
process.env[name] = saved[name];
|
||||
}
|
||||
}
|
||||
});
|
||||
afterAll(cleanupTempDirs);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
// Temp dirs for tests. Call cleanupTempDirs() from an afterAll so a test run
|
||||
// leaves nothing behind in the OS temp dir.
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const made: string[] = [];
|
||||
|
||||
/** mkdtemp under the OS temp dir; removed by the next cleanupTempDirs(). */
|
||||
export function tempDir(prefix: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), prefix));
|
||||
made.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
export function cleanupTempDirs(): void {
|
||||
for (const dir of made.splice(0)) {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// OCI-6.3-action-guard: the zero-interpolation invariant for action.yml —
|
||||
// no `run:` block may contain a `${{ inputs.* }}` or `${{ github.* }}`
|
||||
// expression. Inputs travel through `env:` only, so a future edit that
|
||||
// interpolates an input into a script fails here instead of silently
|
||||
// becoming a command-injection boundary.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import * as YAML from "yaml";
|
||||
|
||||
const REPO_ROOT = join(import.meta.dir, "..", "..");
|
||||
|
||||
interface ActionStep {
|
||||
run?: unknown;
|
||||
}
|
||||
|
||||
function parseAction(text: string): unknown {
|
||||
return YAML.parse(text) as unknown;
|
||||
}
|
||||
|
||||
/** Every `run:` script in the composite action, in step order. */
|
||||
export function runBlocks(action: unknown): string[] {
|
||||
const steps = (action as { runs?: { steps?: ActionStep[] } })?.runs?.steps;
|
||||
if (!Array.isArray(steps)) {
|
||||
throw new Error("action.yml has no runs.steps array.");
|
||||
}
|
||||
return steps
|
||||
.map((step) => step.run)
|
||||
.filter((run): run is string => typeof run === "string");
|
||||
}
|
||||
|
||||
const FORBIDDEN_IN_RUN = /\$\{\{\s*(inputs|github)\./;
|
||||
|
||||
/** True when a `run:` script keeps the zero-interpolation invariant. */
|
||||
export function runBlockIsClean(script: string): boolean {
|
||||
return !FORBIDDEN_IN_RUN.test(script);
|
||||
}
|
||||
|
||||
describe("action.yml zero-interpolation guard", () => {
|
||||
const action = parseAction(
|
||||
readFileSync(join(REPO_ROOT, "action.yml"), "utf8"),
|
||||
);
|
||||
|
||||
test("the action has run: blocks to guard (fail closed on shape drift)", () => {
|
||||
expect(runBlocks(action).length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test("no run: block interpolates inputs or github context", () => {
|
||||
// Filtered form: a failure prints the offending script(s).
|
||||
expect(
|
||||
runBlocks(action).filter((script) => !runBlockIsClean(script)),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
test("injected inputs/github expressions are caught", () => {
|
||||
// Built without a literal `${{` so the fixture itself stays lint-clean.
|
||||
const expr = (path: string): string => `$${"{{ "}${path} }`;
|
||||
expect(runBlockIsClean(`echo "tiers are ${expr("inputs.tiers")}"`)).toBe(
|
||||
false,
|
||||
);
|
||||
// A comment line still interpolates on the runner, so it must fail too.
|
||||
expect(runBlockIsClean(`# ${expr("inputs.tiers")}`)).toBe(false);
|
||||
expect(runBlockIsClean(`tag: ${expr("github.action_path")}/x`)).toBe(false);
|
||||
expect(runBlockIsClean('args+=(--tier "$t")')).toBe(true);
|
||||
});
|
||||
|
||||
test("github.action_path stays out of run: blocks", () => {
|
||||
for (const script of runBlocks(action)) {
|
||||
expect(script).not.toContain("github.action_path");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
// OCI-6.4-action-pins: `actions/checkout`, `actions/setup-node` and
|
||||
// `actions/upload-artifact` floated on superseded v4 majors. They are pinned
|
||||
// to v7 in both `action.yml` and `.github/workflows/ci.yml`
|
||||
// (`oven-sh/setup-bun@v2` is already at HEAD). A revert to v4 — or any other
|
||||
// single-digit major — goes red here.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
const REPO_ROOT = join(import.meta.dir, "..", "..");
|
||||
|
||||
function usesOf(file: string): string[] {
|
||||
const text = readFileSync(join(REPO_ROOT, file), "utf8");
|
||||
return text
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.startsWith("- uses:") || line.startsWith("uses:"))
|
||||
.map((line) => line.replace(/^[-\s]*uses:\s*/, ""));
|
||||
}
|
||||
|
||||
describe("action pins", () => {
|
||||
test("action.yml pins checkout/setup-node/upload-artifact to v7", () => {
|
||||
const uses = usesOf("action.yml");
|
||||
expect(uses).toContain("actions/setup-node@v7");
|
||||
expect(uses).toContain("actions/upload-artifact@v7");
|
||||
for (const u of uses.filter((entry) =>
|
||||
entry.startsWith("actions/checkout@"),
|
||||
)) {
|
||||
expect(u).toBe("actions/checkout@v7");
|
||||
}
|
||||
// No actions/* entry may float on an older major (setup-bun@v2 is
|
||||
// covered by the test below and is already at HEAD).
|
||||
const stale = uses.filter((u) => /actions\/[^@]+@v[1-6]\b/.test(u));
|
||||
expect(stale).toEqual([]);
|
||||
});
|
||||
|
||||
test("ci.yml pins checkout/setup-node to v7", () => {
|
||||
const uses = usesOf(".github/workflows/ci.yml");
|
||||
expect(uses).toContain("actions/checkout@v7");
|
||||
expect(uses).toContain("actions/setup-node@v7");
|
||||
const stale = uses.filter((u) => /actions\/[^@]+@v[1-6]\b/.test(u));
|
||||
expect(stale).toEqual([]);
|
||||
});
|
||||
|
||||
test("setup-bun stays on v2 (already at HEAD)", () => {
|
||||
for (const file of ["action.yml", ".github/workflows/ci.yml"]) {
|
||||
expect(usesOf(file)).toContain("oven-sh/setup-bun@v2");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
// OCI-6.3-working-dir: `working-directory` is runner-evaluated, so the
|
||||
// action guards it via env passthrough (`IN_WD`, keeping the
|
||||
// zero-interpolation invariant). The guard compares the resolved `$PWD`
|
||||
// (the step has already changed into working-directory) against the
|
||||
// resolved workspace root: absolute paths inside the workspace are fine,
|
||||
// paths outside fail before any tier runs. These tests pin the guard
|
||||
// script and its wiring.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import * as YAML from "yaml";
|
||||
|
||||
const REPO_ROOT = join(import.meta.dir, "..", "..");
|
||||
|
||||
interface ActionDoc {
|
||||
inputs?: Record<string, { description?: string }>;
|
||||
runs?: { steps?: Array<{ env?: Record<string, string>; run?: string }> };
|
||||
}
|
||||
|
||||
function action(): ActionDoc {
|
||||
return YAML.parse(
|
||||
readFileSync(join(REPO_ROOT, "action.yml"), "utf8"),
|
||||
) as ActionDoc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirrors the guard in action.yml: the resolved working directory must be
|
||||
* the workspace root or beneath it. Accepts absolute paths inside the
|
||||
* workspace (including `sub/../sub`, already resolved by the runner);
|
||||
* rejects anything outside, including `ws/..` (which resolves outside).
|
||||
*/
|
||||
export function isOutsideWorkspace(pwd: string, workspace: string): boolean {
|
||||
const ws = workspace.endsWith("/") ? workspace : `${workspace}/`;
|
||||
const here = pwd.endsWith("/") ? pwd : `${pwd}/`;
|
||||
return !here.startsWith(ws);
|
||||
}
|
||||
|
||||
describe("working-directory guard", () => {
|
||||
test("working-directory is documented as workspace-bound, absolute-inside allowed", () => {
|
||||
const description =
|
||||
action().inputs?.["working-directory"]?.description ?? "";
|
||||
expect(description).toMatch(/GITHUB_WORKSPACE/);
|
||||
expect(description).toMatch(/absolute/);
|
||||
expect(description).not.toMatch(/unsupported/);
|
||||
});
|
||||
|
||||
test("the input travels via IN_WD env, never interpolated into run:", () => {
|
||||
// Built by concatenation (no literal `${{`) so this file stays lint-clean.
|
||||
const expr = "$" + "{{ inputs.working-directory }}";
|
||||
const steps = action().runs?.steps ?? [];
|
||||
const runStep = steps.find(
|
||||
(step) => typeof step.run === "string" && step.run.includes("IN_WD"),
|
||||
);
|
||||
expect(runStep?.env?.IN_WD).toBe(expr);
|
||||
expect(runStep?.run ?? "").not.toContain(expr);
|
||||
});
|
||||
|
||||
test("the run: script pins the resolved-directory guard", () => {
|
||||
const run = (action().runs?.steps ?? []).find(
|
||||
(step) => typeof step.run === "string" && step.run.includes("IN_WD"),
|
||||
)?.run as string;
|
||||
expect(run).toContain('ws="$(cd "$GITHUB_WORKSPACE" && pwd -P)"');
|
||||
expect(run).toContain('case "$(pwd -P)/" in "$ws"/*)');
|
||||
expect(run).toContain('echo "result=fail" >> "$GITHUB_OUTPUT"');
|
||||
expect(run).toContain("exit 2");
|
||||
expect(run).not.toContain("/*|*..*");
|
||||
});
|
||||
|
||||
test("guard helper accepts inside paths and rejects outside ones", () => {
|
||||
const ws = "/home/runner/work/repo/repo";
|
||||
expect(isOutsideWorkspace(ws, ws)).toBe(false);
|
||||
expect(isOutsideWorkspace(`${ws}/plugin`, ws)).toBe(false);
|
||||
expect(isOutsideWorkspace(`${ws}/sub/../sub`, ws)).toBe(false);
|
||||
expect(isOutsideWorkspace("/tmp/evil", ws)).toBe(true);
|
||||
expect(isOutsideWorkspace("/home/runner/work/other/other", ws)).toBe(true);
|
||||
expect(isOutsideWorkspace("/home/runner/work", ws)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,137 @@
|
||||
// OCI-4.4-ancestor: the tier-1 inventory must be exactly the configured set
|
||||
// (pre/post probes plus the plugin under test), and an ancestor
|
||||
// opencode.json(c) must be detected. Pure-helper tests: the fake opencode
|
||||
// binary cannot plant foreign plugins, so the allow-list is pinned here.
|
||||
import { afterAll, describe, expect, test } from "bun:test";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
detectAncestorConfig,
|
||||
expectedInventoryIds,
|
||||
findUnexpectedPlugins,
|
||||
PROBE_PLUGIN_IDS,
|
||||
} from "../../src/tiers/load.ts";
|
||||
import { cleanupTempDirs, tempDir } from "../helpers/tmp.ts";
|
||||
|
||||
afterAll(cleanupTempDirs);
|
||||
|
||||
describe("expectedInventoryIds", () => {
|
||||
test("without a plugin id only the probes are allowed", () => {
|
||||
expect(expectedInventoryIds(undefined)).toEqual([...PROBE_PLUGIN_IDS]);
|
||||
});
|
||||
|
||||
test("the plugin under test joins the probes when known", () => {
|
||||
expect(expectedInventoryIds("fixture-under-test")).toEqual([
|
||||
"opencode-plugin-ci.probe.pre",
|
||||
"opencode-plugin-ci.probe.post",
|
||||
"fixture-under-test",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findUnexpectedPlugins", () => {
|
||||
const pre = "/tool/probe/pre";
|
||||
const post = "/tool/probe/post";
|
||||
const shim = "/out/load/plugin-under-test";
|
||||
const dirs = [pre, shim, post];
|
||||
const ids = expectedInventoryIds("fixture-under-test");
|
||||
|
||||
test("a clean inventory (builtins + probes + fixture) is expected", () => {
|
||||
const plugins = [
|
||||
{ id: "opencode.tool.shell", source: { type: "builtin" } },
|
||||
{ id: "opencode.config.worktree", source: { type: "builtin" } },
|
||||
{
|
||||
id: "opencode-plugin-ci.probe.pre",
|
||||
source: { type: "local", path: `${pre}/server.js` },
|
||||
},
|
||||
{
|
||||
id: "fixture-under-test",
|
||||
source: { type: "local", path: `${shim}/server.js` },
|
||||
},
|
||||
{
|
||||
id: "opencode-plugin-ci.probe.post",
|
||||
source: { type: "local", path: `${post}/server.js` },
|
||||
},
|
||||
];
|
||||
expect(findUnexpectedPlugins(plugins, ids, dirs)).toEqual([]);
|
||||
});
|
||||
|
||||
test("a foreign npm plugin from an ancestor config is reported", () => {
|
||||
const plugins = [
|
||||
{ id: "opencode.tool.shell", source: { type: "builtin" } },
|
||||
{
|
||||
id: "@heretek-ai/opencode-rea",
|
||||
source: { type: "npm", name: "@heretek-ai/opencode-rea" },
|
||||
},
|
||||
{
|
||||
id: "fixture-under-test",
|
||||
source: { type: "local", path: `${shim}/server.js` },
|
||||
},
|
||||
];
|
||||
expect(findUnexpectedPlugins(plugins, ids, dirs)).toEqual([
|
||||
"@heretek-ai/opencode-rea",
|
||||
]);
|
||||
});
|
||||
|
||||
test("a colliding probe id at a foreign path is reported", () => {
|
||||
const plugins = [
|
||||
{
|
||||
id: "opencode-plugin-ci.probe.pre",
|
||||
source: { type: "local", path: "/evil/server.js" },
|
||||
},
|
||||
];
|
||||
expect(findUnexpectedPlugins(plugins, ids, dirs)).toEqual([
|
||||
"opencode-plugin-ci.probe.pre",
|
||||
]);
|
||||
});
|
||||
|
||||
test("a plugin without a verifiable origin fails closed", () => {
|
||||
expect(findUnexpectedPlugins([{ id: "mystery" }], ids, dirs)).toEqual([
|
||||
"mystery",
|
||||
]);
|
||||
});
|
||||
|
||||
test("a missing plugin id leaves only the probes allowed", () => {
|
||||
expect(
|
||||
findUnexpectedPlugins(
|
||||
[{ id: "stranger", source: { type: "npm", name: "stranger" } }],
|
||||
expectedInventoryIds(undefined),
|
||||
dirs,
|
||||
),
|
||||
).toEqual(["stranger"]);
|
||||
});
|
||||
|
||||
test("duplicates are reported once", () => {
|
||||
const plugins = [
|
||||
{ id: "x", source: { type: "npm", name: "x" } },
|
||||
{ id: "x", source: { type: "npm", name: "x" } },
|
||||
];
|
||||
expect(findUnexpectedPlugins(plugins, [], dirs)).toEqual(["x"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectAncestorConfig", () => {
|
||||
test("finds a planted ancestor opencode.json", () => {
|
||||
const root = tempDir("ancestor-guard-");
|
||||
const deep = join(root, "a", "b", "c");
|
||||
mkdirSync(deep, { recursive: true });
|
||||
writeFileSync(join(root, "a", "opencode.json"), "{}");
|
||||
expect(detectAncestorConfig(deep)).toBe(join(root, "a", "opencode.json"));
|
||||
});
|
||||
|
||||
test("finds opencode.jsonc too", () => {
|
||||
const root = tempDir("ancestor-guard-jsonc-");
|
||||
mkdirSync(join(root, "sub"), { recursive: true });
|
||||
writeFileSync(join(root, "opencode.jsonc"), "{}");
|
||||
expect(detectAncestorConfig(join(root, "sub"))).toBe(
|
||||
join(root, "opencode.jsonc"),
|
||||
);
|
||||
});
|
||||
|
||||
test("returns undefined when no ancestor config exists", () => {
|
||||
const root = tempDir("ancestor-clean-");
|
||||
const deep = join(root, "x", "y");
|
||||
mkdirSync(deep, { recursive: true });
|
||||
expect(detectAncestorConfig(deep)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,7 @@ import { describe, expect, test } from "bun:test";
|
||||
import { main } from "../../src/cli.ts";
|
||||
|
||||
describe("cli --version", () => {
|
||||
test("prints 0.1.0 and returns 0", async () => {
|
||||
test("prints 0.1.1 and returns 0", async () => {
|
||||
const lines: string[] = [];
|
||||
const origLog = console.log;
|
||||
console.log = (...args: unknown[]) => {
|
||||
@@ -14,6 +14,6 @@ describe("cli --version", () => {
|
||||
} finally {
|
||||
console.log = origLog;
|
||||
}
|
||||
expect(lines.join("\n")).toContain("0.1.0");
|
||||
expect(lines.join("\n")).toContain("0.1.1");
|
||||
});
|
||||
});
|
||||
@@ -27,8 +27,6 @@ describe("ConfigSchema defaults", () => {
|
||||
agents: [],
|
||||
skills: [],
|
||||
mcp: [],
|
||||
providers: [],
|
||||
models: [],
|
||||
});
|
||||
expect(config.live.keys).toEqual(["OPENCODE_API_KEY"]);
|
||||
expect(config.live.repeat).toBe(3);
|
||||
@@ -36,8 +34,6 @@ describe("ConfigSchema defaults", () => {
|
||||
expect(config.live.budgetUSD).toBe(1);
|
||||
expect(config.live.timeoutMs).toBe(240_000);
|
||||
expect(config.gate.enabled).toBe(true);
|
||||
expect(config.gate.window).toBe(5);
|
||||
expect(config.gate.threshold).toBe(0.9);
|
||||
expect(config.gate.bootstrap).toBe("fail");
|
||||
expect(config.gate.workflow).toBeUndefined();
|
||||
expect(config.logLevel).toBe("info");
|
||||
@@ -52,8 +48,10 @@ describe("ConfigSchema defaults", () => {
|
||||
|
||||
test("versions accepts semver and latest, non-empty only", () => {
|
||||
expect(
|
||||
ConfigSchema.parse({ opencode: { versions: ["2.0.26", "latest"] } })
|
||||
.opencode.versions,
|
||||
ConfigSchema.parse({
|
||||
opencode: { versions: ["2.0.26", "latest"] },
|
||||
gate: { artifact: "matrix-results" },
|
||||
}).opencode.versions,
|
||||
).toEqual(["2.0.26", "latest"]);
|
||||
expect(ConfigSchema.safeParse({ opencode: { versions: [] } }).success).toBe(
|
||||
false,
|
||||
@@ -150,16 +148,25 @@ describe("cross-field rules", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("judgeReply is scripted-only", () => {
|
||||
test("judgeReply is no longer a scenario field", () => {
|
||||
const parsed = ScenarioSchema.safeParse({
|
||||
id: "live-x",
|
||||
mode: "live",
|
||||
prompt: "hi",
|
||||
...scriptedBase,
|
||||
judgeReply: '{"pass":true}',
|
||||
});
|
||||
expect(parsed.success).toBe(false);
|
||||
if (!parsed.success) {
|
||||
expect(parsed.error.issues[0]?.message).toMatch(/"judgeReply"/);
|
||||
expect(parsed.error.issues[0]?.message).toMatch(/judgeReply/);
|
||||
}
|
||||
});
|
||||
|
||||
test("removed config fields fail validation instead of being ignored", () => {
|
||||
for (const raw of [
|
||||
{ expect: { providers: ["p"] } },
|
||||
{ expect: { models: ["m"] } },
|
||||
{ gate: { window: 10 } },
|
||||
{ gate: { threshold: 0.5 } },
|
||||
]) {
|
||||
expect(ConfigSchema.safeParse(raw).success).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -239,11 +246,11 @@ describe("scenario ids and shapes", () => {
|
||||
expect(parsed.success).toBe(true);
|
||||
});
|
||||
|
||||
test("scenario timeoutMs defaults to 120000", () => {
|
||||
test("scenario timeoutMs has no schema default (tiers apply their own)", () => {
|
||||
const parsed = ScenarioSchema.safeParse(scriptedBase);
|
||||
expect(parsed.success).toBe(true);
|
||||
if (parsed.success) {
|
||||
expect(parsed.data.timeoutMs).toBe(120_000);
|
||||
expect(parsed.data.timeoutMs).toBeUndefined();
|
||||
expect(parsed.data.expect.exitCode).toBe(0);
|
||||
expect(parsed.data.expect.noErrors).toBe(true);
|
||||
}
|
||||
|
||||
@@ -76,11 +76,11 @@ describe("resolveEntries", () => {
|
||||
resolve(dir, "custom/entry.ts"),
|
||||
);
|
||||
});
|
||||
test("missing explicit entry falls through to detection", () => {
|
||||
test("missing explicit entry throws instead of falling back", () => {
|
||||
const dir = makeDir();
|
||||
writeFile(dir, "server.ts");
|
||||
expect(resolveEntries(dir, "does/not-exist.ts").server).toBe(
|
||||
resolve(dir, "server.ts"),
|
||||
expect(() => resolveEntries(dir, "does/not-exist.ts")).toThrow(
|
||||
/plugin\.entry "does\/not-exist\.ts" does not exist/,
|
||||
);
|
||||
});
|
||||
test("absolute explicit entry is used as is", () => {
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
// OCI-7.3-evidence-leaks: only env/headers are redacted at the source, so
|
||||
// prompt/tool I/O, stub requests, events and the report get value-side
|
||||
// scrubbing of known live secret values before they are persisted. The
|
||||
// in-memory evidence stays raw so assertions still see — and fail on —
|
||||
// leaked values.
|
||||
import { afterAll, afterEach, describe, expect, test } from "bun:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { renderMarkdown } from "../../src/report/markdown.ts";
|
||||
import type { ResultsFile } from "../../src/report/results.ts";
|
||||
import { type Stub, startStub } from "../../src/stub/stub.ts";
|
||||
import {
|
||||
collectSecretValues,
|
||||
redactSecretValues,
|
||||
} from "../../src/util/redact.ts";
|
||||
import { cleanupTempDirs, tempDir } from "../helpers/tmp.ts";
|
||||
|
||||
afterAll(cleanupTempDirs);
|
||||
|
||||
const CANARY = "sk-test-SYSTEM-SECRET-0001";
|
||||
|
||||
describe("redactSecretValues", () => {
|
||||
test("replaces every occurrence with [redacted]", () => {
|
||||
expect(redactSecretValues(`a ${CANARY} b ${CANARY}`, [CANARY])).toBe(
|
||||
"a [redacted] b [redacted]",
|
||||
);
|
||||
});
|
||||
|
||||
test("longer overlapping values scrub completely", () => {
|
||||
expect(
|
||||
redactSecretValues("xxsecret12xx", ["secret12", "xxsecret12xx"]),
|
||||
).toBe("[redacted]");
|
||||
});
|
||||
|
||||
test("values with regex metacharacters match literally", () => {
|
||||
const value = "a.b*c+d?e(f)g[h]i{j}k|l^m$n";
|
||||
expect(redactSecretValues(`[${value}]`, [value])).toBe("[[redacted]]");
|
||||
});
|
||||
|
||||
test("empty, whitespace-only and short values are skipped", () => {
|
||||
expect(redactSecretValues("abc 123", ["", " ", "123"])).toBe("abc 123");
|
||||
// Anything under 8 chars — including common env noise like "false"
|
||||
// (e.g. TOKENIZERS_PARALLELISM=false) — is never scrubbed.
|
||||
expect(
|
||||
redactSecretValues("Defaults to false; code 1234567", [
|
||||
"false",
|
||||
"1234567",
|
||||
]),
|
||||
).toBe("Defaults to false; code 1234567");
|
||||
});
|
||||
|
||||
test("scrubs each value's JSON-escaped form as well", () => {
|
||||
const value = `sec"ret\\key-123`;
|
||||
expect(value.length).toBeGreaterThanOrEqual(8);
|
||||
const line = JSON.stringify({ content: `print ${value} now` });
|
||||
expect(line).toContain(value.slice(0, 3));
|
||||
// The raw value does not appear literally in the JSON line (quotes
|
||||
// and backslashes are escaped), yet scrubbing must still catch it.
|
||||
expect(line).not.toContain(value);
|
||||
const scrubbed = redactSecretValues(line, [value]);
|
||||
expect(scrubbed).not.toContain(JSON.stringify(value).slice(1, -1));
|
||||
expect(scrubbed).toContain("[redacted]");
|
||||
// And the raw form is still scrubbed where it appears literally.
|
||||
expect(redactSecretValues(`saw ${value} here`, [value])).toBe(
|
||||
"saw [redacted] here",
|
||||
);
|
||||
});
|
||||
|
||||
test("never throws and returns text unchanged on malformed input", () => {
|
||||
expect(
|
||||
redactSecretValues("keep false intact", ["false", 42 as never]),
|
||||
).toBe("keep false intact");
|
||||
expect(
|
||||
redactSecretValues("untouched", undefined as unknown as string[]),
|
||||
).toBe("untouched");
|
||||
});
|
||||
test("clean text and an empty secret list pass through", () => {
|
||||
expect(redactSecretValues("nothing here", [CANARY])).toBe("nothing here");
|
||||
expect(redactSecretValues(`has ${CANARY}`, [])).toBe(`has ${CANARY}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("collectSecretValues", () => {
|
||||
const saved = new Map<string, string | undefined>();
|
||||
|
||||
afterEach(() => {
|
||||
for (const [name, value] of saved) {
|
||||
if (value === undefined) {
|
||||
delete process.env[name];
|
||||
} else {
|
||||
process.env[name] = value;
|
||||
}
|
||||
}
|
||||
saved.clear();
|
||||
});
|
||||
|
||||
function setEnv(name: string, value: string): void {
|
||||
if (!saved.has(name)) {
|
||||
saved.set(
|
||||
name,
|
||||
Object.hasOwn(process.env, name) ? process.env[name] : undefined,
|
||||
);
|
||||
}
|
||||
process.env[name] = value;
|
||||
}
|
||||
|
||||
test("picks up explicit key values and ignores host env", () => {
|
||||
setEnv("OCI_SCRUB_TEST_KEY", CANARY);
|
||||
setEnv("OCI_SCRUB_TEST_PLAIN", "not-a-secret-name");
|
||||
setEnv("TOKENIZERS_PARALLELISM", "false");
|
||||
// Host env is never scanned, even for secret-looking names.
|
||||
expect(collectSecretValues()).toEqual([]);
|
||||
const values = collectSecretValues({ MY_API: "explicit-key-value" });
|
||||
expect(values).toContain("explicit-key-value");
|
||||
expect(values).not.toContain(CANARY);
|
||||
expect(values).not.toContain("not-a-secret-name");
|
||||
expect(values).not.toContain("false");
|
||||
});
|
||||
|
||||
test("an explicitly passed value is picked up even if secret-named", () => {
|
||||
setEnv("OCI_SCRUB_TEST_KEY", CANARY);
|
||||
expect(collectSecretValues({ MY_API: CANARY })).toContain(CANARY);
|
||||
});
|
||||
});
|
||||
|
||||
describe("evidence scrubbing", () => {
|
||||
const stubs: Stub[] = [];
|
||||
afterEach(async () => {
|
||||
while (stubs.length > 0) {
|
||||
await stubs.pop()?.stop();
|
||||
}
|
||||
});
|
||||
|
||||
async function post(url: string, body: unknown): Promise<unknown> {
|
||||
const res = await fetch(`${url}/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ ...(body as object), stream: false }),
|
||||
});
|
||||
return res.json();
|
||||
}
|
||||
|
||||
test("stub recordFile scrubs secret values but memory stays raw", async () => {
|
||||
const dir = tempDir("evidence-scrub-");
|
||||
const recordFile = join(dir, "stub-requests.jsonl");
|
||||
const stub = await startStub({
|
||||
script: [{ text: "done" }],
|
||||
recordFile,
|
||||
redactValues: [CANARY],
|
||||
});
|
||||
stubs.push(stub);
|
||||
await post(stub.url, {
|
||||
messages: [{ role: "user", content: `print ${CANARY}` }],
|
||||
tools: [{ type: "function", function: { name: "read" } }],
|
||||
});
|
||||
const persisted = readFileSync(recordFile, "utf8");
|
||||
expect(persisted).not.toContain(CANARY);
|
||||
expect(persisted).toContain("[redacted]");
|
||||
// Assertions evaluate the in-memory record, so the leak is still
|
||||
// visible (and catchable) there.
|
||||
expect(JSON.stringify(stub.requests[0]?.body)).toContain(CANARY);
|
||||
});
|
||||
|
||||
test("host env noise (TOKENIZERS_PARALLELISM=false) leaves evidence JSONL valid", async () => {
|
||||
const had = Object.hasOwn(process.env, "TOKENIZERS_PARALLELISM")
|
||||
? process.env.TOKENIZERS_PARALLELISM
|
||||
: undefined;
|
||||
process.env.TOKENIZERS_PARALLELISM = "false";
|
||||
try {
|
||||
const dir = tempDir("evidence-scrub-r2-");
|
||||
const recordFile = join(dir, "stub-requests.jsonl");
|
||||
const stub = await startStub({
|
||||
script: [{ text: "Defaults to false." }],
|
||||
recordFile,
|
||||
redactValues: collectSecretValues(),
|
||||
});
|
||||
stubs.push(stub);
|
||||
await post(stub.url, {
|
||||
messages: [{ role: "user", content: "Defaults to false." }],
|
||||
tools: [{ type: "function", function: { name: "read" } }],
|
||||
});
|
||||
const persisted = readFileSync(recordFile, "utf8");
|
||||
expect(persisted).toContain("false");
|
||||
expect(persisted).not.toContain("[redacted]");
|
||||
for (const line of persisted.trim().split("\n")) {
|
||||
JSON.parse(line);
|
||||
}
|
||||
} finally {
|
||||
if (had === undefined) {
|
||||
delete process.env.TOKENIZERS_PARALLELISM;
|
||||
} else {
|
||||
process.env.TOKENIZERS_PARALLELISM = had;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("report markdown scrubs secret values from failures", () => {
|
||||
const results: ResultsFile = {
|
||||
version: 1,
|
||||
at: "2026-10-18T00:00:00.000Z",
|
||||
tool: "test",
|
||||
plugin: "test",
|
||||
results: [
|
||||
{
|
||||
id: "leaky",
|
||||
tier: "scripted",
|
||||
opencode: "2.0.26",
|
||||
pass: false,
|
||||
failures: [`text: expected not to contain "x", got "${CANARY}".`],
|
||||
durationMs: 1,
|
||||
textExcerpt: `saw ${CANARY}`,
|
||||
},
|
||||
],
|
||||
};
|
||||
const raw = renderMarkdown(results, { allowedVersions: [] });
|
||||
expect(raw).toContain(CANARY);
|
||||
const scrubbed = redactSecretValues(raw, [CANARY]);
|
||||
expect(scrubbed).not.toContain(CANARY);
|
||||
expect(scrubbed).toContain("[redacted]");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,80 @@
|
||||
// OCI-2.5-files-symlink: `files:` assertions resolved symlinks by following
|
||||
// them, so `linketc -> /etc` made `linketc/passwd` read host files into
|
||||
// `got` failure strings. After the lexical check, `evaluateFiles` resolves
|
||||
// the real path and re-checks containment, failing closed on realpath
|
||||
// errors. A legit `sub/../ok.txt` (lexically inside, really inside) still
|
||||
// passes.
|
||||
import { afterAll, describe, expect, test } from "bun:test";
|
||||
import { mkdirSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { type Evidence, evaluate } from "../../src/assert/expect.ts";
|
||||
import type { ScenarioExpect } from "../../src/config/schema.ts";
|
||||
import { ProbeLog } from "../../src/probe/read.ts";
|
||||
import type { RunOutput } from "../../src/sandbox/sandbox.ts";
|
||||
import { cleanupTempDirs, tempDir } from "../helpers/tmp.ts";
|
||||
|
||||
afterAll(cleanupTempDirs);
|
||||
|
||||
function evidence(projectDir: string): Evidence {
|
||||
return {
|
||||
run: {
|
||||
code: 0,
|
||||
timedOut: false,
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
events: [],
|
||||
durationMs: 1,
|
||||
} as RunOutput,
|
||||
probe: new ProbeLog([]),
|
||||
stubRequests: [],
|
||||
projectDir,
|
||||
mode: "scripted",
|
||||
};
|
||||
}
|
||||
|
||||
function filesExpect(files: ScenarioExpect["files"]): ScenarioExpect {
|
||||
return { exitCode: 0, noErrors: true, files };
|
||||
}
|
||||
|
||||
describe("files symlink containment", () => {
|
||||
test("a symlink escaping the project dir is rejected", () => {
|
||||
const dir = tempDir("files-symlink-");
|
||||
symlinkSync("/etc", join(dir, "linketc"));
|
||||
const failures = evaluate(
|
||||
filesExpect([{ path: "linketc/passwd", exists: true }]),
|
||||
evidence(dir),
|
||||
);
|
||||
expect(failures.join("\n")).toMatch(/escapes the project dir/);
|
||||
});
|
||||
|
||||
test("a symlink read for content is rejected, not exfiltrated", () => {
|
||||
const dir = tempDir("files-symlink-");
|
||||
symlinkSync("/etc", join(dir, "linketc"));
|
||||
const failures = evaluate(
|
||||
filesExpect([{ path: "linketc/hostname", contains: "x" }]),
|
||||
evidence(dir),
|
||||
);
|
||||
expect(failures.join("\n")).toMatch(/escapes the project dir/);
|
||||
expect(failures.join("\n")).not.toContain("no such file");
|
||||
});
|
||||
|
||||
test("a legit sub/../ path inside the project still passes", () => {
|
||||
const dir = tempDir("files-symlink-");
|
||||
mkdirSync(join(dir, "sub"), { recursive: true });
|
||||
writeFileSync(join(dir, "ok.txt"), "hello");
|
||||
const failures = evaluate(
|
||||
filesExpect([{ path: "sub/../ok.txt", exists: true }]),
|
||||
evidence(dir),
|
||||
);
|
||||
expect(failures).toEqual([]);
|
||||
});
|
||||
|
||||
test("a plain lexical escape is still rejected", () => {
|
||||
const dir = tempDir("files-symlink-");
|
||||
const failures = evaluate(
|
||||
filesExpect([{ path: "../evil.txt", exists: false }]),
|
||||
evidence(dir),
|
||||
);
|
||||
expect(failures.join("\n")).toMatch(/escapes the project dir/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,27 @@
|
||||
// Review S6 and S7: pull request runs read the base branch's gate history,
|
||||
// and `run --gate-artifact` (what the Action passes) names the history
|
||||
// artifact.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { parseRunArgs } from "../../src/cli.ts";
|
||||
import { historyBranch } from "../../src/gate/history.ts";
|
||||
|
||||
describe("gate wiring", () => {
|
||||
test("pull request runs read the base branch; other runs their own ref", () => {
|
||||
expect(
|
||||
historyBranch({ GITHUB_REF_NAME: "4/merge", GITHUB_BASE_REF: "main" }),
|
||||
).toBe("main");
|
||||
expect(
|
||||
historyBranch({ GITHUB_REF_NAME: "main", GITHUB_BASE_REF: "" }),
|
||||
).toBe("main");
|
||||
expect(historyBranch({ GITHUB_REF_NAME: "release" })).toBe("release");
|
||||
expect(historyBranch({})).toBeUndefined();
|
||||
});
|
||||
|
||||
test("--gate-artifact is parsed in both spellings", () => {
|
||||
expect(parseRunArgs(["--gate-artifact", "leg-a"]).gateArtifact).toBe(
|
||||
"leg-a",
|
||||
);
|
||||
expect(parseRunArgs(["--gate-artifact=leg-b"]).gateArtifact).toBe("leg-b");
|
||||
expect(parseRunArgs([]).gateArtifact).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
// The live attempt loop (k-of-n, early stop, judge gating, budget), driven by
|
||||
// the fake opencode binary so no real model or key is needed.
|
||||
import { afterAll, describe, expect, test } from "bun:test";
|
||||
import { join, resolve } from "node:path";
|
||||
import { ConfigSchema, ScenarioSchema } from "../../src/config/schema.ts";
|
||||
import { type LiveBudget, runLiveScenario } from "../../src/tiers/live.ts";
|
||||
import {
|
||||
type FakeRun,
|
||||
fakeRunCount,
|
||||
makeFakeOpencode,
|
||||
} from "../helpers/fake-opencode.ts";
|
||||
import { cleanupTempDirs, tempDir } from "../helpers/tmp.ts";
|
||||
|
||||
const PLUGIN = resolve(import.meta.dir, "../fixtures/plugins/tool-basic");
|
||||
const ENTRY = join(PLUGIN, "src/server.ts");
|
||||
|
||||
const PASS: FakeRun = {
|
||||
text: "echo:hi",
|
||||
cost: 0.01,
|
||||
toolCalls: [
|
||||
{
|
||||
tool: "echo_tool",
|
||||
input: { text: "hi" },
|
||||
result: { content: "echo:hi" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const FAIL: FakeRun = { text: "I did not use the tool", cost: 0.01 };
|
||||
|
||||
async function live(
|
||||
runs: FakeRun[],
|
||||
opts: { budget?: LiveBudget; judge?: boolean } = {},
|
||||
) {
|
||||
const dir = tempDir("live-loop-");
|
||||
const bin = makeFakeOpencode(dir, { runs });
|
||||
const config = ConfigSchema.parse({ live: { model: "fake/model" } });
|
||||
const scenario = ScenarioSchema.parse({
|
||||
id: "live-smoke",
|
||||
mode: "live",
|
||||
prompt: "Use the echo tool on the word hi",
|
||||
expect: { tools: [{ name: "echo_tool" }] },
|
||||
...(opts.judge ? { judge: { rubric: "The reply shows echo:hi." } } : {}),
|
||||
});
|
||||
const result = await runLiveScenario({
|
||||
scenario,
|
||||
config,
|
||||
pluginPath: PLUGIN,
|
||||
serverEntry: ENTRY,
|
||||
opencodeBin: bin,
|
||||
outDir: join(dir, "out"),
|
||||
version: "2.0.26",
|
||||
budget: opts.budget ?? { spent: 0, limit: 1 },
|
||||
model: "fake/model",
|
||||
keys: { OPENCODE_API_KEY: "fake" },
|
||||
});
|
||||
return { result, runs: fakeRunCount(dir) };
|
||||
}
|
||||
|
||||
afterAll(cleanupTempDirs);
|
||||
|
||||
describe("live attempt loop", () => {
|
||||
test("pass, fail, pass passes after 3 attempts", async () => {
|
||||
const { result, runs } = await live([PASS, FAIL, PASS]);
|
||||
expect(result.pass).toBe(true);
|
||||
expect(result.skipped).toBeUndefined();
|
||||
expect(result.attempts?.map((a) => a.pass)).toEqual([true, false, true]);
|
||||
expect(runs).toBe(3);
|
||||
});
|
||||
|
||||
test("stops early after 2 passes", async () => {
|
||||
const { result, runs } = await live([PASS, PASS, PASS]);
|
||||
expect(result.pass).toBe(true);
|
||||
expect(result.attempts).toHaveLength(2);
|
||||
expect(runs).toBe(2);
|
||||
});
|
||||
|
||||
test("stops early once passing is impossible", async () => {
|
||||
const { result, runs } = await live([FAIL, FAIL, PASS]);
|
||||
expect(result.pass).toBe(false);
|
||||
expect(result.attempts).toHaveLength(2);
|
||||
expect(runs).toBe(2);
|
||||
expect((result.failures ?? []).join("\n")).toContain(
|
||||
"0 of 2 attempt(s) passed, needed 2 of 3",
|
||||
);
|
||||
});
|
||||
|
||||
test("the judge runs only after the deterministic checks pass", async () => {
|
||||
const verdict: FakeRun = { text: '{"pass":true,"reason":"ok"}' };
|
||||
const { result, runs } = await live([FAIL, FAIL], { judge: true });
|
||||
expect(result.pass).toBe(false);
|
||||
expect(result.judge).toBeUndefined();
|
||||
expect(runs).toBe(2);
|
||||
const judged = await live([PASS, verdict, PASS, verdict], { judge: true });
|
||||
expect(judged.result.pass).toBe(true);
|
||||
expect(judged.result.judge).toEqual({ pass: true, reason: "ok" });
|
||||
expect(judged.runs).toBe(4);
|
||||
});
|
||||
|
||||
test("running out of budget after a failed attempt is a failure, not a skip", async () => {
|
||||
const { result, runs } = await live([{ ...FAIL, cost: 1.5 }]);
|
||||
expect(runs).toBe(1);
|
||||
expect(result.skipped).toBeUndefined();
|
||||
expect(result.pass).toBe(false);
|
||||
expect(result.failures?.[0]).toBe(
|
||||
"live scenario live-smoke: 0 of 1 attempt(s) passed, needed 2 of 3 (budget exhausted ($1.50 of $1.00)).",
|
||||
);
|
||||
});
|
||||
|
||||
test("running out of budget before any attempt is a skip", async () => {
|
||||
const { result, runs } = await live([PASS], {
|
||||
budget: { spent: 1, limit: 1 },
|
||||
});
|
||||
expect(runs).toBe(0);
|
||||
expect(result.pass).toBe(true);
|
||||
expect(result.skipped).toBe("budget exhausted ($1.00 of $1.00)");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
// OCI-4.3-order: tier-1 sandbox plugin order `[pre, plugin, post]`. Hook
|
||||
// order follows the config `plugins` array, so the pre-probe records what
|
||||
// the plugin under test receives and the post-probe records what it
|
||||
// produces. A `[post, shim, pre]` swap sabotage silently inverts pre/post
|
||||
// attribution for tier-1 evidence; this test pins the order at the single
|
||||
// construction site (`buildLoadPlugins` in `src/tiers/load.ts`, called by
|
||||
// `runLoad`). NOTE: a `runLoad`-level hook-order e2e is inapplicable — tier 1
|
||||
// runs one `{"text": "ok"}` stub turn with no tool calls, so tier-1
|
||||
// `probe.jsonl` holds no hook records to compare; the scripted copy of this
|
||||
// order IS pinned by the `hook-mutate-pass` e2e.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { buildLoadPlugins } from "../../src/tiers/load.ts";
|
||||
|
||||
const REPO_ROOT = join(import.meta.dir, "..", "..");
|
||||
|
||||
describe("tier-1 plugin order", () => {
|
||||
test("buildLoadPlugins returns [pre, shim, post]", () => {
|
||||
const plugins = buildLoadPlugins(
|
||||
"probe/pre",
|
||||
"shim",
|
||||
"probe/post",
|
||||
{ opt: 1 },
|
||||
"out/probe.jsonl",
|
||||
);
|
||||
expect(plugins.map((p) => p.package)).toEqual([
|
||||
"probe/pre",
|
||||
"shim",
|
||||
"probe/post",
|
||||
]);
|
||||
expect(plugins[0]?.options).toEqual({
|
||||
out: "out/probe.jsonl",
|
||||
phase: "pre",
|
||||
});
|
||||
expect(plugins[1]?.options).toEqual({ opt: 1 });
|
||||
expect(plugins[2]?.options).toEqual({
|
||||
out: "out/probe.jsonl",
|
||||
phase: "post",
|
||||
});
|
||||
});
|
||||
|
||||
test("runLoad builds its sandbox from buildLoadPlugins in order", () => {
|
||||
// Fail closed on shape drift: if runLoad ever stops calling the pinned
|
||||
// constructor, this test — not tier 1 — must say so.
|
||||
const source = readFileSync(
|
||||
join(REPO_ROOT, "src", "tiers", "load.ts"),
|
||||
"utf8",
|
||||
);
|
||||
expect(source).toContain("plugins: buildLoadPlugins(");
|
||||
const call = source.slice(source.indexOf("plugins: buildLoadPlugins("));
|
||||
const pre = call.indexOf("pre,");
|
||||
const shim = call.indexOf("shimDir,");
|
||||
const post = call.indexOf("post,");
|
||||
expect(pre).toBeGreaterThanOrEqual(0);
|
||||
expect(shim).toBeGreaterThan(pre);
|
||||
expect(post).toBeGreaterThan(shim);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,32 @@
|
||||
// Review S4: a value that was never recorded fails every matcher except
|
||||
// `exists: false`, so a misspelled field cannot pass a negative matcher.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { check } from "../../src/assert/matchers.ts";
|
||||
|
||||
describe("matchers on a value that was never recorded", () => {
|
||||
test("notContains fails instead of passing vacuously", () => {
|
||||
expect(check(undefined, { notContains: "secret" }, "input.comand")).toEqual(
|
||||
["input.comand: no value was recorded (check the field name)."],
|
||||
);
|
||||
});
|
||||
|
||||
test("notMatches and equals fail the same way", () => {
|
||||
expect(check(undefined, { notMatches: "x" }, "a")).toEqual([
|
||||
"a: no value was recorded (check the field name).",
|
||||
]);
|
||||
expect(check(undefined, { equals: "x" }, "a")).toEqual([
|
||||
"a: no value was recorded (check the field name).",
|
||||
]);
|
||||
});
|
||||
|
||||
test("exists: false passes, exists: true fails", () => {
|
||||
expect(check(undefined, { exists: false }, "x")).toEqual([]);
|
||||
expect(check(undefined, { exists: true }, "x")).toEqual([
|
||||
"x: expected value to exist, got undefined.",
|
||||
]);
|
||||
});
|
||||
|
||||
test("null counts as recorded", () => {
|
||||
expect(check(null, { notContains: "secret" }, "x")).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
// Review S5: hooks record a namespaced tool as <namespace>_<name>, while
|
||||
// catalogs use tools.ns.name and inner calls ns.name. Assertions accept all
|
||||
// three spellings, and a bare name fails with a hint.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { tmpdir } from "node:os";
|
||||
import { evaluate, normalizeToolName } from "../../src/assert/expect.ts";
|
||||
import { ScenarioSchema } from "../../src/config/schema.ts";
|
||||
import { ProbeLog, type ProbeRecord } from "../../src/probe/read.ts";
|
||||
|
||||
describe("namespaced tool names", () => {
|
||||
test("normalizeToolName maps every spelling to the effective name", () => {
|
||||
expect(normalizeToolName("ns_spike_echo")).toBe("ns_spike_echo");
|
||||
expect(normalizeToolName("ns.spike_echo")).toBe("ns_spike_echo");
|
||||
expect(normalizeToolName("tools.ns.spike_echo")).toBe("ns_spike_echo");
|
||||
expect(normalizeToolName('tools.ns["my-tool"]')).toBe("ns_my-tool");
|
||||
expect(normalizeToolName("echo_tool")).toBe("echo_tool");
|
||||
});
|
||||
|
||||
function evidence(tool: string) {
|
||||
const records: ProbeRecord[] = [
|
||||
{
|
||||
kind: "tool.before",
|
||||
phase: "post",
|
||||
seq: 0,
|
||||
tool,
|
||||
callID: "c1",
|
||||
input: { text: "hi" },
|
||||
},
|
||||
{
|
||||
kind: "tool.after",
|
||||
phase: "post",
|
||||
seq: 1,
|
||||
tool,
|
||||
callID: "c1",
|
||||
status: "completed",
|
||||
result: { content: "echo:hi" },
|
||||
},
|
||||
];
|
||||
return {
|
||||
run: {
|
||||
code: 0,
|
||||
timedOut: false,
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
events: [],
|
||||
durationMs: 0,
|
||||
},
|
||||
probe: new ProbeLog(records),
|
||||
stubRequests: [],
|
||||
projectDir: tmpdir(),
|
||||
mode: "scripted" as const,
|
||||
};
|
||||
}
|
||||
|
||||
function expectBlock(block: unknown) {
|
||||
return ScenarioSchema.parse({
|
||||
id: "ns",
|
||||
mode: "scripted",
|
||||
prompt: "hi",
|
||||
script: [],
|
||||
expect: block,
|
||||
}).expect;
|
||||
}
|
||||
|
||||
test("the dotted catalog form matches the recorded effective name", () => {
|
||||
const failures = evaluate(
|
||||
expectBlock({ tools: [{ name: "ns.spike_echo", status: "completed" }] }),
|
||||
evidence("ns_spike_echo"),
|
||||
);
|
||||
expect(failures).toEqual([]);
|
||||
});
|
||||
|
||||
test("a bare name fails with a hint naming the namespaced form", () => {
|
||||
const failures = evaluate(
|
||||
expectBlock({ tools: [{ name: "spike_echo" }] }),
|
||||
evidence("ns_spike_echo"),
|
||||
);
|
||||
expect(failures.join("\n")).toContain("did you mean ns_spike_echo?");
|
||||
});
|
||||
|
||||
test("noTools catches the dotted form", () => {
|
||||
const failures = evaluate(
|
||||
expectBlock({ noTools: ["ns.spike_echo"] }),
|
||||
evidence("ns_spike_echo"),
|
||||
);
|
||||
expect(failures.join("\n")).toContain(
|
||||
'expected tool "ns.spike_echo" not to be called',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
// OCI-2.5-noerrors: the `?? true` fallback in `evaluate` is the only thing
|
||||
// standing between an opencode `error` event and a silent pass when
|
||||
// `expect.noErrors` is unset. The user-facing default is schema-pinned
|
||||
// (`noErrors: true`); this test pins the fallback itself, plus the fact that
|
||||
// an explicit `noErrors: false` still lets error events through. NOTE: no
|
||||
// scripted e2e fixture is added — no stub turn emits `error` events on a
|
||||
// passing path, so the unit pin is the whole coverage.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { type Evidence, evaluate } from "../../src/assert/expect.ts";
|
||||
import type { ScenarioExpect } from "../../src/config/schema.ts";
|
||||
import type { RunEvent } from "../../src/opencode/events.ts";
|
||||
import { ProbeLog } from "../../src/probe/read.ts";
|
||||
import type { RunOutput } from "../../src/sandbox/sandbox.ts";
|
||||
|
||||
function run(events: RunEvent[]): RunOutput {
|
||||
return {
|
||||
code: 0,
|
||||
timedOut: false,
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
events,
|
||||
durationMs: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function evidence(events: RunEvent[]): Evidence {
|
||||
return {
|
||||
run: run(events),
|
||||
probe: new ProbeLog([]),
|
||||
stubRequests: [],
|
||||
projectDir: mkdtempSync(join(tmpdir(), "noerrors-")),
|
||||
mode: "scripted",
|
||||
};
|
||||
}
|
||||
|
||||
const errorEvents: RunEvent[] = [{ type: "error", error: { message: "boom" } }];
|
||||
|
||||
describe("noErrors fallback", () => {
|
||||
test("an unset noErrors still fails on an error event", () => {
|
||||
// `noErrors` is required in the parsed output type, so unset is
|
||||
// modeled by deleting it: this is the `?? true` fallback path.
|
||||
const scenario = { exitCode: 0, noErrors: true } as ScenarioExpect;
|
||||
delete (scenario as unknown as { noErrors?: boolean }).noErrors;
|
||||
expect("noErrors" in scenario).toBe(false);
|
||||
const failures = evaluate(scenario, evidence(errorEvents));
|
||||
expect(failures.join("\n")).toContain("error event");
|
||||
});
|
||||
|
||||
test("an explicit noErrors: true fails on an error event", () => {
|
||||
const failures = evaluate(
|
||||
{ exitCode: 0, noErrors: true },
|
||||
evidence(errorEvents),
|
||||
);
|
||||
expect(failures.join("\n")).toContain("error event");
|
||||
});
|
||||
|
||||
test("noErrors: false lets error events through", () => {
|
||||
const failures = evaluate(
|
||||
{ exitCode: 0, noErrors: false },
|
||||
evidence(errorEvents),
|
||||
);
|
||||
expect(failures).toEqual([]);
|
||||
});
|
||||
|
||||
test("a clean run passes with noErrors unset", () => {
|
||||
const scenario = { exitCode: 0, noErrors: true } as ScenarioExpect;
|
||||
delete (scenario as unknown as { noErrors?: boolean }).noErrors;
|
||||
const failures = evaluate(scenario, evidence([]));
|
||||
expect(failures).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,71 @@
|
||||
// OCI-6.4-files-guard: the published tarball must ship every load-bearing
|
||||
// file. `probe/` is required at runtime (the sandbox `plugins` array points
|
||||
// at it), so dropping it from `files` would ship a tarball that fails every
|
||||
// tier with a confusing plugin-load error. This pins `files`/`bin` and the
|
||||
// packed contents; re-drop `"probe"` from `files` and this goes red.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
const REPO_ROOT = join(import.meta.dir, "..", "..");
|
||||
|
||||
const EXPECTED_FILES = [
|
||||
"bin",
|
||||
"src",
|
||||
"probe",
|
||||
"schema",
|
||||
"examples",
|
||||
"README.md",
|
||||
"LICENSE",
|
||||
"NOTICE",
|
||||
];
|
||||
|
||||
const REQUIRED_TARBALL_FILES = [
|
||||
"probe/core.js",
|
||||
"probe/pre/server.js",
|
||||
"probe/post/server.js",
|
||||
"schema/config.json",
|
||||
"bin/opencode-plugin-ci.ts",
|
||||
];
|
||||
|
||||
function packageJson(): {
|
||||
files?: unknown;
|
||||
bin?: unknown;
|
||||
} {
|
||||
return JSON.parse(readFileSync(join(REPO_ROOT, "package.json"), "utf8")) as {
|
||||
files?: unknown;
|
||||
bin?: unknown;
|
||||
};
|
||||
}
|
||||
|
||||
describe("packaging guard", () => {
|
||||
test("package.json files is exactly the documented set", () => {
|
||||
expect(packageJson().files).toEqual(EXPECTED_FILES);
|
||||
});
|
||||
|
||||
test("bin maps opencode-plugin-ci to the CLI script", () => {
|
||||
expect(packageJson().bin).toEqual({
|
||||
"opencode-plugin-ci": "bin/opencode-plugin-ci.ts",
|
||||
});
|
||||
});
|
||||
|
||||
test("npm pack ships the load-bearing files", async () => {
|
||||
const proc = Bun.spawn(["npm", "pack", "--dry-run"], {
|
||||
cwd: REPO_ROOT,
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
stdin: "ignore",
|
||||
});
|
||||
const [stdout, stderr] = await Promise.all([
|
||||
new Response(proc.stdout as ReadableStream).text(),
|
||||
new Response(proc.stderr as ReadableStream).text(),
|
||||
]);
|
||||
const code = await proc.exited;
|
||||
expect(code).toBe(0);
|
||||
const output = `${stdout}\n${stderr}`;
|
||||
const missing = REQUIRED_TARBALL_FILES.filter(
|
||||
(file) => !output.includes(file),
|
||||
);
|
||||
expect(missing).toEqual([]);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
// OCI-6.1-allowed-wiring: `partitionVersions` splits the opencode matrix
|
||||
// into the required first version and the allowed-to-fail rest, and `runCi`
|
||||
// feeds the rest to the report renderer. Dropping that wiring (the
|
||||
// `slice(1)` → `[]` sabotage, which renders allowed legs as required)
|
||||
// goes red here.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { partitionVersions } from "../../src/run.ts";
|
||||
|
||||
describe("partitionVersions", () => {
|
||||
test("first version is required, the rest are allowed", () => {
|
||||
expect(partitionVersions(["2.0.26", "latest"])).toEqual({
|
||||
required: "2.0.26",
|
||||
allowedVersions: ["latest"],
|
||||
});
|
||||
});
|
||||
|
||||
test("a single version has no allowed versions", () => {
|
||||
expect(partitionVersions(["2.0.26"])).toEqual({
|
||||
required: "2.0.26",
|
||||
allowedVersions: [],
|
||||
});
|
||||
});
|
||||
|
||||
test("three versions keep order", () => {
|
||||
expect(partitionVersions(["2.0.26", "2.0.32", "latest"])).toEqual({
|
||||
required: "2.0.26",
|
||||
allowedVersions: ["2.0.32", "latest"],
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
// OCI-1.4-cap: the probe's 200-item array cap truncated tier-1 inventory
|
||||
// lists, so a tool union over 200 entries failed tier 1 on clipping rather
|
||||
// than on the plugin. Inventory records bypass the array cap (strings are
|
||||
// still clipped). This test drives `makeProbe` with a 201-tool inventory and
|
||||
// asserts every entry — including the last — survives to `probe.jsonl`.
|
||||
import { afterAll, describe, expect, test } from "bun:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
// @ts-expect-error: probe/core.js is intentionally untyped JS (excluded
|
||||
// from tsconfig: TS7016); this test pins its runtime inventory framing.
|
||||
import { makeProbe } from "../../probe/core.js";
|
||||
import { readProbe } from "../../src/probe/read.ts";
|
||||
import { cleanupTempDirs, tempDir } from "../helpers/tmp.ts";
|
||||
|
||||
afterAll(cleanupTempDirs);
|
||||
|
||||
type Handler = (input: Record<string, unknown>) => unknown;
|
||||
|
||||
async function inventoryWithTools(tools: string[]): Promise<string> {
|
||||
const dir = tempDir("probe-cap-");
|
||||
const out = join(dir, "probe.jsonl");
|
||||
const handlers = new Map<string, Handler>();
|
||||
const ctx = {
|
||||
options: { out },
|
||||
tool: {
|
||||
hook: async (name: string, fn: Handler) => {
|
||||
handlers.set(`tool.${name}`, fn);
|
||||
},
|
||||
list: async () => tools.map((id) => ({ id })),
|
||||
},
|
||||
shell: {
|
||||
hook: async (name: string, fn: Handler) => {
|
||||
handlers.set(`shell.${name}`, fn);
|
||||
},
|
||||
},
|
||||
session: {
|
||||
hook: async (name: string, fn: Handler) => {
|
||||
handlers.set(`session.${name}`, fn);
|
||||
},
|
||||
},
|
||||
event: { subscribe: () => (async function* () {})() },
|
||||
plugin: { list: async () => [] },
|
||||
command: { list: async () => [] },
|
||||
agent: { list: async () => [] },
|
||||
skill: { list: async () => [] },
|
||||
mcp: { list: async () => [] },
|
||||
};
|
||||
await makeProbe("post").setup(ctx);
|
||||
await handlers.get("session.prompt")?.({ prompt: "hi" });
|
||||
return out;
|
||||
}
|
||||
|
||||
describe("probe inventory cap bypass", () => {
|
||||
test("a 201-tool inventory keeps every tool, including the last", async () => {
|
||||
const tools = Array.from({ length: 200 }, (_, i) => `tool_${i}`);
|
||||
tools.push("echo_tool");
|
||||
const out = await inventoryWithTools(tools);
|
||||
const probe = readProbe(out);
|
||||
expect(probe.inventory?.tools).toHaveLength(201);
|
||||
expect(probe.inventory?.tools).toContain("echo_tool");
|
||||
expect(probe.inventory?.tools?.[200]).toBe("echo_tool");
|
||||
});
|
||||
|
||||
test("long strings are still clipped inside inventory records", async () => {
|
||||
const out = await inventoryWithTools([`x`.repeat(5000)]);
|
||||
const lines = readFileSync(out, "utf8").split("\n").filter(Boolean);
|
||||
const inventory = lines
|
||||
.map((line) => JSON.parse(line) as { kind?: string })
|
||||
.find((record) => record.kind === "inventory");
|
||||
expect(JSON.stringify(inventory)).toContain("…[+");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
// OCI-3.3-meta: probe metadata-last ordering. `writeRecord` spreads metadata
|
||||
// LAST (`{ ...payload, phase, seq, t, kind }`), so a payload field can never
|
||||
// overwrite `kind`/`phase`. This test writes a record whose data contains
|
||||
// colliding `kind`/`phase`/`seq` and asserts the parsed line keeps the
|
||||
// metadata values. A payload-wins sabotage (`{ phase, seq, t, kind,
|
||||
// ...payload }`) goes red here.
|
||||
import { afterAll, describe, expect, test } from "bun:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
// @ts-expect-error: probe/core.js is intentionally untyped JS (excluded
|
||||
// from tsconfig: TS7016); this test pins its runtime record framing.
|
||||
import { makeProbe, writeRecord } from "../../probe/core.js";
|
||||
import { cleanupTempDirs, tempDir } from "../helpers/tmp.ts";
|
||||
|
||||
afterAll(cleanupTempDirs);
|
||||
|
||||
function outFile(): string {
|
||||
return join(tempDir("probe-meta-"), "probe.jsonl");
|
||||
}
|
||||
|
||||
function parsed(out: string): Record<string, unknown>[] {
|
||||
return readFileSync(out, "utf8")
|
||||
.split("\n")
|
||||
.filter((line) => line.trim() !== "")
|
||||
.map((line) => JSON.parse(line) as Record<string, unknown>);
|
||||
}
|
||||
|
||||
describe("probe metadata-last", () => {
|
||||
test("colliding payload keys never overwrite kind/phase/seq", () => {
|
||||
const out = outFile();
|
||||
writeRecord(
|
||||
out,
|
||||
"tool.before",
|
||||
{ tool: "shell", kind: "forged", phase: "forged", seq: 999, t: 1 },
|
||||
"pre",
|
||||
7,
|
||||
);
|
||||
const [record] = parsed(out);
|
||||
expect(record?.kind).toBe("tool.before");
|
||||
expect(record?.phase).toBe("pre");
|
||||
expect(record?.seq).toBe(7);
|
||||
expect(typeof record?.t).toBe("number");
|
||||
expect(record?.t).not.toBe(1);
|
||||
// The payload itself survives, nested under its own keys.
|
||||
expect(record?.tool).toBe("shell");
|
||||
});
|
||||
|
||||
test("makeProbe lines carry metadata from the probe, not the payload", async () => {
|
||||
const out = outFile();
|
||||
const handlers = new Map<string, (input: unknown) => unknown>();
|
||||
const ctx = {
|
||||
options: { out },
|
||||
tool: {
|
||||
hook: async (name: string, fn: (input: unknown) => unknown) => {
|
||||
handlers.set(`tool.${name}`, fn);
|
||||
},
|
||||
},
|
||||
shell: {
|
||||
hook: async (name: string, fn: (input: unknown) => unknown) => {
|
||||
handlers.set(`shell.${name}`, fn);
|
||||
},
|
||||
},
|
||||
session: {
|
||||
hook: async (name: string, fn: (input: unknown) => unknown) => {
|
||||
handlers.set(`session.${name}`, fn);
|
||||
},
|
||||
},
|
||||
event: { subscribe: () => (async function* () {})() },
|
||||
plugin: { list: async () => [] },
|
||||
};
|
||||
await makeProbe("pre").setup(ctx);
|
||||
const before = handlers.get("tool.execute.before");
|
||||
await before?.({
|
||||
tool: "shell",
|
||||
id: "c1",
|
||||
input: { command: "echo hi", kind: "forged", seq: 999 },
|
||||
});
|
||||
const [record] = parsed(out).filter((r) => r.kind === "tool.before");
|
||||
expect(record?.phase).toBe("pre");
|
||||
expect(record?.seq).toBe(1);
|
||||
expect(record?.kind).toBe("tool.before");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,96 @@
|
||||
// OCI-7.3-redact-untested: the probe's SECRET_KEY regex is the security
|
||||
// boundary between live keys and persisted evidence. Each alternative
|
||||
// (KEY, TOKEN, SECRET, PASSWORD, AUTH, COOKIE, mixed case) is asserted
|
||||
// individually with its own canary, so narrowing or breaking the regex for
|
||||
// any single alternative goes red.
|
||||
import { afterAll, describe, expect, test } from "bun:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
// @ts-expect-error: probe/core.js is intentionally untyped JS (excluded
|
||||
// from tsconfig: TS7016); this test pins its runtime redaction behavior.
|
||||
import { makeProbe } from "../../probe/core.js";
|
||||
import { cleanupTempDirs, tempDir } from "../helpers/tmp.ts";
|
||||
|
||||
afterAll(cleanupTempDirs);
|
||||
|
||||
const ALTERNATIVES = ["KEY", "TOKEN", "SECRET", "PASSWORD", "AUTH", "COOKIE"];
|
||||
|
||||
type HookHandler = (input: Record<string, unknown>) => unknown;
|
||||
|
||||
interface ProbeRecord {
|
||||
kind: string;
|
||||
env?: Record<string, unknown>;
|
||||
headers?: Record<string, unknown>;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
async function runProbe(
|
||||
env: Record<string, string>,
|
||||
headers: Record<string, string>,
|
||||
): Promise<ProbeRecord[]> {
|
||||
const dir = tempDir("probe-redact-");
|
||||
const out = join(dir, "probe.jsonl");
|
||||
const handlers = new Map<string, HookHandler>();
|
||||
const register =
|
||||
(prefix: string) =>
|
||||
async (name: string, fn: HookHandler): Promise<void> => {
|
||||
handlers.set(`${prefix}.${name}`, fn);
|
||||
};
|
||||
const ctx = {
|
||||
options: { out },
|
||||
tool: { hook: register("tool") },
|
||||
shell: { hook: register("shell") },
|
||||
session: { hook: register("session") },
|
||||
event: { subscribe: () => (async function* () {})() },
|
||||
plugin: { list: async () => [] },
|
||||
};
|
||||
const probe = makeProbe("pre");
|
||||
await probe.setup(ctx);
|
||||
const shellBefore = handlers.get("shell.create.before");
|
||||
await shellBefore?.({ command: "env", cwd: "/", shell: "bash", env });
|
||||
const modelRequest = handlers.get("session.model.request");
|
||||
await modelRequest?.({
|
||||
kind: "request",
|
||||
model: "m",
|
||||
baseURL: "u",
|
||||
headers,
|
||||
});
|
||||
const lines = readFileSync(out, "utf8")
|
||||
.split("\n")
|
||||
.filter((line) => line.trim() !== "");
|
||||
return lines.map((line) => JSON.parse(line) as ProbeRecord);
|
||||
}
|
||||
|
||||
describe("probe SECRET_KEY redaction", () => {
|
||||
for (const alt of ALTERNATIVES) {
|
||||
test(`redacts ${alt} (mixed case) in shell env and model headers`, async () => {
|
||||
// Canary values contain only their own alternative, so dropping
|
||||
// any single alternative from the regex leaks exactly its canary.
|
||||
const envKey = `My_Svc_${alt.toLowerCase()}_Value`;
|
||||
const envCanary = `canary-${alt}-0001`;
|
||||
const headerCanary = `header-${alt}-0002`;
|
||||
const records = await runProbe(
|
||||
{ [envKey]: envCanary },
|
||||
{ [`X-Custom-${alt}`]: headerCanary },
|
||||
);
|
||||
const shell = records.find((r) => r.kind === "shell.before");
|
||||
expect(shell?.env?.[envKey]).toBe("[redacted]");
|
||||
expect(JSON.stringify(shell)).not.toContain(envCanary);
|
||||
const request = records.find((r) => r.kind === "session.model.request");
|
||||
expect(request?.headers?.[`X-Custom-${alt}`]).toBe("[redacted]");
|
||||
expect(JSON.stringify(request)).not.toContain(headerCanary);
|
||||
});
|
||||
}
|
||||
|
||||
test("non-secret values pass through untouched", async () => {
|
||||
const records = await runProbe(
|
||||
{ NORMAL_VAR: "visible", API_KEY: "sk-live-value" },
|
||||
{ "Content-Type": "application/json" },
|
||||
);
|
||||
const shell = records.find((r) => r.kind === "shell.before");
|
||||
expect(shell?.env?.NORMAL_VAR).toBe("visible");
|
||||
expect(shell?.env?.API_KEY).toBe("[redacted]");
|
||||
const request = records.find((r) => r.kind === "session.model.request");
|
||||
expect(request?.headers?.["Content-Type"]).toBe("application/json");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
// OCI-1.4-providers + OCI-5.6-inert-window: already remediated by the M7
|
||||
// removal (fail-closed). `expect.providers`/`expect.models` (deferred to v2)
|
||||
// and `gate.window`/`gate.threshold` (upstream verdict hard-codes 5/0.9)
|
||||
// are rejected by strict schemas instead of being silently ignored. This
|
||||
// test pins the removal: re-accepting any of these fields goes red here.
|
||||
// (Overlaps `test/unit/config-schema.test.ts`, which asserts the same four
|
||||
// rejections; this file names the audit IDs so the remediation is auditable.)
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { ConfigSchema } from "../../src/config/schema.ts";
|
||||
|
||||
describe("removed fields stay removed", () => {
|
||||
test("expect.providers and expect.models fail validation", () => {
|
||||
expect(
|
||||
ConfigSchema.safeParse({ expect: { providers: ["mine"] } }).success,
|
||||
).toBe(false);
|
||||
expect(ConfigSchema.safeParse({ expect: { models: ["m"] } }).success).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("gate.window and gate.threshold fail validation", () => {
|
||||
expect(ConfigSchema.safeParse({ gate: { window: 10 } }).success).toBe(
|
||||
false,
|
||||
);
|
||||
expect(ConfigSchema.safeParse({ gate: { threshold: 0.5 } }).success).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
// Review S1: one live budget for the whole run, shared by every version.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { runCi } from "../../src/run.ts";
|
||||
import { type FakeRun, fakeRunCount } from "../helpers/fake-opencode.ts";
|
||||
import {
|
||||
isolateCiEnv,
|
||||
makePlugin,
|
||||
PLUGIN_JS,
|
||||
useFake,
|
||||
} from "../helpers/run-ci.ts";
|
||||
|
||||
isolateCiEnv();
|
||||
|
||||
describe("live budget", () => {
|
||||
test("one live budget is shared across opencode versions", async () => {
|
||||
const pass: FakeRun = {
|
||||
cost: 0.6,
|
||||
toolCalls: [{ tool: "echo_tool", result: { content: "echo:hi" } }],
|
||||
};
|
||||
const fake = useFake([pass]);
|
||||
process.env.OPENCODE_API_KEY = "fake";
|
||||
const cwd = makePlugin({
|
||||
"index.ts": PLUGIN_JS,
|
||||
".opencode-ci/config.yaml":
|
||||
'opencode:\n versions: ["2.0.26", "latest"]\ngate:\n artifact: run-budget-results\nlive:\n model: fake/model\n budgetUSD: 1\n',
|
||||
".opencode-ci/scenarios/live.yaml":
|
||||
"id: live-smoke\nmode: live\nprompt: hi\nexpect:\n tools:\n - name: echo_tool\n",
|
||||
});
|
||||
const outcome = await runCi({ cwd, tiers: ["live"], quiet: true });
|
||||
const rows = outcome.resultsFile.results;
|
||||
expect(rows.find((r) => r.opencode === "2.0.26")?.pass).toBe(true);
|
||||
expect(rows.find((r) => r.opencode === "latest")?.skipped).toBe(
|
||||
"budget exhausted ($1.20 of $1.00)",
|
||||
);
|
||||
expect(fakeRunCount(fake)).toBe(2);
|
||||
expect(outcome.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,42 @@
|
||||
// Review B3: the bun-test case runs .opencode-ci/**/*.test.ts and reports
|
||||
// both outcomes. `bun test .opencode-ci` never matches (bun skips
|
||||
// dot-directories), so this test fails until each file is passed by path.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { runCi } from "../../src/run.ts";
|
||||
import {
|
||||
isolateCiEnv,
|
||||
makePlugin,
|
||||
PLUGIN_JS,
|
||||
useFake,
|
||||
} from "../helpers/run-ci.ts";
|
||||
|
||||
isolateCiEnv();
|
||||
|
||||
describe("bun-test case", () => {
|
||||
test("a passing test file makes bun-test pass", async () => {
|
||||
useFake([{}]);
|
||||
const cwd = makePlugin({
|
||||
"index.ts": PLUGIN_JS,
|
||||
".opencode-ci/ok.test.ts":
|
||||
'import { expect, test } from "bun:test";\ntest("fine", () => expect(1).toBe(1));\n',
|
||||
});
|
||||
const outcome = await runCi({ cwd, tiers: ["scripted"], quiet: true });
|
||||
const row = outcome.resultsFile.results.find((r) => r.id === "bun-test");
|
||||
expect(row?.pass).toBe(true);
|
||||
expect(outcome.exitCode).toBe(0);
|
||||
});
|
||||
|
||||
test("a failing test file in a subdir makes bun-test fail with its name", async () => {
|
||||
useFake([{}]);
|
||||
const cwd = makePlugin({
|
||||
"index.ts": PLUGIN_JS,
|
||||
".opencode-ci/nested/bad.test.ts":
|
||||
'import { expect, test } from "bun:test";\ntest("deliberately fails", () => expect(1).toBe(2));\n',
|
||||
});
|
||||
const outcome = await runCi({ cwd, tiers: ["scripted"], quiet: true });
|
||||
const row = outcome.resultsFile.results.find((r) => r.id === "bun-test");
|
||||
expect(row?.pass).toBe(false);
|
||||
expect((row?.failures ?? []).join("\n")).toContain("deliberately fails");
|
||||
expect(outcome.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
// Review B1, B2, S2 and the install/build exit code: install and build run
|
||||
// before entries are resolved, tier 1 loads the configured plugin.entry, a
|
||||
// missing plugin.entry is a configuration error, and a failing build is the
|
||||
// plugin's failure (exit 1), not a configuration error (exit 2).
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { ConfigError } from "../../src/config/load.ts";
|
||||
import { resolveEntries } from "../../src/plugin/entry.ts";
|
||||
import { StepError } from "../../src/plugin/install.ts";
|
||||
import { runCi } from "../../src/run.ts";
|
||||
import {
|
||||
isolateCiEnv,
|
||||
makePlugin,
|
||||
PLUGIN_JS,
|
||||
useFake,
|
||||
} from "../helpers/run-ci.ts";
|
||||
import { tempDir } from "../helpers/tmp.ts";
|
||||
|
||||
isolateCiEnv();
|
||||
|
||||
const SHIM = ".opencode-ci/out/2.0.26/load/plugin-under-test/server.js";
|
||||
|
||||
describe("entries and the install/build steps", () => {
|
||||
test("runs plugin.build before resolving a dist/ entry", async () => {
|
||||
useFake([{}]);
|
||||
const cwd = makePlugin({
|
||||
"package.json":
|
||||
'{ "name": "fixture-built", "type": "module", "main": "./dist/server.js" }',
|
||||
"src/server.js": PLUGIN_JS,
|
||||
".opencode-ci/config.yaml":
|
||||
"plugin:\n build: mkdir -p dist && cp src/server.js dist/server.js\n",
|
||||
});
|
||||
const outcome = await runCi({ cwd, tiers: ["load"], quiet: true });
|
||||
expect(outcome.exitCode).toBe(0);
|
||||
expect(readFileSync(join(cwd, SHIM), "utf8")).toContain("/dist/server.js");
|
||||
});
|
||||
|
||||
test("tier 1 loads plugin.entry, not auto-detection", async () => {
|
||||
useFake([{}]);
|
||||
const cwd = makePlugin({
|
||||
"package.json": '{ "name": "fixture-explicit", "type": "module" }',
|
||||
"src/plugin.ts": PLUGIN_JS,
|
||||
".opencode-ci/config.yaml": "plugin:\n entry: src/plugin.ts\n",
|
||||
});
|
||||
const outcome = await runCi({ cwd, tiers: ["load"], quiet: true });
|
||||
expect(outcome.exitCode).toBe(0);
|
||||
expect(readFileSync(join(cwd, SHIM), "utf8")).toContain("/src/plugin.ts");
|
||||
});
|
||||
|
||||
test("a missing plugin.entry is a configuration error even when index.ts exists", async () => {
|
||||
useFake([{}]);
|
||||
const cwd = makePlugin({
|
||||
"package.json": '{ "name": "fixture-typo", "type": "module" }',
|
||||
"index.ts": PLUGIN_JS,
|
||||
".opencode-ci/config.yaml": "plugin:\n entry: src/typo.ts\n",
|
||||
});
|
||||
await expect(runCi({ cwd, tiers: ["load"], quiet: true })).rejects.toThrow(
|
||||
ConfigError,
|
||||
);
|
||||
});
|
||||
|
||||
test("an explicit entry still resolves the TUI entry", () => {
|
||||
const dir = tempDir("entry-both-");
|
||||
mkdirSync(join(dir, "src"));
|
||||
writeFileSync(join(dir, "src/plugin.ts"), PLUGIN_JS);
|
||||
writeFileSync(join(dir, "tui.ts"), PLUGIN_JS);
|
||||
const entries = resolveEntries(dir, "src/plugin.ts");
|
||||
expect(entries.server).toBe(join(dir, "src/plugin.ts"));
|
||||
expect(entries.tui).toBe(join(dir, "tui.ts"));
|
||||
});
|
||||
|
||||
test("a failing build is the plugin's failure, not a config error", async () => {
|
||||
useFake([{}]);
|
||||
const cwd = makePlugin({
|
||||
"index.ts": PLUGIN_JS,
|
||||
".opencode-ci/config.yaml": "plugin:\n build: exit 3\n",
|
||||
});
|
||||
await expect(runCi({ cwd, quiet: true })).rejects.toThrow(StepError);
|
||||
await expect(runCi({ cwd, quiet: true })).rejects.toThrow(
|
||||
"build step failed (exit 3): exit code 3.",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,64 @@
|
||||
// runCi exit codes and skips across required and allowed opencode versions.
|
||||
// These already pass on v1; they prove the fake opencode binary works.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { runCi } from "../../src/run.ts";
|
||||
import { fakeRunCount } from "../helpers/fake-opencode.ts";
|
||||
import {
|
||||
isolateCiEnv,
|
||||
makePlugin,
|
||||
PLUGIN_JS,
|
||||
useFake,
|
||||
} from "../helpers/run-ci.ts";
|
||||
|
||||
isolateCiEnv();
|
||||
|
||||
const TWO_VERSIONS =
|
||||
'opencode:\n versions: ["2.0.26", "latest"]\ngate:\n artifact: run-exit-codes-results\n';
|
||||
|
||||
describe("runCi exit codes", () => {
|
||||
test("a plugin that loads passes tier 1", async () => {
|
||||
const fake = useFake([{}]);
|
||||
const cwd = makePlugin({ "index.ts": PLUGIN_JS });
|
||||
const outcome = await runCi({ cwd, tiers: ["load"], quiet: true });
|
||||
expect(outcome.exitCode).toBe(0);
|
||||
expect(fakeRunCount(fake)).toBe(1);
|
||||
});
|
||||
|
||||
test("a required-version tier 1 failure exits 1", async () => {
|
||||
useFake([{ pluginStatus: "missing" }, {}]);
|
||||
const cwd = makePlugin({
|
||||
"index.ts": PLUGIN_JS,
|
||||
".opencode-ci/config.yaml": TWO_VERSIONS,
|
||||
});
|
||||
const outcome = await runCi({ cwd, tiers: ["load"], quiet: true });
|
||||
expect(outcome.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
test("an allowed-version tier 1 failure exits 0 and is reported", async () => {
|
||||
useFake([{}, { pluginStatus: "missing" }]);
|
||||
const cwd = makePlugin({
|
||||
"index.ts": PLUGIN_JS,
|
||||
".opencode-ci/config.yaml": TWO_VERSIONS,
|
||||
});
|
||||
const outcome = await runCi({ cwd, tiers: ["load"], quiet: true });
|
||||
expect(outcome.exitCode).toBe(0);
|
||||
expect(readFileSync(outcome.reportPath, "utf8")).toContain(
|
||||
"⚠️ allowed failure",
|
||||
);
|
||||
});
|
||||
|
||||
test("a failing tier 1 skips that version's scripted scenarios", async () => {
|
||||
const fake = useFake([{ pluginStatus: "failed" }]);
|
||||
const cwd = makePlugin({
|
||||
"index.ts": PLUGIN_JS,
|
||||
".opencode-ci/scenarios/smoke.yaml":
|
||||
"id: smoke\nmode: scripted\nprompt: hi\nscript:\n - text: done\n",
|
||||
});
|
||||
const outcome = await runCi({ cwd, quiet: true });
|
||||
expect(outcome.exitCode).toBe(1);
|
||||
const smoke = outcome.resultsFile.results.find((r) => r.id === "smoke");
|
||||
expect(smoke?.skipped).toBe("tier 1 failed");
|
||||
expect(fakeRunCount(fake)).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
// Review B5 and input hygiene: a re-run into the same out dir never sees the
|
||||
// previous run's evidence, evidence dirs are only ever reset inside the out
|
||||
// dir, and bad --scenario / --opencode values are rejected up front.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { join } from "node:path";
|
||||
import { parseRunArgs } from "../../src/cli.ts";
|
||||
import { runCi } from "../../src/run.ts";
|
||||
import { resetDir } from "../../src/util/fs.ts";
|
||||
import type { FakeRun } from "../helpers/fake-opencode.ts";
|
||||
import {
|
||||
isolateCiEnv,
|
||||
makePlugin,
|
||||
PLUGIN_JS,
|
||||
useFake,
|
||||
} from "../helpers/run-ci.ts";
|
||||
import { tempDir } from "../helpers/tmp.ts";
|
||||
|
||||
isolateCiEnv();
|
||||
|
||||
describe("run inputs and evidence dirs", () => {
|
||||
test("a second run in the same out dir never sees the first run's evidence", async () => {
|
||||
const called: FakeRun = {
|
||||
toolCalls: [{ tool: "echo_tool", result: { content: "echo:hi" } }],
|
||||
};
|
||||
// Run 1: tier 1, then the scenario (the tool is called).
|
||||
// Run 2: tier 1, then the scenario (the tool is NOT called).
|
||||
useFake([{}, called, {}, {}]);
|
||||
const cwd = makePlugin({
|
||||
"index.ts": PLUGIN_JS,
|
||||
".opencode-ci/scenarios/echo.yaml":
|
||||
"id: echo\nmode: scripted\nprompt: hi\nscript:\n - text: done\nexpect:\n tools:\n - name: echo_tool\n",
|
||||
});
|
||||
const first = await runCi({ cwd, quiet: true });
|
||||
expect(first.exitCode).toBe(0);
|
||||
const second = await runCi({ cwd, quiet: true });
|
||||
expect(second.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
test("resetDir refuses paths outside the root", () => {
|
||||
const root = tempDir("reset-root-");
|
||||
expect(() => resetDir(join(root, ".."), root)).toThrow("refusing to reset");
|
||||
expect(() => resetDir(root, root)).toThrow("refusing to reset");
|
||||
resetDir(join(root, "2.0.26", "load"), root);
|
||||
});
|
||||
|
||||
test("an unknown --scenario id is a configuration error", async () => {
|
||||
useFake([{}]);
|
||||
const cwd = makePlugin({
|
||||
"index.ts": PLUGIN_JS,
|
||||
".opencode-ci/scenarios/smoke.yaml":
|
||||
"id: smoke\nmode: scripted\nprompt: hi\nscript:\n - text: done\n",
|
||||
});
|
||||
await expect(
|
||||
runCi({ cwd, scenarios: ["smok"], quiet: true }),
|
||||
).rejects.toThrow("unknown --scenario id(s): smok (known: smoke).");
|
||||
});
|
||||
|
||||
test("--opencode accepts x.y.z and latest, rejects path syntax", () => {
|
||||
expect(parseRunArgs(["--opencode", "2.0.26,latest"]).opencodes).toEqual([
|
||||
"2.0.26",
|
||||
"latest",
|
||||
]);
|
||||
expect(() => parseRunArgs(["--opencode", "../.."])).toThrow(
|
||||
'Invalid --opencode "../..": expected x.y.z or "latest".',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
// OCI-4.4-workspace-escape: scenario workspace writes must stay inside the
|
||||
// project dir. `../`, `sub/../../` and absolute paths throw ConfigError
|
||||
// (exit 2 upstream) instead of escaping; a legit file still writes.
|
||||
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { ConfigError } from "../../src/config/load.ts";
|
||||
import { Sandbox } from "../../src/sandbox/sandbox.ts";
|
||||
import { cleanupTempDirs, tempDir } from "../helpers/tmp.ts";
|
||||
|
||||
const savedTmpdir = process.env.TMPDIR;
|
||||
|
||||
beforeEach(() => {
|
||||
// Contain the sandbox roots (including residue from throwing
|
||||
// constructors) so cleanupTempDirs() removes everything.
|
||||
process.env.TMPDIR = tempDir("sandbox-escape-");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (savedTmpdir === undefined) {
|
||||
delete process.env.TMPDIR;
|
||||
} else {
|
||||
process.env.TMPDIR = savedTmpdir;
|
||||
}
|
||||
cleanupTempDirs();
|
||||
});
|
||||
|
||||
function makeSandbox(workspace: Record<string, string>): Sandbox {
|
||||
return new Sandbox({
|
||||
opencodeBin: "/nonexistent/opencode",
|
||||
plugins: [],
|
||||
workspace,
|
||||
});
|
||||
}
|
||||
|
||||
describe("sandbox workspace containment", () => {
|
||||
test("a legit workspace file still writes", async () => {
|
||||
const sandbox = makeSandbox({
|
||||
"ok.txt": "ok",
|
||||
"sub/nested.txt": "nested",
|
||||
});
|
||||
expect(readFileSync(join(sandbox.project, "ok.txt"), "utf8")).toBe("ok");
|
||||
expect(readFileSync(join(sandbox.project, "sub/nested.txt"), "utf8")).toBe(
|
||||
"nested",
|
||||
);
|
||||
await sandbox.dispose();
|
||||
});
|
||||
|
||||
for (const rel of [
|
||||
"../escape.txt",
|
||||
"sub/../../escape2.txt",
|
||||
"../../sandbox-root-escape.txt",
|
||||
"/tmp/absolute-escape.txt",
|
||||
]) {
|
||||
test(`workspace path "${rel}" throws instead of escaping`, () => {
|
||||
expect(() => makeSandbox({ [rel]: "pwned" })).toThrow(ConfigError);
|
||||
expect(() => makeSandbox({ [rel]: "pwned" })).toThrow(
|
||||
/must be relative and stay inside the project dir/,
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,42 @@
|
||||
// Review B5 follow-ups: scenario ids that collide with the tool's own case
|
||||
// ids are rejected, and workspace files must stay inside the project dir.
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { ScenarioSchema } from "../../src/config/schema.ts";
|
||||
|
||||
const base = { mode: "scripted", prompt: "hi", script: [] };
|
||||
|
||||
describe("scenario rules", () => {
|
||||
test("ids load and bun-test are reserved", () => {
|
||||
for (const id of ["load", "bun-test"]) {
|
||||
const parsed = ScenarioSchema.safeParse({ ...base, id });
|
||||
expect(parsed.success).toBe(false);
|
||||
if (!parsed.success) {
|
||||
expect(parsed.error.issues[0]?.message).toBe(
|
||||
`scenario "${id}": id "${id}" is reserved (the tool uses it for its own case and evidence dir)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("workspace paths must stay inside the project dir", () => {
|
||||
for (const key of ["../escape.txt", "a/../../b", "/etc/passwd"]) {
|
||||
const parsed = ScenarioSchema.safeParse({
|
||||
...base,
|
||||
id: "ws",
|
||||
workspace: { [key]: "x" },
|
||||
});
|
||||
expect(parsed.success).toBe(false);
|
||||
if (!parsed.success) {
|
||||
expect(parsed.error.issues[0]?.message).toBe(
|
||||
`scenario "ws": workspace path "${key}" must be relative and stay inside the project dir`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const ok = ScenarioSchema.safeParse({
|
||||
...base,
|
||||
id: "ws",
|
||||
workspace: { "a/b.txt": "x", ".env": "A=1" },
|
||||
});
|
||||
expect(ok.success).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user