test(testbed): use Node 22.23.4 and verify downloads (#1902)

The internal testbed image moves from Node 22.22.1 to 22.23.4 (CVE-2026-111497). Its download is checked against Node's official SHASUMS256, and the check binds the extracted archive, so a mismatch stops the build.

The scenario runner now settles its containers by recorded id instead of `rm -f`. It removes the cidfile only after a confirmed removal, so a re-run into the same folder isn't refused, and it prints a retained container's name.

This image isn't shipped. Its rebuild hasn't been qualified on its own: the first ordinary run of the route that consumes it is the proof.

docker/testbed/Dockerfile

@@ -24,7 +24,7 @@ RUN set -eux; \
     rm -rf /var/lib/apt/lists/*
 
 # --- layer 2: pinned node (an in-range engines version — see scripts/check-engines) ---
-ARG NODE_VERSION=22.22.1
+ARG NODE_VERSION=22.23.4
 ARG TARGETARCH
 RUN set -eux; \
     case "${TARGETARCH}" in \
@@ -33,10 +33,14 @@ RUN set -eux; \
       "") echo "TARGETARCH is empty — the legacy (non-BuildKit) builder does not populate it. Pass --build-arg TARGETARCH explicitly (build-testbed-image.sh does) or enable BuildKit. REFUSING to default to amd64, which would install wrong-arch Node." >&2; exit 1 ;; \
       *) echo "unsupported TARGETARCH=${TARGETARCH}" >&2; exit 1 ;; \
     esac; \
-    curl -fsSLo /tmp/node.tar.xz \
-      "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz"; \
-    tar -xJf /tmp/node.tar.xz -C /usr/local --strip-components=1 --no-same-owner; \
-    rm -f /tmp/node.tar.xz; \
+    NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz"; \
+    curl -fsSLo "/tmp/${NODE_TARBALL}" \
+      "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}"; \
+    curl -fsSLo /tmp/SHASUMS256.txt \
+      "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt"; \
+    (cd /tmp && awk -v tarball="${NODE_TARBALL}" '$2 == tarball' SHASUMS256.txt | sha256sum -c -); \
+    tar -xJf "/tmp/${NODE_TARBALL}" -C /usr/local --strip-components=1 --no-same-owner; \
+    rm -f "/tmp/${NODE_TARBALL}" /tmp/SHASUMS256.txt; \
     node --version; npm --version
 
 # --- layer 3: the OpenRig package from the LOCAL pack tarball, never the npm registry ---

scripts/build-testbed-image.sh

@@ -38,9 +38,12 @@ NODE_VERSION="$(sed -n 's/^ARG NODE_VERSION=\([0-9][0-9.]*\).*/\1/p' "${TESTBED_
 # --- assemble a clean build context: Dockerfile + entrypoint + the openrig pack + staged stub assets ---
 CONTEXT="$(mktemp -d)"
 LOAD_CONTAINER=""
+LOAD_CID="${OUT_DIR}/image-load.cid"
+IMAGE_ID=""
+test ! -e "${LOAD_CID}" || { echo "Retained cidfile: ${LOAD_CID}" >&2; exit 2; }
 cleanup() {
   if [ -n "${LOAD_CONTAINER}" ]; then
-    node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 30 docker rm -f "${LOAD_CONTAINER}" >/dev/null ||
+    node "${REPO_ROOT}/scripts/scenario-executor.mjs" settle "${LOAD_CID}" "${LOAD_CONTAINER}" "${IMAGE_ID}" >/dev/null ||
       echo "[testbed] cleanup incomplete; retained container name: ${LOAD_CONTAINER}" >&2
   fi
   rm -rf "${CONTEXT}"
@@ -98,8 +101,9 @@ echo "[testbed] effect proof: daemon LOAD inside the container (better-sqlite3 m
 # fails `rig daemon start` here → set -e → non-zero → the build verb fails BEFORE the A/B pin.
 LOAD_CONTAINER="openrig-testbed-load-$(node -p 'require("node:crypto").randomUUID()')"
 printf '%s\n' "${LOAD_CONTAINER}" > "${OUT_DIR}/image-load.container-name.txt"
+IMAGE_ID="$(docker image inspect --format '{{.Id}}' "${IMAGE_TAG}")"
 load_status=0
-node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 120 docker run --name "${LOAD_CONTAINER}" \
+node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 120 docker run --name "${LOAD_CONTAINER}" --cidfile "${LOAD_CID}" \
   --platform "${TARGET_PLATFORM}" --network none --cap-drop ALL --security-opt no-new-privileges \
   --cpus 2 --memory 2g --memory-swap 2g --pids-limit 256 "${IMAGE_TAG}" bash -lc \
   'set -euo pipefail; trap "rig daemon stop >/dev/null 2>&1 || true" EXIT; rig --version; rig daemon start --no-kernel; curl -fsS http://127.0.0.1:7433/healthz; rig daemon status' \
@@ -111,7 +115,7 @@ node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 30 docker inspect "${L
   > "${OUT_DIR}/image-load.container.json" || inspect_status=$?
 [ "${load_status}" -eq 0 ] || exit "${load_status}"
 [ "${inspect_status}" -eq 0 ] || exit "${inspect_status}"
-node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 30 docker rm -f "${LOAD_CONTAINER}" >/dev/null
+node "${REPO_ROOT}/scripts/scenario-executor.mjs" settle "${LOAD_CID}" "${LOAD_CONTAINER}" "${IMAGE_ID}" >/dev/null || { echo "Cleanup incomplete; retained container name: ${LOAD_CONTAINER}" >&2; LOAD_CONTAINER=""; exit 2; }
 LOAD_CONTAINER=""
 
 # --- emit the reproducible manifest + census receipt via the tested node orchestrator ---

scripts/build-testbed-image.test.mjs

@@ -137,30 +137,45 @@ const p=spawnSync(${JSON.stringify(process.execPath)},args,{stdio:'inherit'});pr
 `, { mode: 0o755 });
     writeFileSync(join(root, "bin/docker"), `#!${process.execPath}
 import fs from 'node:fs';const args=process.argv.slice(2);const marker=process.env.FAKE_CONTAINER;
+const id='a'.repeat(64),image='sha256:'+'b'.repeat(64);
 if(args[0]==='version')console.log(JSON.stringify({Os:'linux',Arch:'amd64'}));
+else if(args[0]==='image')console.log(image);
 else if(args[0]==='build'){}
 else if(args[0]==='run'){
-  const at=args.indexOf('--name');if(at<0)process.exit(9);
-  fs.writeFileSync(marker,args[at+1]);
+  const at=args.indexOf('--name'),cid=args.indexOf('--cidfile');if(at<0||cid<0)process.exit(9);
+  fs.writeFileSync(args[cid+1],id);fs.writeFileSync(marker,JSON.stringify({Id:id,Name:'/'+args[at+1],Image:image,State:{Running:process.env.LOAD_CASE==='timeout',Status:process.env.LOAD_CASE==='timeout'?'running':'exited'}}));
   if(process.env.LOAD_CASE==='timeout')setInterval(()=>{},1000);
   else process.exit(process.env.LOAD_CASE==='failure'?7:0);
-}else if(args[0]==='inspect')console.log('[]');
+}else if(args[0]==='inspect')console.log(JSON.stringify([JSON.parse(fs.readFileSync(marker))]));
+else if(args[0]==='kill'){
+  if(args.join(' ')!=='kill --signal TERM '+id)process.exit(8);
+  const state=JSON.parse(fs.readFileSync(marker));state.State={Running:false,Status:'exited'};fs.writeFileSync(marker,JSON.stringify(state));
+}else if(args[0]==='wait'){if(JSON.parse(fs.readFileSync(marker)).State.Running)process.exit(8);console.log('0');}
 else if(args[0]==='rm'){
-  if(fs.readFileSync(marker,'utf8')!==args.at(-1))process.exit(8);
+  if(process.env.LOAD_CASE==='retain')process.exit(8);
+  if(args.length!==2||args[1]!==id||JSON.parse(fs.readFileSync(marker)).State.Running)process.exit(8);
   fs.unlinkSync(marker);
 }else process.exit(8);
 `, { mode: 0o755 });
-    for (const [mode, status] of [["success", 0], ["failure", 7], ["timeout", 124]]) {
+    for (const [mode, status] of [["success", 0], ["success", 0], ["failure", 7], ["timeout", 124], ["retain", 2]]) {
       const out = join(root, mode), marker = join(root, "container");
       const run = spawnSync("/bin/bash", [join(root, "scripts/build-testbed-image.sh"), out], {
         env: { PATH: `${join(root, "bin")}:/usr/bin:/bin`, HOME: root, TMPDIR: root, LOAD_CASE: mode, FAKE_CONTAINER: marker },
         encoding: "utf8", timeout: 8000,
       });
       assert.equal(run.status, status, `${mode}: ${run.stderr}`);
-      assert.equal(Number(readFileSync(join(out, "image-load.exit-code.txt"), "utf8")), status);
+      assert.equal(Number(readFileSync(join(out, "image-load.exit-code.txt"), "utf8")), mode === "retain" ? 0 : status);
       assert.match(readFileSync(join(out, "image-load.container-name.txt"), "utf8"), /^openrig-testbed-load-/);
-      assert.equal(existsSync(marker), false, `${mode}: container must be removed even after timeout`);
+      assert.equal(existsSync(marker), mode === "retain", `${mode}: only confirmed removal clears the container`);
+      assert.equal(existsSync(join(out, "image-load.cid")), mode === "retain", `${mode}: cidfile follows confirmed removal`);
       assert.equal(existsSync(join(out, "manifest.json")), status === 0);
+      if (mode === "retain") {
+        const retry = spawnSync("/bin/bash", [join(root, "scripts/build-testbed-image.sh"), out], {
+          env: { PATH: `${join(root, "bin")}:/usr/bin:/bin`, HOME: root, TMPDIR: root, LOAD_CASE: mode, FAKE_CONTAINER: marker },
+          encoding: "utf8", timeout: 3000,
+        });
+        assert.equal(retry.status, 2); assert.match(retry.stderr, /Retained cidfile/);
+      }
     }
   } finally { rmSync(root, { recursive: true, force: true }); }
 });

scripts/run-pr-scenarios.sh

@@ -61,9 +61,11 @@ BASE="openrig-testbed:$SHA"
 IMAGE="openrig-pr-scenarios:$SHA"
 CONTEXT="$(mktemp -d)"
 CONTAINER=""
+CONTAINER_CID=""
+IMAGE_ID=""
 cleanup() {
   if [ -n "$CONTAINER" ]; then
-    node scripts/scenario-executor.mjs timeout 30 docker rm -f "$CONTAINER" >/dev/null || echo "Cleanup incomplete; retained container name: $CONTAINER" >&2
+    node scripts/scenario-executor.mjs settle "$CONTAINER_CID" "$CONTAINER" "$IMAGE_ID" >/dev/null || echo "Cleanup incomplete; retained container name: $CONTAINER" >&2
   fi
   rm -rf "$CONTEXT"
 }
@@ -78,17 +80,20 @@ node_modules/.bin/esbuild packages/test-system/ci/run.mjs --bundle --platform=no
   --outfile="$CONTEXT/runner.mjs" --metafile="$OUT/runner-inputs.json"
 docker build --network none --platform "$TARGET_PLATFORM" --build-arg TESTBED_IMAGE="$BASE" -t "$IMAGE" "$CONTEXT"
 docker image inspect "$IMAGE" > "$OUT/image-inspect.json"
+IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$IMAGE")"
 
 attempt=0
 run_case() {  # run_case <case> <mode>
   local scenario="$1" mode="$2" status=0 LOG
   attempt=$((attempt + 1))
   LOG="$OUT/$attempt-$scenario-$mode.log"
+  CONTAINER_CID="$OUT/$attempt-$scenario-$mode.cid"
+  test ! -e "$CONTAINER_CID" || { echo "Retained cidfile: $CONTAINER_CID" >&2; exit 2; }
   CONTAINER="openrig-pr-${SHA:0:12}-$attempt-$$"
   printf '%s\n' "$CONTAINER" > "$OUT/$attempt-$scenario-$mode.container-name.txt"
   # Fresh writable scratch only. No mounts, host networking, credentials or Docker
   # socket; the container is non-root, resource bounded and removed even on failure.
-  node scripts/scenario-executor.mjs timeout 300 docker run --name "$CONTAINER" --platform "$TARGET_PLATFORM" --network none \
+  node scripts/scenario-executor.mjs timeout 300 docker run --name "$CONTAINER" --cidfile "$CONTAINER_CID" --platform "$TARGET_PLATFORM" --network none \
     --read-only --tmpfs /tmp:rw,exec,nosuid,nodev,size=512m,mode=1777 \
     --cap-drop ALL --security-opt no-new-privileges --pids-limit 256 \
     --memory 2g --memory-swap 2g --cpus 2 "$IMAGE" node /opt/openrig-testbed/runner.mjs "$mode" "$scenario" \
@@ -96,7 +101,7 @@ run_case() {  # run_case <case> <mode>
   printf '%s\n' "$status" > "$OUT/$attempt-$scenario-$mode.exit-code.txt"
   cat "$LOG"
   node scripts/scenario-executor.mjs timeout 30 docker inspect "$CONTAINER" > "$OUT/$attempt-$scenario-$mode.container.json"
-  node scripts/scenario-executor.mjs timeout 30 docker rm -f "$CONTAINER" >/dev/null
+  node scripts/scenario-executor.mjs settle "$CONTAINER_CID" "$CONTAINER" "$IMAGE_ID" >/dev/null || { echo "Cleanup incomplete; retained container name: $CONTAINER" >&2; CONTAINER=""; exit 2; }
   CONTAINER=""
   node --input-type=module - "$mode" "$status" "$LOG" "$scenario" <<'JS'
 import { readFileSync } from 'node:fs';

scripts/scenario-executor.mjs

@@ -1,6 +1,6 @@
 // Client-side helpers for the existing scenario scripts; never runs a daemon.
 import { spawn, spawnSync } from 'node:child_process';
-import { writeFileSync } from 'node:fs';
+import { readFileSync, unlinkSync, writeFileSync } from 'node:fs';
 import { constants } from 'node:os';
 import { resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
@@ -12,6 +12,37 @@ export function dockerPlatform(server) {
   return `linux/${server.Arch}`;
 }
 
+// A client timeout does not establish that its engine-side container stopped.
+// Unknown identity or settlement leaves the container for its owner to reconcile.
+export function settleContainer({ id, name, image }, invoke = args => {
+  const result = spawnSync('docker', args, { encoding: 'utf8', timeout: 35000, maxBuffer: 1024 * 1024 });
+  if (result.error || result.status !== 0) throw new Error(result.error?.message ?? result.stderr);
+  return result.stdout;
+}) {
+  if (!/^[a-f0-9]{64}$/.test(id) || !/^sha256:[a-f0-9]{64}$/.test(image) || !name) {
+    throw new Error('Exact created container ID, name and image ID required');
+  }
+  const inspect = () => {
+    const objects = JSON.parse(invoke(['inspect', id]));
+    const object = objects[0];
+    if (objects.length !== 1 || object.Id !== id || object.Name !== '/' + name || object.Image !== image) {
+      throw new Error('Container identity unknown; retained');
+    }
+    return object.State;
+  };
+  let state = inspect();
+  if (state.Running === true) {
+    // Explicit TERM has no automatic KILL escalation. A failed wait retains it.
+    invoke(['kill', '--signal', 'TERM', id]);
+    invoke(['wait', id]);
+    state = inspect();
+  }
+  if (state.Running !== false || !['created', 'exited'].includes(state.Status)) {
+    throw new Error('Container not confirmed stopped; retained');
+  }
+  invoke(['rm', id]);
+}
+
 export async function runWithDeadline(argv, { timeoutMs, killAfterMs = 15000 }) {
   if (!argv.length || !Number.isFinite(timeoutMs) || timeoutMs <= 0 || killAfterMs <= 0) {
     throw new Error('Command and positive finite deadline required');
@@ -63,10 +94,14 @@ if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.ur
       const platform = dockerPlatform(server);
       if (args[0]) writeFileSync(args[0], JSON.stringify({ platform, server }, null, 2) + '\n');
       console.log(platform);
+    } else if (operation === 'settle') {
+      const [cidFile, name, image] = args;
+      settleContainer({ id: readFileSync(cidFile, 'utf8').trim(), name, image });
+      unlinkSync(cidFile);
     } else if (operation === 'timeout') {
       process.exitCode = await runWithDeadline(args.slice(1), { timeoutMs: Number(args[0]) * 1000 });
     } else {
-      throw new Error('Expected platform [receipt-path] or timeout <seconds> <command> [args...]');
+      throw new Error('Expected platform, settle <cid-file> <name> <image-id>, or timeout <seconds> <command> [args...]');
     }
   } catch (error) {
     console.error(error.message);

scripts/scenario-executor.test.mjs

@@ -58,3 +58,26 @@ test('ordinary invocation and ambiguous remote selection refuse before any build
     assert.match(run.stderr, message);
   }
 });
+
+test('settlement uses the recorded ID, TERM and stopped-state removal; unknown effects stay retained', async () => {
+  const { settleContainer } = await import('./scenario-executor.mjs');
+  assert.equal(typeof settleContainer, 'function');
+  const id='a'.repeat(64), image='sha256:'+'b'.repeat(64), name='testbed-owned';
+  function fixture({running=false,mismatch=false,waitUnknown=false,stillRunning=false}={}) {
+    const calls=[];let inspected=0;
+    const invoke=args=>{
+      calls.push(args);
+      if(args[0]==='inspect')return JSON.stringify([{Id:mismatch?'c'.repeat(64):id,Name:'/'+name,Image:image,State:{Running:inspected++===0?running:stillRunning,Status:(inspected===1?running:stillRunning)?'running':'exited'}}]);
+      if(args[0]==='wait'&&waitUnknown)throw Error('wait timed out; engine state unknown');
+      return '';
+    };
+    return {calls,invoke};
+  }
+  const owned={id,name,image};
+  const stopped=fixture();settleContainer(owned,stopped.invoke);assert.deepEqual(stopped.calls,[['inspect',id],['rm',id]]);
+  const live=fixture({running:true});settleContainer(owned,live.invoke);assert.deepEqual(live.calls,[['inspect',id],['kill','--signal','TERM',id],['wait',id],['inspect',id],['rm',id]]);
+  for(const opts of [{mismatch:true},{running:true,waitUnknown:true},{running:true,stillRunning:true}]){
+    const f=fixture(opts);assert.throws(()=>settleContainer(owned,f.invoke));assert.ok(!f.calls.some(x=>x[0]==='rm'));
+  }
+  const f=fixture();assert.throws(()=>settleContainer({...owned,id:'not-an-id'},f.invoke));assert.equal(f.calls.length,0);
+});