diff --git a/docker/testbed/Dockerfile b/docker/testbed/Dockerfile index 4e9a6d7df..54764c77c 100644 --- a/docker/testbed/Dockerfile +++ b/docker/testbed/Dockerfile @@ -24,7 +24,7 @@ RUN set -eux; \ rm -rf /var/lib/apt/lists/* # --- layer 2: pinned node (an in-range engines version — see scripts/check-engines) --- -ARG NODE_VERSION=22.22.1 +ARG NODE_VERSION=22.23.4 ARG TARGETARCH RUN set -eux; \ case "${TARGETARCH}" in \ @@ -33,10 +33,14 @@ RUN set -eux; \ "") echo "TARGETARCH is empty — the legacy (non-BuildKit) builder does not populate it. Pass --build-arg TARGETARCH explicitly (build-testbed-image.sh does) or enable BuildKit. REFUSING to default to amd64, which would install wrong-arch Node." >&2; exit 1 ;; \ *) echo "unsupported TARGETARCH=${TARGETARCH}" >&2; exit 1 ;; \ esac; \ - curl -fsSLo /tmp/node.tar.xz \ - "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz"; \ - tar -xJf /tmp/node.tar.xz -C /usr/local --strip-components=1 --no-same-owner; \ - rm -f /tmp/node.tar.xz; \ + NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz"; \ + curl -fsSLo "/tmp/${NODE_TARBALL}" \ + "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}"; \ + curl -fsSLo /tmp/SHASUMS256.txt \ + "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt"; \ + (cd /tmp && awk -v tarball="${NODE_TARBALL}" '$2 == tarball' SHASUMS256.txt | sha256sum -c -); \ + tar -xJf "/tmp/${NODE_TARBALL}" -C /usr/local --strip-components=1 --no-same-owner; \ + rm -f "/tmp/${NODE_TARBALL}" /tmp/SHASUMS256.txt; \ node --version; npm --version # --- layer 3: the OpenRig package from the LOCAL pack tarball, never the npm registry --- diff --git a/scripts/build-testbed-image.sh b/scripts/build-testbed-image.sh index 10b3950bd..ccf4a70ab 100755 --- a/scripts/build-testbed-image.sh +++ b/scripts/build-testbed-image.sh @@ -38,9 +38,12 @@ NODE_VERSION="$(sed -n 's/^ARG NODE_VERSION=\([0-9][0-9.]*\).*/\1/p' "${TESTBED_ # --- assemble a clean build context: Dockerfile + entrypoint + the openrig pack + staged stub assets --- CONTEXT="$(mktemp -d)" LOAD_CONTAINER="" +LOAD_CID="${OUT_DIR}/image-load.cid" +IMAGE_ID="" +test ! -e "${LOAD_CID}" || { echo "Retained cidfile: ${LOAD_CID}" >&2; exit 2; } cleanup() { if [ -n "${LOAD_CONTAINER}" ]; then - node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 30 docker rm -f "${LOAD_CONTAINER}" >/dev/null || + node "${REPO_ROOT}/scripts/scenario-executor.mjs" settle "${LOAD_CID}" "${LOAD_CONTAINER}" "${IMAGE_ID}" >/dev/null || echo "[testbed] cleanup incomplete; retained container name: ${LOAD_CONTAINER}" >&2 fi rm -rf "${CONTEXT}" @@ -98,8 +101,9 @@ echo "[testbed] effect proof: daemon LOAD inside the container (better-sqlite3 m # fails `rig daemon start` here → set -e → non-zero → the build verb fails BEFORE the A/B pin. LOAD_CONTAINER="openrig-testbed-load-$(node -p 'require("node:crypto").randomUUID()')" printf '%s\n' "${LOAD_CONTAINER}" > "${OUT_DIR}/image-load.container-name.txt" +IMAGE_ID="$(docker image inspect --format '{{.Id}}' "${IMAGE_TAG}")" load_status=0 -node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 120 docker run --name "${LOAD_CONTAINER}" \ +node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 120 docker run --name "${LOAD_CONTAINER}" --cidfile "${LOAD_CID}" \ --platform "${TARGET_PLATFORM}" --network none --cap-drop ALL --security-opt no-new-privileges \ --cpus 2 --memory 2g --memory-swap 2g --pids-limit 256 "${IMAGE_TAG}" bash -lc \ 'set -euo pipefail; trap "rig daemon stop >/dev/null 2>&1 || true" EXIT; rig --version; rig daemon start --no-kernel; curl -fsS http://127.0.0.1:7433/healthz; rig daemon status' \ @@ -111,7 +115,7 @@ node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 30 docker inspect "${L > "${OUT_DIR}/image-load.container.json" || inspect_status=$? [ "${load_status}" -eq 0 ] || exit "${load_status}" [ "${inspect_status}" -eq 0 ] || exit "${inspect_status}" -node "${REPO_ROOT}/scripts/scenario-executor.mjs" timeout 30 docker rm -f "${LOAD_CONTAINER}" >/dev/null +node "${REPO_ROOT}/scripts/scenario-executor.mjs" settle "${LOAD_CID}" "${LOAD_CONTAINER}" "${IMAGE_ID}" >/dev/null || { echo "Cleanup incomplete; retained container name: ${LOAD_CONTAINER}" >&2; LOAD_CONTAINER=""; exit 2; } LOAD_CONTAINER="" # --- emit the reproducible manifest + census receipt via the tested node orchestrator --- diff --git a/scripts/build-testbed-image.test.mjs b/scripts/build-testbed-image.test.mjs index e2ab92fea..6a28b1ee6 100644 --- a/scripts/build-testbed-image.test.mjs +++ b/scripts/build-testbed-image.test.mjs @@ -137,30 +137,45 @@ const p=spawnSync(${JSON.stringify(process.execPath)},args,{stdio:'inherit'});pr `, { mode: 0o755 }); writeFileSync(join(root, "bin/docker"), `#!${process.execPath} import fs from 'node:fs';const args=process.argv.slice(2);const marker=process.env.FAKE_CONTAINER; +const id='a'.repeat(64),image='sha256:'+'b'.repeat(64); if(args[0]==='version')console.log(JSON.stringify({Os:'linux',Arch:'amd64'})); +else if(args[0]==='image')console.log(image); else if(args[0]==='build'){} else if(args[0]==='run'){ - const at=args.indexOf('--name');if(at<0)process.exit(9); - fs.writeFileSync(marker,args[at+1]); + const at=args.indexOf('--name'),cid=args.indexOf('--cidfile');if(at<0||cid<0)process.exit(9); + fs.writeFileSync(args[cid+1],id);fs.writeFileSync(marker,JSON.stringify({Id:id,Name:'/'+args[at+1],Image:image,State:{Running:process.env.LOAD_CASE==='timeout',Status:process.env.LOAD_CASE==='timeout'?'running':'exited'}})); if(process.env.LOAD_CASE==='timeout')setInterval(()=>{},1000); else process.exit(process.env.LOAD_CASE==='failure'?7:0); -}else if(args[0]==='inspect')console.log('[]'); +}else if(args[0]==='inspect')console.log(JSON.stringify([JSON.parse(fs.readFileSync(marker))])); +else if(args[0]==='kill'){ + if(args.join(' ')!=='kill --signal TERM '+id)process.exit(8); + const state=JSON.parse(fs.readFileSync(marker));state.State={Running:false,Status:'exited'};fs.writeFileSync(marker,JSON.stringify(state)); +}else if(args[0]==='wait'){if(JSON.parse(fs.readFileSync(marker)).State.Running)process.exit(8);console.log('0');} else if(args[0]==='rm'){ - if(fs.readFileSync(marker,'utf8')!==args.at(-1))process.exit(8); + if(process.env.LOAD_CASE==='retain')process.exit(8); + if(args.length!==2||args[1]!==id||JSON.parse(fs.readFileSync(marker)).State.Running)process.exit(8); fs.unlinkSync(marker); }else process.exit(8); `, { mode: 0o755 }); - for (const [mode, status] of [["success", 0], ["failure", 7], ["timeout", 124]]) { + for (const [mode, status] of [["success", 0], ["success", 0], ["failure", 7], ["timeout", 124], ["retain", 2]]) { const out = join(root, mode), marker = join(root, "container"); const run = spawnSync("/bin/bash", [join(root, "scripts/build-testbed-image.sh"), out], { env: { PATH: `${join(root, "bin")}:/usr/bin:/bin`, HOME: root, TMPDIR: root, LOAD_CASE: mode, FAKE_CONTAINER: marker }, encoding: "utf8", timeout: 8000, }); assert.equal(run.status, status, `${mode}: ${run.stderr}`); - assert.equal(Number(readFileSync(join(out, "image-load.exit-code.txt"), "utf8")), status); + assert.equal(Number(readFileSync(join(out, "image-load.exit-code.txt"), "utf8")), mode === "retain" ? 0 : status); assert.match(readFileSync(join(out, "image-load.container-name.txt"), "utf8"), /^openrig-testbed-load-/); - assert.equal(existsSync(marker), false, `${mode}: container must be removed even after timeout`); + assert.equal(existsSync(marker), mode === "retain", `${mode}: only confirmed removal clears the container`); + assert.equal(existsSync(join(out, "image-load.cid")), mode === "retain", `${mode}: cidfile follows confirmed removal`); assert.equal(existsSync(join(out, "manifest.json")), status === 0); + if (mode === "retain") { + const retry = spawnSync("/bin/bash", [join(root, "scripts/build-testbed-image.sh"), out], { + env: { PATH: `${join(root, "bin")}:/usr/bin:/bin`, HOME: root, TMPDIR: root, LOAD_CASE: mode, FAKE_CONTAINER: marker }, + encoding: "utf8", timeout: 3000, + }); + assert.equal(retry.status, 2); assert.match(retry.stderr, /Retained cidfile/); + } } } finally { rmSync(root, { recursive: true, force: true }); } }); diff --git a/scripts/run-pr-scenarios.sh b/scripts/run-pr-scenarios.sh index 8bf2ca320..9b91fb7e1 100644 --- a/scripts/run-pr-scenarios.sh +++ b/scripts/run-pr-scenarios.sh @@ -61,9 +61,11 @@ BASE="openrig-testbed:$SHA" IMAGE="openrig-pr-scenarios:$SHA" CONTEXT="$(mktemp -d)" CONTAINER="" +CONTAINER_CID="" +IMAGE_ID="" cleanup() { if [ -n "$CONTAINER" ]; then - node scripts/scenario-executor.mjs timeout 30 docker rm -f "$CONTAINER" >/dev/null || echo "Cleanup incomplete; retained container name: $CONTAINER" >&2 + node scripts/scenario-executor.mjs settle "$CONTAINER_CID" "$CONTAINER" "$IMAGE_ID" >/dev/null || echo "Cleanup incomplete; retained container name: $CONTAINER" >&2 fi rm -rf "$CONTEXT" } @@ -78,17 +80,20 @@ node_modules/.bin/esbuild packages/test-system/ci/run.mjs --bundle --platform=no --outfile="$CONTEXT/runner.mjs" --metafile="$OUT/runner-inputs.json" docker build --network none --platform "$TARGET_PLATFORM" --build-arg TESTBED_IMAGE="$BASE" -t "$IMAGE" "$CONTEXT" docker image inspect "$IMAGE" > "$OUT/image-inspect.json" +IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$IMAGE")" attempt=0 run_case() { # run_case local scenario="$1" mode="$2" status=0 LOG attempt=$((attempt + 1)) LOG="$OUT/$attempt-$scenario-$mode.log" + CONTAINER_CID="$OUT/$attempt-$scenario-$mode.cid" + test ! -e "$CONTAINER_CID" || { echo "Retained cidfile: $CONTAINER_CID" >&2; exit 2; } CONTAINER="openrig-pr-${SHA:0:12}-$attempt-$$" printf '%s\n' "$CONTAINER" > "$OUT/$attempt-$scenario-$mode.container-name.txt" # Fresh writable scratch only. No mounts, host networking, credentials or Docker # socket; the container is non-root, resource bounded and removed even on failure. - node scripts/scenario-executor.mjs timeout 300 docker run --name "$CONTAINER" --platform "$TARGET_PLATFORM" --network none \ + node scripts/scenario-executor.mjs timeout 300 docker run --name "$CONTAINER" --cidfile "$CONTAINER_CID" --platform "$TARGET_PLATFORM" --network none \ --read-only --tmpfs /tmp:rw,exec,nosuid,nodev,size=512m,mode=1777 \ --cap-drop ALL --security-opt no-new-privileges --pids-limit 256 \ --memory 2g --memory-swap 2g --cpus 2 "$IMAGE" node /opt/openrig-testbed/runner.mjs "$mode" "$scenario" \ @@ -96,7 +101,7 @@ run_case() { # run_case printf '%s\n' "$status" > "$OUT/$attempt-$scenario-$mode.exit-code.txt" cat "$LOG" node scripts/scenario-executor.mjs timeout 30 docker inspect "$CONTAINER" > "$OUT/$attempt-$scenario-$mode.container.json" - node scripts/scenario-executor.mjs timeout 30 docker rm -f "$CONTAINER" >/dev/null + node scripts/scenario-executor.mjs settle "$CONTAINER_CID" "$CONTAINER" "$IMAGE_ID" >/dev/null || { echo "Cleanup incomplete; retained container name: $CONTAINER" >&2; CONTAINER=""; exit 2; } CONTAINER="" node --input-type=module - "$mode" "$status" "$LOG" "$scenario" <<'JS' import { readFileSync } from 'node:fs'; diff --git a/scripts/scenario-executor.mjs b/scripts/scenario-executor.mjs index c410b8f56..672a7a3f2 100644 --- a/scripts/scenario-executor.mjs +++ b/scripts/scenario-executor.mjs @@ -1,6 +1,6 @@ // Client-side helpers for the existing scenario scripts; never runs a daemon. import { spawn, spawnSync } from 'node:child_process'; -import { writeFileSync } from 'node:fs'; +import { readFileSync, unlinkSync, writeFileSync } from 'node:fs'; import { constants } from 'node:os'; import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -12,6 +12,37 @@ export function dockerPlatform(server) { return `linux/${server.Arch}`; } +// A client timeout does not establish that its engine-side container stopped. +// Unknown identity or settlement leaves the container for its owner to reconcile. +export function settleContainer({ id, name, image }, invoke = args => { + const result = spawnSync('docker', args, { encoding: 'utf8', timeout: 35000, maxBuffer: 1024 * 1024 }); + if (result.error || result.status !== 0) throw new Error(result.error?.message ?? result.stderr); + return result.stdout; +}) { + if (!/^[a-f0-9]{64}$/.test(id) || !/^sha256:[a-f0-9]{64}$/.test(image) || !name) { + throw new Error('Exact created container ID, name and image ID required'); + } + const inspect = () => { + const objects = JSON.parse(invoke(['inspect', id])); + const object = objects[0]; + if (objects.length !== 1 || object.Id !== id || object.Name !== '/' + name || object.Image !== image) { + throw new Error('Container identity unknown; retained'); + } + return object.State; + }; + let state = inspect(); + if (state.Running === true) { + // Explicit TERM has no automatic KILL escalation. A failed wait retains it. + invoke(['kill', '--signal', 'TERM', id]); + invoke(['wait', id]); + state = inspect(); + } + if (state.Running !== false || !['created', 'exited'].includes(state.Status)) { + throw new Error('Container not confirmed stopped; retained'); + } + invoke(['rm', id]); +} + export async function runWithDeadline(argv, { timeoutMs, killAfterMs = 15000 }) { if (!argv.length || !Number.isFinite(timeoutMs) || timeoutMs <= 0 || killAfterMs <= 0) { throw new Error('Command and positive finite deadline required'); @@ -63,10 +94,14 @@ if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.ur const platform = dockerPlatform(server); if (args[0]) writeFileSync(args[0], JSON.stringify({ platform, server }, null, 2) + '\n'); console.log(platform); + } else if (operation === 'settle') { + const [cidFile, name, image] = args; + settleContainer({ id: readFileSync(cidFile, 'utf8').trim(), name, image }); + unlinkSync(cidFile); } else if (operation === 'timeout') { process.exitCode = await runWithDeadline(args.slice(1), { timeoutMs: Number(args[0]) * 1000 }); } else { - throw new Error('Expected platform [receipt-path] or timeout [args...]'); + throw new Error('Expected platform, settle , or timeout [args...]'); } } catch (error) { console.error(error.message); diff --git a/scripts/scenario-executor.test.mjs b/scripts/scenario-executor.test.mjs index 869ed4514..f3ab9758c 100644 --- a/scripts/scenario-executor.test.mjs +++ b/scripts/scenario-executor.test.mjs @@ -58,3 +58,26 @@ test('ordinary invocation and ambiguous remote selection refuse before any build assert.match(run.stderr, message); } }); + +test('settlement uses the recorded ID, TERM and stopped-state removal; unknown effects stay retained', async () => { + const { settleContainer } = await import('./scenario-executor.mjs'); + assert.equal(typeof settleContainer, 'function'); + const id='a'.repeat(64), image='sha256:'+'b'.repeat(64), name='testbed-owned'; + function fixture({running=false,mismatch=false,waitUnknown=false,stillRunning=false}={}) { + const calls=[];let inspected=0; + const invoke=args=>{ + calls.push(args); + if(args[0]==='inspect')return JSON.stringify([{Id:mismatch?'c'.repeat(64):id,Name:'/'+name,Image:image,State:{Running:inspected++===0?running:stillRunning,Status:(inspected===1?running:stillRunning)?'running':'exited'}}]); + if(args[0]==='wait'&&waitUnknown)throw Error('wait timed out; engine state unknown'); + return ''; + }; + return {calls,invoke}; + } + const owned={id,name,image}; + const stopped=fixture();settleContainer(owned,stopped.invoke);assert.deepEqual(stopped.calls,[['inspect',id],['rm',id]]); + const live=fixture({running:true});settleContainer(owned,live.invoke);assert.deepEqual(live.calls,[['inspect',id],['kill','--signal','TERM',id],['wait',id],['inspect',id],['rm',id]]); + for(const opts of [{mismatch:true},{running:true,waitUnknown:true},{running:true,stillRunning:true}]){ + const f=fixture(opts);assert.throws(()=>settleContainer(owned,f.invoke));assert.ok(!f.calls.some(x=>x[0]==='rm')); + } + const f=fixture();assert.throws(()=>settleContainer({...owned,id:'not-an-id'},f.invoke));assert.equal(f.calls.length,0); +});