From 5aeee2ceec3d0b70c44f7a3686a4d921ba6d7fc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=93=E8=89=AF?= <1204183885@qq.com> Date: Sun, 25 Oct 2026 16:51:21 +0800 Subject: [PATCH] fix(release): authenticate managed OSS preflight (#15504) --- .github/workflows/release-managed-runtime.yml | 78 +++-- docs/design/managed-runtime-oss-preflight.md | 66 ++++ .../managed-runtime-oss-preflight.zh-CN.md | 56 ++++ scripts/managed-runtime-release.js | 106 +++++- scripts/tests/managed-runtime-release.test.js | 303 +++++++++++++++++- 5 files changed, 575 insertions(+), 34 deletions(-) create mode 100644 docs/design/managed-runtime-oss-preflight.md create mode 100644 docs/design/managed-runtime-oss-preflight.zh-CN.md diff --git a/.github/workflows/release-managed-runtime.yml b/.github/workflows/release-managed-runtime.yml index 6fd0ee4db19..b9a1d1d2f08 100644 --- a/.github/workflows/release-managed-runtime.yml +++ b/.github/workflows/release-managed-runtime.yml @@ -93,6 +93,7 @@ jobs: CLI_VERSION: '${{ needs.prepare.outputs.version }}' SOURCE_SHA: '${{ needs.prepare.outputs.sha }}' BUILD_TIMESTAMP: '${{ needs.prepare.outputs.timestamp }}' + ALIYUN_OSS_BUCKET: "${{ vars.ALIYUN_OSS_BUCKET || 'qwen-code-assets' }}" OSS_PUBLIC_BASE_URL: "${{ vars.ALIYUN_OSS_PUBLIC_BASE_URL || 'https://qwen-code-assets.oss-cn-hangzhou.aliyuncs.com' }}" steps: - name: 'Checkout the pinned source' @@ -101,6 +102,14 @@ jobs: ref: '${{ needs.prepare.outputs.sha }}' persist-credentials: false + - name: 'Checkout release tooling from the workflow revision' + uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 + with: + ref: '${{ github.workflow_sha }}' + path: '.qwen/managed-release-tools' + sparse-checkout: 'scripts' + persist-credentials: false + - name: 'Set up Java 21' uses: 'actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654' # v5.2.0 with: @@ -137,7 +146,7 @@ jobs: - name: 'Verify the executable JAR and publication profile' run: |- mvn --batch-mode --no-transfer-progress -f packages/managed-agent-server/pom.xml clean verify -Pcentral-release -Dgpg.skip=true -Dproject.build.outputTimestamp="${BUILD_TIMESTAMP}" - node scripts/managed-runtime-release.js --jar "packages/managed-agent-server/target/qwen-managed-agent-server-${CLI_VERSION}.jar" --version "${CLI_VERSION}" --out-dir dist/managed-runtime + node .qwen/managed-release-tools/scripts/managed-runtime-release.js --jar "packages/managed-agent-server/target/qwen-managed-agent-server-${CLI_VERSION}.jar" --version "${CLI_VERSION}" --out-dir dist/managed-runtime - name: 'Retain verified JAR and checksum' uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # v7.0.1 @@ -146,6 +155,36 @@ jobs: path: 'dist/managed-runtime/*' if-no-files-found: 'error' + - name: 'Install the verified OSS uploader' + if: "${{ github.event_name == 'release' || !inputs.dry_run }}" + env: + OSSUTIL_URL: 'https://gosspublic.alicdn.com/ossutil/1.7.19/ossutil-v1.7.19-linux-amd64.zip' + OSSUTIL_SHA256: 'dcc512e4a893e16bbee63bc769339d8e56b21744fd83c8212a9d8baf28767343' + run: |- + set -euo pipefail + work=$(mktemp -d) + trap 'rm -rf "${work}"' EXIT + curl -fsSL --connect-timeout 15 --max-time 300 "${OSSUTIL_URL}" -o "${work}/ossutil.zip" + echo "${OSSUTIL_SHA256} ${work}/ossutil.zip" | sha256sum -c - + unzip -q "${work}/ossutil.zip" -d "${work}" + binary=$(find "${work}" -type f -name 'ossutil64' -print -quit) + test -n "${binary}" + mkdir -p "${HOME}/.local/bin" + install -m 0755 "${binary}" "${HOME}/.local/bin/ossutil" + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: 'Configure authenticated OSS access' + if: "${{ github.event_name == 'release' || !inputs.dry_run }}" + env: + ALIYUN_OSS_ACCESS_KEY_ID: '${{ secrets.ALIYUN_OSS_ACCESS_KEY_ID }}' + ALIYUN_OSS_ACCESS_KEY_SECRET: '${{ secrets.ALIYUN_OSS_ACCESS_KEY_SECRET }}' + ALIYUN_OSS_ENDPOINT: "${{ vars.ALIYUN_OSS_ENDPOINT || 'https://oss-cn-hangzhou.aliyuncs.com' }}" + run: |- + set -euo pipefail + umask 077 + [[ -n "${ALIYUN_OSS_ACCESS_KEY_ID}" && -n "${ALIYUN_OSS_ACCESS_KEY_SECRET}" ]] + ossutil config -e "${ALIYUN_OSS_ENDPOINT}" -i "${ALIYUN_OSS_ACCESS_KEY_ID}" -k "${ALIYUN_OSS_ACCESS_KEY_SECRET}" -L EN -c "${RUNNER_TEMP}/managed-ossutilconfig" + - name: 'Recheck the immutable tag and existing publication bytes' id: 'publication' if: "${{ github.event_name == 'release' || !inputs.dry_run }}" @@ -162,18 +201,18 @@ jobs: gh api "repos/QwenLM/qwen-code/releases/tags/${RELEASE_TAG}" | jq --exit-status '.draft == false' jar="qwen-managed-agent-server-${CLI_VERSION}.jar" central="https://repo.maven.apache.org/maven2/com/alibaba/qwen-managed-agent-server/${CLI_VERSION}/${jar}" - state=$(node scripts/managed-runtime-release.js --check-url "${central}" --file "dist/managed-runtime/${jar}") || exit 1 + state=$(node .qwen/managed-release-tools/scripts/managed-runtime-release.js --check-url "${central}" --file "dist/managed-runtime/${jar}") || exit 1 echo "central=${state}" >> "${GITHUB_OUTPUT}" : > "${RUNNER_TEMP}/managed-github-missing" : > "${RUNNER_TEMP}/managed-oss-missing" for asset in "${jar}" "${jar}.sha256"; do github="https://github.com/QwenLM/qwen-code/releases/download/${RELEASE_TAG}/${asset}" - oss="${OSS_PUBLIC_BASE_URL}/releases/qwen-code/${RELEASE_TAG}/${asset}" - state=$(node scripts/managed-runtime-release.js --check-url "${github}" --file "dist/managed-runtime/${asset}") || exit 1 + oss="oss://${ALIYUN_OSS_BUCKET}/releases/qwen-code/${RELEASE_TAG}/${asset}" + state=$(node .qwen/managed-release-tools/scripts/managed-runtime-release.js --check-url "${github}" --file "dist/managed-runtime/${asset}") || exit 1 if [[ "${state}" == 'missing' ]]; then echo "dist/managed-runtime/${asset}" >> "${RUNNER_TEMP}/managed-github-missing" fi - state=$(node scripts/managed-runtime-release.js --check-url "${oss}" --file "dist/managed-runtime/${asset}") || exit 1 + state=$(node .qwen/managed-release-tools/scripts/managed-runtime-release.js --check-oss "${oss}" --config "${RUNNER_TEMP}/managed-ossutilconfig" --file "dist/managed-runtime/${asset}") || exit 1 if [[ "${state}" == 'missing' ]]; then echo "dist/managed-runtime/${asset}" >> "${RUNNER_TEMP}/managed-oss-missing" fi @@ -212,7 +251,7 @@ jobs: set -euo pipefail jar="qwen-managed-agent-server-${CLI_VERSION}.jar" for attempt in {1..40}; do - state=$(node scripts/managed-runtime-release.js --check-url "https://repo.maven.apache.org/maven2/com/alibaba/qwen-managed-agent-server/${CLI_VERSION}/${jar}" --file "dist/managed-runtime/${jar}") + state=$(node .qwen/managed-release-tools/scripts/managed-runtime-release.js --check-url "https://repo.maven.apache.org/maven2/com/alibaba/qwen-managed-agent-server/${CLI_VERSION}/${jar}" --file "dist/managed-runtime/${jar}") if [[ "${state}" == 'present' ]]; then exit 0; fi sleep 30 done @@ -228,37 +267,12 @@ jobs: mapfile -t assets < "${RUNNER_TEMP}/managed-github-missing" if (( ${#assets[@]} )); then gh release upload "${RELEASE_TAG}" "${assets[@]}"; fi - - name: 'Install the verified OSS uploader' - if: "${{ github.event_name == 'release' || !inputs.dry_run }}" - env: - OSSUTIL_URL: 'https://gosspublic.alicdn.com/ossutil/1.7.19/ossutil-v1.7.19-linux-amd64.zip' - OSSUTIL_SHA256: 'dcc512e4a893e16bbee63bc769339d8e56b21744fd83c8212a9d8baf28767343' - run: |- - set -euo pipefail - work=$(mktemp -d) - trap 'rm -rf "${work}"' EXIT - curl -fsSL --connect-timeout 15 --max-time 300 "${OSSUTIL_URL}" -o "${work}/ossutil.zip" - echo "${OSSUTIL_SHA256} ${work}/ossutil.zip" | sha256sum -c - - unzip -q "${work}/ossutil.zip" -d "${work}" - binary=$(find "${work}" -type f -name 'ossutil64' -print -quit) - test -n "${binary}" - mkdir -p "${HOME}/.local/bin" - install -m 0755 "${binary}" "${HOME}/.local/bin/ossutil" - echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" - - name: 'Upload only missing OSS release assets' if: "${{ github.event_name == 'release' || !inputs.dry_run }}" - env: - ALIYUN_OSS_ACCESS_KEY_ID: '${{ secrets.ALIYUN_OSS_ACCESS_KEY_ID }}' - ALIYUN_OSS_ACCESS_KEY_SECRET: '${{ secrets.ALIYUN_OSS_ACCESS_KEY_SECRET }}' - ALIYUN_OSS_ENDPOINT: "${{ vars.ALIYUN_OSS_ENDPOINT || 'https://oss-cn-hangzhou.aliyuncs.com' }}" - ALIYUN_OSS_BUCKET: "${{ vars.ALIYUN_OSS_BUCKET || 'qwen-code-assets' }}" run: |- set -euo pipefail mapfile -t assets < "${RUNNER_TEMP}/managed-oss-missing" if (( ${#assets[@]} == 0 )); then exit 0; fi - [[ -n "${ALIYUN_OSS_ACCESS_KEY_ID}" && -n "${ALIYUN_OSS_ACCESS_KEY_SECRET}" ]] - ossutil config -e "${ALIYUN_OSS_ENDPOINT}" -i "${ALIYUN_OSS_ACCESS_KEY_ID}" -k "${ALIYUN_OSS_ACCESS_KEY_SECRET}" -L EN -c "${RUNNER_TEMP}/managed-ossutilconfig" node scripts/upload-aliyun-oss-assets.js --bucket "${ALIYUN_OSS_BUCKET}" --config "${RUNNER_TEMP}/managed-ossutilconfig" --prefix "releases/qwen-code/${RELEASE_TAG}" "${assets[@]}" - name: 'Verify both public download channels' @@ -268,6 +282,6 @@ jobs: jar="qwen-managed-agent-server-${CLI_VERSION}.jar" for base in "https://github.com/QwenLM/qwen-code/releases/download/${RELEASE_TAG}" "${OSS_PUBLIC_BASE_URL}/releases/qwen-code/${RELEASE_TAG}"; do for asset in "${jar}" "${jar}.sha256"; do - [[ "$(node scripts/managed-runtime-release.js --check-url "${base}/${asset}" --file "dist/managed-runtime/${asset}")" == 'present' ]] + [[ "$(node .qwen/managed-release-tools/scripts/managed-runtime-release.js --check-url "${base}/${asset}" --file "dist/managed-runtime/${asset}")" == 'present' ]] done done diff --git a/docs/design/managed-runtime-oss-preflight.md b/docs/design/managed-runtime-oss-preflight.md new file mode 100644 index 00000000000..43ec28982d8 --- /dev/null +++ b/docs/design/managed-runtime-oss-preflight.md @@ -0,0 +1,66 @@ +# Authenticated OSS preflight for Managed runtime releases + +[English](managed-runtime-oss-preflight.md) | [简体中文](managed-runtime-oss-preflight.zh-CN.md) + +## Problem and scope + +Anonymous OSS GET can return `403 AccessDenied` for an absent object under the +bucket ACL. Treating that response as missing would also permit overwriting an +inaccessible existing release. Only the Managed runtime release preflight changes; +Java artifacts, product code, public verification, and publication immutability +checks remain unchanged. + +## Decision + +Keep the artifact checkout pinned to the release's `SOURCE_SHA`. Separately check +out `scripts` from `github.workflow_sha` into the already ignored +`.qwen/managed-release-tools` directory, without persisted credentials. Run the +release helper there so backfills of old tags can use the new preflight. Its +imports resolve through the root installation; the source checkout, Git metadata, +version, and build timestamp remain unchanged. + +Install the existing SHA-256-pinned ossutil 1.7.19 and configure its existing +production-release credentials before preflight. Use the same bucket, object key, +and configuration for preflight and upload. Create the configuration with +`umask 077`; dry runs skip both setup and authenticated reads. + +Add `--check-oss OSS_URL --config PATH --file PATH` to the release helper. Accept +only an `oss://bucket/object` URL without credentials, port, query, or fragment. +Run `ossutil cat` without a shell, writing stdout into a private temporary file, +with a 300-second timeout and bounded stderr. This command cannot write remotely. + +The pinned [cat implementation](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/cat.go) +calls `GetObject`, copies raw bytes to stdout, and appends one newline. The +[command wrapper](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/command_manager.go#L57) +then prints `\n%.6f(s) elapsed\n`. Read at most 64 trailer bytes after the expected +artifact size and require precisely `\n\n.<6 digits>(s) elapsed\n`. +Remove that trailer and compare SHA-256 using the existing helper. This also detects truncated output even though +this version of cat ignores the `io.Copy` error. Original terminal newlines are +preserved. Always remove the temporary download. + +`cp` is unsuitable here: its [download precheck](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/cp.go#L2489) +uses [GetObjectDetailedMeta](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/command.go#L658), +a HEAD request, whose empty error body cannot reliably distinguish missing keys. +The CLI [prints errors on stdout and exits 1](https://github.com/aliyun/ossutil/blob/v1.7.19/ossutil.go). +Only exit 1 with the complete canonical line beginning +`Error: oss: service returned error: StatusCode=404, ErrorCode=NoSuchKey, ErrorMessage=` +is missing. The format comes from its pinned +[OSS SDK v3.0.2](https://github.com/aliyun/aliyun-oss-go-sdk/blob/v3.0.2/oss/error.go). +Read at most 16 KiB of error output. AccessDenied, NoSuchBucket, unstructured 404, +network errors, subprocess errors, timeouts, and mismatched bytes fail closed. +Do not echo subprocess diagnostics or credentials. + +## Verification and limits + +Real subprocess shims cover identical binary bytes, terminal newlines, exact +NoSuchKey/404, other error classes, mismatches, timeout, spawn failure, argument +validation, and download cleanup. Workflow tests enforce setup order, pinned +uploader, credential/configuration reuse, dry-run guards, and final public HTTPS +verification. A local old-tag fixture checks that the workflow-selected helper runs +with root dependencies while the old helper remains unchanged. The new behavioral tests fail against the original script because +`--check-oss` is unsupported, then pass with the fix. + +No real credentials, authenticated remote requests, workflow dispatch, or +publication are part of local validation. Actual OSS access still depends on the +existing credential permissions; insufficient access deliberately fails the +release. Upgrading ossutil requires rechecking its stdout and error contracts. diff --git a/docs/design/managed-runtime-oss-preflight.zh-CN.md b/docs/design/managed-runtime-oss-preflight.zh-CN.md new file mode 100644 index 00000000000..206ad5b7ff3 --- /dev/null +++ b/docs/design/managed-runtime-oss-preflight.zh-CN.md @@ -0,0 +1,56 @@ +# Managed runtime 发布的 OSS 认证预检 + +[English](managed-runtime-oss-preflight.md) | [简体中文](managed-runtime-oss-preflight.zh-CN.md) + +## 问题与范围 + +受 bucket ACL 影响，匿名 OSS GET 对不存在的对象也可能返回 `403 AccessDenied`。 +把此响应视为缺失，会允许覆盖无权访问的现有发布。本次只修改 Managed runtime +发布预检；Java 产物、产品代码、公开访问验证及发布不可变性检查保持不变。 + +## 方案 + +产物源码继续检出发布的 `SOURCE_SHA`。另将 `github.workflow_sha` 的 `scripts` 检出到 +已有忽略规则覆盖的 `.qwen/managed-release-tools` 目录，不持久化凭据。使用此处的 +发布辅助脚本，使旧标签补发也能运行新增预检。依赖通过根目录安装解析；源码检出、 +Git 元数据、版本和构建时间戳保持不变。 + +在预检前安装现有 SHA-256 固定的 ossutil 1.7.19，并配置现有 production-release +凭据。预检与上传使用相同 bucket、对象路径和配置。以 `umask 077` 创建配置文件； +dry run 跳过配置与认证读取。 + +发布脚本增加 `--check-oss OSS_URL --config PATH --file PATH`。仅接受不含凭据、 +端口、查询参数或片段的 `oss://bucket/object` URL。直接启动 `ossutil cat`，不经过 +shell，将 stdout 写入私有临时文件，设置 300 秒超时并限制 stderr 大小。 +此命令不能写入远端。 + +固定版本的 [cat 实现](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/cat.go) +调用 `GetObject`，把原始字节复制到 stdout，再追加一个换行符。 +[命令包装层](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/command_manager.go#L57) +随后输出 `\n%.6f(s) elapsed\n`。从预期产物大小处最多读取 64 字节，要求尾部精确符合 +`\n\n<秒数>.<6 位小数>(s) elapsed\n`，移除此尾部后用现有辅助函数比较 SHA-256。即使此版本 cat +忽略 `io.Copy` 的错误，这也能检测输出截断。保留产物自身的末尾换行符，并始终清理 +临时下载文件。 + +`cp` 不适合此场景：其[下载预检](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/cp.go#L2489) +使用 [GetObjectDetailedMeta](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/command.go#L658)， +这是 HEAD 请求，空错误响应体不能可靠区分缺失对象。 +CLI [把错误打印到 stdout 并以状态 1 退出](https://github.com/aliyun/ossutil/blob/v1.7.19/ossutil.go)。 +只有退出状态为 1，且输出为完整规范错误行、开头是 +`Error: oss: service returned error: StatusCode=404, ErrorCode=NoSuchKey, ErrorMessage=`， +才认定对象缺失。此格式来自固定依赖 +[OSS SDK v3.0.2](https://github.com/aliyun/aliyun-oss-go-sdk/blob/v3.0.2/oss/error.go)。 +错误输出最多读取 16 KiB。AccessDenied、NoSuchBucket、非结构化 404、网络错误、 +子进程错误、超时和字节不一致均阻止发布。不回显子进程诊断或凭据。 + +## 验证与限制 + +真实子进程替身覆盖相同二进制字节、末尾换行符、精确 NoSuchKey/404、其他错误类别、 +字节不一致、超时、启动失败、参数校验和下载清理。工作流测试约束配置顺序、固定 +上传器、凭据与配置复用、dry-run 条件及最终公开 HTTPS 验证。本地旧标签夹具验证 +工作流选择的辅助脚本能解析根目录依赖，同时旧脚本保持不变。新增行为测试在原始 +脚本上因不支持 `--check-oss` 而失败，修复后通过。 + +本地验证不访问真实凭据，不执行认证远端请求、工作流触发或发布。真实 OSS 访问仍 +依赖现有凭据权限；权限不足时应阻止发布。升级 ossutil 时须重新核对其 stdout 和 +错误格式契约。 diff --git a/scripts/managed-runtime-release.js b/scripts/managed-runtime-release.js index 6f43b1d1ccb..1d975a1e424 100644 --- a/scripts/managed-runtime-release.js +++ b/scripts/managed-runtime-release.js @@ -5,6 +5,8 @@ */ import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import os from 'node:os'; import fs from 'node:fs'; import path from 'node:path'; import { Readable, Transform } from 'node:stream'; @@ -82,6 +84,90 @@ export async function checkManagedRuntimeAsset(url, file, fetchImpl = fetch) { return 'present'; } +export async function checkManagedRuntimeOssAsset( + url, + file, + config, + timeout = 300_000, +) { + const parsed = new URL(url); + if ( + parsed.protocol !== 'oss:' || + !parsed.hostname || + parsed.username || + parsed.password || + parsed.port || + parsed.pathname.length <= 1 || + parsed.search || + parsed.hash + ) { + throw new Error( + 'Release artifact verification requires an OSS object URL.', + ); + } + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'managed-oss-')); + const download = path.join(temporary, 'artifact'); + try { + const output = fs.openSync(download, 'w'); + let result; + try { + // Unlike cp's HEAD precheck, cat uses GET and preserves OSS error codes. + result = spawnSync('ossutil', ['cat', url, '-c', path.resolve(config)], { + stdio: ['ignore', output, 'pipe'], + timeout, + killSignal: 'SIGKILL', + maxBuffer: 64 * 1024, + }); + } finally { + fs.closeSync(output); + } + if (result.error || result.status !== 0) { + if ( + !result.error && + result.status === 1 && + fs.statSync(download).size <= 16 * 1024 && + /^Error: oss: service returned error: StatusCode=404, ErrorCode=NoSuchKey, ErrorMessage="[^\r\n]*", RequestId=[^\r\n]+\r?\n?$/.test( + fs.readFileSync(download, 'utf8'), + ) + ) { + return 'missing'; + } + throw new Error( + 'Cannot verify existing OSS artifact; refusing publication.', + ); + } + const size = fs.statSync(file).size; + // cat appends a newline, then the 1.7.19 command wrapper prints elapsed time. + const trailer = Buffer.alloc(64); + const input = fs.openSync(download, 'r'); + let length; + try { + length = fs.readSync(input, trailer, 0, trailer.length, size); + } finally { + fs.closeSync(input); + } + if ( + fs.statSync(download).size !== size + length || + !/^\n\n\d+\.\d{6}\(s\) elapsed\n$/.test( + trailer.toString('utf8', 0, length), + ) + ) { + throw new Error( + 'Existing release artifact has different bytes; refusing overwrite.', + ); + } + fs.truncateSync(download, size); + if ((await sha256File(download)) !== (await sha256File(file))) { + throw new Error( + 'Existing release artifact has different bytes; refusing overwrite.', + ); + } + return 'present'; + } finally { + fs.rmSync(temporary, { recursive: true, force: true }); + } +} + if (isMainModule(import.meta.url)) { try { const args = parseArgs(process.argv.slice(2), { @@ -89,25 +175,43 @@ if (isMainModule(import.meta.url)) { '--version': { key: 'version', type: 'value' }, '--out-dir': { key: 'outDir', type: 'value' }, '--check-url': { key: 'url', type: 'value' }, + '--check-oss': { key: 'oss', type: 'value' }, + '--config': { key: 'config', type: 'value' }, '--file': { key: 'file', type: 'value' }, }); if (args.help) { console.log( - 'Stage: --jar PATH --version VERSION --out-dir DIR\nVerify without replacing: --check-url HTTPS_URL --file PATH (prints present or missing).', + 'Stage: --jar PATH --version VERSION --out-dir DIR\nVerify without replacing: --check-url HTTPS_URL --file PATH (prints present or missing).\nAuthenticated OSS: --check-oss OSS_URL --config PATH --file PATH.', ); } else if ( args.url && + !args.oss && + !args.config && args.file && !args.jar && !args.version && !args.outDir ) { console.log(await checkManagedRuntimeAsset(args.url, args.file)); + } else if ( + args.oss && + args.config && + args.file && + !args.url && + !args.jar && + !args.version && + !args.outDir + ) { + console.log( + await checkManagedRuntimeOssAsset(args.oss, args.file, args.config), + ); } else if ( args.jar && args.version && args.outDir && !args.url && + !args.oss && + !args.config && !args.file ) { console.log(JSON.stringify(await stageManagedRuntimeRelease(args))); diff --git a/scripts/tests/managed-runtime-release.test.js b/scripts/tests/managed-runtime-release.test.js index d5c2ba88dd3..861e71c2c33 100644 --- a/scripts/tests/managed-runtime-release.test.js +++ b/scripts/tests/managed-runtime-release.test.js @@ -14,6 +14,7 @@ import { parse } from 'yaml'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { checkManagedRuntimeAsset, + checkManagedRuntimeOssAsset, stageManagedRuntimeRelease, } from '../managed-runtime-release.js'; @@ -38,7 +39,10 @@ beforeEach(() => { jar = path.join(temporary, 'server.jar'); fs.writeFileSync(jar, bytes); }); -afterEach(() => fs.rmSync(temporary, { recursive: true, force: true })); +afterEach(() => { + vi.unstubAllEnvs(); + fs.rmSync(temporary, { recursive: true, force: true }); +}); it('stages the exact server bytes and the checksum consumed by the first-use installer', async () => { const outDir = path.join(temporary, 'release'); @@ -150,6 +154,213 @@ it.each([ expect(fetchImpl).not.toHaveBeenCalled(); }); +describe.skipIf(process.platform === 'win32')( + 'authenticated OSS verification', + () => { + function check({ + content = bytes, + error = '', + status = 0, + url = 'oss://release-bucket/releases/server.jar', + extra = [], + script = path.join(root, 'scripts/managed-runtime-release.js'), + } = {}) { + const bin = path.join(temporary, 'bin'); + fs.mkdirSync(bin, { recursive: true }); + fs.writeFileSync(path.join(bin, 'package.json'), '{"type":"commonjs"}'); + fs.writeFileSync( + path.join(bin, 'ossutil'), + `#!${process.execPath} +const fs = require('node:fs'); +fs.writeFileSync(process.env.OSS_ARGS, JSON.stringify(process.argv.slice(2))); +process.stdout.write(Buffer.from(${JSON.stringify(content.toString('base64'))}, 'base64')); +process.stdout.write(${JSON.stringify(error || '\n\n0.001234(s) elapsed\n')}); +process.exit(${status}); +`, + { mode: 0o755 }, + ); + const result = spawnSync( + process.execPath, + [ + fs.realpathSync(script), + '--check-oss', + url, + '--config', + path.join(temporary, 'config'), + '--file', + jar, + ...extra, + ], + { + env: { + ...process.env, + PATH: `${bin}${path.delimiter}${process.env.PATH}`, + OSS_ARGS: path.join(temporary, 'args'), + TMPDIR: temporary, + }, + encoding: 'utf8', + }, + ); + expect( + fs + .readdirSync(temporary) + .filter((name) => name.startsWith('managed-oss-')), + ).toEqual([]); + return result; + } + + it('downloads authenticated bytes using only the configured read command', () => { + const result = check(); + expect(result.stderr).toBe(''); + expect(result.status).toBe(0); + expect(result.stdout).toBe('present\n'); + expect( + JSON.parse(fs.readFileSync(path.join(temporary, 'args'), 'utf8')), + ).toEqual([ + 'cat', + 'oss://release-bucket/releases/server.jar', + '-c', + path.join(temporary, 'config'), + ]); + expect(fs.readFileSync(jar)).toEqual(bytes); + }); + + it('runs current workflow tooling alongside an unchanged old-tag source tree', () => { + const command = workflow.jobs.release.steps + .find((item) => item.id === 'publication') + .run.match(/node (\S+) --check-oss/)[1]; + const script = path.join(temporary, command); + fs.mkdirSync(path.dirname(script), { recursive: true }); + for (const name of [ + 'managed-runtime-release.js', + 'release-script-utils.js', + ]) { + fs.copyFileSync( + path.join(root, 'scripts', name), + path.join(path.dirname(script), name), + ); + } + fs.writeFileSync( + path.join(temporary, 'package.json'), + '{"type":"module"}', + ); + fs.symlinkSync( + path.join(root, 'node_modules'), + path.join(temporary, 'node_modules'), + 'dir', + ); + const oldSource = path.join( + temporary, + 'scripts/managed-runtime-release.js', + ); + fs.mkdirSync(path.dirname(oldSource), { recursive: true }); + fs.writeFileSync( + oldSource, + 'throw new Error("old tag has no --check-oss");', + ); + const result = check({ script }); + expect(result.stderr).toBe(''); + expect(result.status).toBe(0); + expect(result.stdout).toBe('present\n'); + expect(fs.readFileSync(oldSource, 'utf8')).toBe( + 'throw new Error("old tag has no --check-oss");', + ); + }); + + it('preserves an artifact terminal newline when removing the CLI trailer', () => { + const content = Buffer.concat([bytes, Buffer.from('\n')]); + fs.writeFileSync(jar, content); + expect(check({ content }).stdout).toBe('present\n'); + }); + + it('refuses unexpected successful CLI output instead of discarding arbitrary bytes', () => { + const result = check({ error: '\n' }); + expect(result.status).toBe(1); + expect(result.stderr).toContain('refusing overwrite'); + }); + + it('recognizes only the canonical structured NoSuchKey 404 as missing', () => { + const result = check({ + content: Buffer.alloc(0), + status: 1, + error: + 'Error: oss: service returned error: StatusCode=404, ErrorCode=NoSuchKey, ErrorMessage="The specified key does not exist.", RequestId=request-id\n', + }); + expect(result.status).toBe(0); + expect(result.stdout).toBe('missing\n'); + }); + + it.each([ + 'Error: oss: service returned error: StatusCode=403, ErrorCode=AccessDenied, ErrorMessage="Access denied", RequestId=id\n', + 'Error: oss: service returned error: StatusCode=404, ErrorCode=NoSuchBucket, ErrorMessage="Missing bucket", RequestId=id\n', + 'Error: oss: service returned error: StatusCode=403, ErrorCode=NoSuchKey, ErrorMessage="Denied", RequestId=id\n', + 'Error: oss: service returned error: StatusCode=404, ErrorCode=AccessDenied, ErrorMessage="NoSuchKey", RequestId=id\n', + 'Error: HTTP 404 NoSuchKey\n', + 'Error: network unavailable\n', + ])('fails closed for inaccessible or unconfirmed absence: %s', (error) => { + const result = check({ content: Buffer.alloc(0), status: 1, error }); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toContain('Cannot verify existing OSS artifact'); + }); + + it.each([ + Buffer.alloc(bytes.length, 1), + bytes.subarray(0, 4), + Buffer.concat([bytes, Buffer.from('extra')]), + ])('refuses different bytes and cleans its download', (content) => { + const result = check({ content }); + expect(result.status).toBe(1); + expect(result.stderr).toContain('refusing overwrite'); + }); + + it.each([ + 'https://example.com/object', + 'oss://user:password@bucket/object', + 'oss://bucket/', + 'oss://bucket/key?versionId=old', + ])('rejects invalid OSS sources before invoking ossutil: %s', (url) => { + expect(check({ url }).status).toBe(1); + expect(fs.existsSync(path.join(temporary, 'args'))).toBe(false); + }); + + it('kills a stalled reader and removes its partial download', async () => { + fs.writeFileSync( + path.join(temporary, 'ossutil'), + `#!${process.execPath} +setInterval(() => {}, 1000); +`, + { mode: 0o755 }, + ); + vi.stubEnv('PATH', temporary); + vi.stubEnv('TMPDIR', temporary); + await expect( + checkManagedRuntimeOssAsset('oss://bucket/key', jar, 'config', 100), + ).rejects.toThrow('Cannot verify existing OSS artifact'); + expect( + fs + .readdirSync(temporary) + .filter((name) => name.startsWith('managed-oss-')), + ).toEqual([]); + }); + + it('fails closed when ossutil cannot start', async () => { + vi.stubEnv('PATH', temporary); + await expect( + checkManagedRuntimeOssAsset('oss://bucket/key', jar, 'config'), + ).rejects.toThrow('Cannot verify existing OSS artifact'); + }); + + it('rejects mixed public and authenticated verification arguments', () => { + expect( + check({ extra: ['--check-url', 'https://example.com/server.jar'] }) + .status, + ).toBe(1); + expect(fs.existsSync(path.join(temporary, 'args'))).toBe(false); + }); + }, +); + describe('Managed runtime publication gates', () => { const steps = workflow.jobs.release.steps; const step = (name) => steps.find((item) => item.name === name); @@ -170,6 +381,7 @@ describe('Managed runtime publication gates', () => { 'Publish the signed server artifact to Central', 'Attach only missing GitHub release assets', 'Install the verified OSS uploader', + 'Configure authenticated OSS access', 'Upload only missing OSS release assets', ]) { expect(step(name).if).toContain( @@ -190,9 +402,73 @@ describe('Managed runtime publication gates', () => { ); }); + it('configures the pinned OSS reader before preflight and reuses the upload destination and credentials', () => { + const install = step('Install the verified OSS uploader'); + const configure = step('Configure authenticated OSS access'); + const preflight = step( + 'Recheck the immutable tag and existing publication bytes', + ); + const upload = step('Upload only missing OSS release assets'); + expect(steps.indexOf(install)).toBeLessThan(steps.indexOf(configure)); + expect(steps.indexOf(configure)).toBeLessThan(steps.indexOf(preflight)); + expect(steps.indexOf(preflight)).toBeLessThan( + steps.indexOf(step('Publish the signed server artifact to Central')), + ); + expect(install.env.OSSUTIL_URL).toContain('/1.7.19/'); + expect(install.env.OSSUTIL_SHA256).toBe( + 'dcc512e4a893e16bbee63bc769339d8e56b21744fd83c8212a9d8baf28767343', + ); + expect(configure.env.ALIYUN_OSS_ACCESS_KEY_ID).toBe( + '${{ secrets.ALIYUN_OSS_ACCESS_KEY_ID }}', + ); + expect(configure.env.ALIYUN_OSS_ACCESS_KEY_SECRET).toBe( + '${{ secrets.ALIYUN_OSS_ACCESS_KEY_SECRET }}', + ); + expect(configure.run).toContain('umask 077'); + expect(configure.run).toContain( + '-L EN -c "${RUNNER_TEMP}/managed-ossutilconfig"', + ); + expect(workflow.jobs.release.env.ALIYUN_OSS_BUCKET).toContain( + 'vars.ALIYUN_OSS_BUCKET', + ); + expect(preflight.run).toContain( + 'oss="oss://${ALIYUN_OSS_BUCKET}/releases/qwen-code/${RELEASE_TAG}/${asset}"', + ); + expect(preflight.run).toContain( + '--check-oss "${oss}" --config "${RUNNER_TEMP}/managed-ossutilconfig"', + ); + expect(upload.run).toContain( + '--bucket "${ALIYUN_OSS_BUCKET}" --config "${RUNNER_TEMP}/managed-ossutilconfig"', + ); + expect(upload.env).toBeUndefined(); + expect(step('Verify both public download channels').run).toContain( + '--check-url "${base}/${asset}"', + ); + expect(step('Verify both public download channels').run).toContain( + 'OSS_PUBLIC_BASE_URL', + ); + }); + it('pins checkout and reproducible timestamps, and verifies the SDK before packaging the server', () => { expect(steps[0].with.ref).toBe('${{ needs.prepare.outputs.sha }}'); expect(steps[0].with['persist-credentials']).toBe(false); + const tools = step('Checkout release tooling from the workflow revision'); + expect(tools.with).toEqual({ + ref: '${{ github.workflow_sha }}', + path: '.qwen/managed-release-tools', + 'sparse-checkout': 'scripts', + 'persist-credentials': false, + }); + expect(tools.uses).toBe(steps[0].uses); + for (const item of steps.filter((item) => + item.run?.includes('managed-runtime-release.js'), + )) { + expect(item.run).not.toContain('node scripts/managed-runtime-release.js'); + expect(item.run).toContain( + 'node .qwen/managed-release-tools/scripts/managed-runtime-release.js', + ); + } + expect( step('Verify and install the SDK from the same source').run, ).toContain('clean install'); @@ -271,6 +547,30 @@ describe.skipIf(process.platform === 'win32')('release gate execution', () => { (item) => item.id === 'publication', ).run; + it('requires both credentials before creating a private shared OSS configuration', () => { + const configure = workflow.jobs.release.steps.find( + (item) => item.name === 'Configure authenticated OSS access', + ).run; + executable( + 'ossutil', + 'touch oss-called; for value do :; done; : > "$value"', + ); + const env = { + ALIYUN_OSS_ACCESS_KEY_ID: 'fixture-id', + ALIYUN_OSS_ACCESS_KEY_SECRET: '', + ALIYUN_OSS_ENDPOINT: 'https://example.invalid', + }; + expect(run(configure, env).status).not.toBe(0); + expect(fs.existsSync(path.join(temporary, 'oss-called'))).toBe(false); + expect( + run(configure, { ...env, ALIYUN_OSS_ACCESS_KEY_SECRET: 'fixture-secret' }) + .status, + ).toBe(0); + expect( + fs.statSync(path.join(temporary, 'managed-ossutilconfig')).mode & 0o777, + ).toBe(0o600); + }); + it('rejects a non-main manual run before consulting the release', () => { executable('gh', 'touch gh-called; exit 1'); expect(run(gate, { WORKFLOW_REF: 'refs/heads/untrusted' }).status).not.toBe( @@ -312,6 +612,7 @@ describe.skipIf(process.platform === 'win32')('release gate execution', () => { SOURCE_SHA: 'original-sha', CLI_VERSION: '0.25.4', OSS_PUBLIC_BASE_URL: 'https://example.com', + ALIYUN_OSS_BUCKET: 'release-bucket', }); expect(result.status).not.toBe(0); expect(result.stderr).toContain('mismatch');