mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-11-07 06:19:43 -10:00
fix(release): authenticate managed OSS preflight (#15504)
This commit is contained in:
5 files changed
+575
-34
No files matched your search
@@ -93,6 +93,7 @@ jobs:
|
||||
CLI_VERSION: '${{ needs.prepare.outputs.version }}'
|
||||
SOURCE_SHA: '${{ needs.prepare.outputs.sha }}'
|
||||
BUILD_TIMESTAMP: '${{ needs.prepare.outputs.timestamp }}'
|
||||
ALIYUN_OSS_BUCKET: "${{ vars.ALIYUN_OSS_BUCKET || 'qwen-code-assets' }}"
|
||||
OSS_PUBLIC_BASE_URL: "${{ vars.ALIYUN_OSS_PUBLIC_BASE_URL || 'https://qwen-code-assets.oss-cn-hangzhou.aliyuncs.com' }}"
|
||||
steps:
|
||||
- name: 'Checkout the pinned source'
|
||||
@@ -101,6 +102,14 @@ jobs:
|
||||
ref: '${{ needs.prepare.outputs.sha }}'
|
||||
persist-credentials: false
|
||||
|
||||
- name: 'Checkout release tooling from the workflow revision'
|
||||
uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3
|
||||
with:
|
||||
ref: '${{ github.workflow_sha }}'
|
||||
path: '.qwen/managed-release-tools'
|
||||
sparse-checkout: 'scripts'
|
||||
persist-credentials: false
|
||||
|
||||
- name: 'Set up Java 21'
|
||||
uses: 'actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654' # v5.2.0
|
||||
with:
|
||||
@@ -137,7 +146,7 @@ jobs:
|
||||
- name: 'Verify the executable JAR and publication profile'
|
||||
run: |-
|
||||
mvn --batch-mode --no-transfer-progress -f packages/managed-agent-server/pom.xml clean verify -Pcentral-release -Dgpg.skip=true -Dproject.build.outputTimestamp="${BUILD_TIMESTAMP}"
|
||||
node scripts/managed-runtime-release.js --jar "packages/managed-agent-server/target/qwen-managed-agent-server-${CLI_VERSION}.jar" --version "${CLI_VERSION}" --out-dir dist/managed-runtime
|
||||
node .qwen/managed-release-tools/scripts/managed-runtime-release.js --jar "packages/managed-agent-server/target/qwen-managed-agent-server-${CLI_VERSION}.jar" --version "${CLI_VERSION}" --out-dir dist/managed-runtime
|
||||
|
||||
- name: 'Retain verified JAR and checksum'
|
||||
uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # v7.0.1
|
||||
@@ -146,6 +155,36 @@ jobs:
|
||||
path: 'dist/managed-runtime/*'
|
||||
if-no-files-found: 'error'
|
||||
|
||||
- name: 'Install the verified OSS uploader'
|
||||
if: "${{ github.event_name == 'release' || !inputs.dry_run }}"
|
||||
env:
|
||||
OSSUTIL_URL: 'https://gosspublic.alicdn.com/ossutil/1.7.19/ossutil-v1.7.19-linux-amd64.zip'
|
||||
OSSUTIL_SHA256: 'dcc512e4a893e16bbee63bc769339d8e56b21744fd83c8212a9d8baf28767343'
|
||||
run: |-
|
||||
set -euo pipefail
|
||||
work=$(mktemp -d)
|
||||
trap 'rm -rf "${work}"' EXIT
|
||||
curl -fsSL --connect-timeout 15 --max-time 300 "${OSSUTIL_URL}" -o "${work}/ossutil.zip"
|
||||
echo "${OSSUTIL_SHA256} ${work}/ossutil.zip" | sha256sum -c -
|
||||
unzip -q "${work}/ossutil.zip" -d "${work}"
|
||||
binary=$(find "${work}" -type f -name 'ossutil64' -print -quit)
|
||||
test -n "${binary}"
|
||||
mkdir -p "${HOME}/.local/bin"
|
||||
install -m 0755 "${binary}" "${HOME}/.local/bin/ossutil"
|
||||
echo "${HOME}/.local/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: 'Configure authenticated OSS access'
|
||||
if: "${{ github.event_name == 'release' || !inputs.dry_run }}"
|
||||
env:
|
||||
ALIYUN_OSS_ACCESS_KEY_ID: '${{ secrets.ALIYUN_OSS_ACCESS_KEY_ID }}'
|
||||
ALIYUN_OSS_ACCESS_KEY_SECRET: '${{ secrets.ALIYUN_OSS_ACCESS_KEY_SECRET }}'
|
||||
ALIYUN_OSS_ENDPOINT: "${{ vars.ALIYUN_OSS_ENDPOINT || 'https://oss-cn-hangzhou.aliyuncs.com' }}"
|
||||
run: |-
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
[[ -n "${ALIYUN_OSS_ACCESS_KEY_ID}" && -n "${ALIYUN_OSS_ACCESS_KEY_SECRET}" ]]
|
||||
ossutil config -e "${ALIYUN_OSS_ENDPOINT}" -i "${ALIYUN_OSS_ACCESS_KEY_ID}" -k "${ALIYUN_OSS_ACCESS_KEY_SECRET}" -L EN -c "${RUNNER_TEMP}/managed-ossutilconfig"
|
||||
|
||||
- name: 'Recheck the immutable tag and existing publication bytes'
|
||||
id: 'publication'
|
||||
if: "${{ github.event_name == 'release' || !inputs.dry_run }}"
|
||||
@@ -162,18 +201,18 @@ jobs:
|
||||
gh api "repos/QwenLM/qwen-code/releases/tags/${RELEASE_TAG}" | jq --exit-status '.draft == false'
|
||||
jar="qwen-managed-agent-server-${CLI_VERSION}.jar"
|
||||
central="https://repo.maven.apache.org/maven2/com/alibaba/qwen-managed-agent-server/${CLI_VERSION}/${jar}"
|
||||
state=$(node scripts/managed-runtime-release.js --check-url "${central}" --file "dist/managed-runtime/${jar}") || exit 1
|
||||
state=$(node .qwen/managed-release-tools/scripts/managed-runtime-release.js --check-url "${central}" --file "dist/managed-runtime/${jar}") || exit 1
|
||||
echo "central=${state}" >> "${GITHUB_OUTPUT}"
|
||||
: > "${RUNNER_TEMP}/managed-github-missing"
|
||||
: > "${RUNNER_TEMP}/managed-oss-missing"
|
||||
for asset in "${jar}" "${jar}.sha256"; do
|
||||
github="https://github.com/QwenLM/qwen-code/releases/download/${RELEASE_TAG}/${asset}"
|
||||
oss="${OSS_PUBLIC_BASE_URL}/releases/qwen-code/${RELEASE_TAG}/${asset}"
|
||||
state=$(node scripts/managed-runtime-release.js --check-url "${github}" --file "dist/managed-runtime/${asset}") || exit 1
|
||||
oss="oss://${ALIYUN_OSS_BUCKET}/releases/qwen-code/${RELEASE_TAG}/${asset}"
|
||||
state=$(node .qwen/managed-release-tools/scripts/managed-runtime-release.js --check-url "${github}" --file "dist/managed-runtime/${asset}") || exit 1
|
||||
if [[ "${state}" == 'missing' ]]; then
|
||||
echo "dist/managed-runtime/${asset}" >> "${RUNNER_TEMP}/managed-github-missing"
|
||||
fi
|
||||
state=$(node scripts/managed-runtime-release.js --check-url "${oss}" --file "dist/managed-runtime/${asset}") || exit 1
|
||||
state=$(node .qwen/managed-release-tools/scripts/managed-runtime-release.js --check-oss "${oss}" --config "${RUNNER_TEMP}/managed-ossutilconfig" --file "dist/managed-runtime/${asset}") || exit 1
|
||||
if [[ "${state}" == 'missing' ]]; then
|
||||
echo "dist/managed-runtime/${asset}" >> "${RUNNER_TEMP}/managed-oss-missing"
|
||||
fi
|
||||
@@ -212,7 +251,7 @@ jobs:
|
||||
set -euo pipefail
|
||||
jar="qwen-managed-agent-server-${CLI_VERSION}.jar"
|
||||
for attempt in {1..40}; do
|
||||
state=$(node scripts/managed-runtime-release.js --check-url "https://repo.maven.apache.org/maven2/com/alibaba/qwen-managed-agent-server/${CLI_VERSION}/${jar}" --file "dist/managed-runtime/${jar}")
|
||||
state=$(node .qwen/managed-release-tools/scripts/managed-runtime-release.js --check-url "https://repo.maven.apache.org/maven2/com/alibaba/qwen-managed-agent-server/${CLI_VERSION}/${jar}" --file "dist/managed-runtime/${jar}")
|
||||
if [[ "${state}" == 'present' ]]; then exit 0; fi
|
||||
sleep 30
|
||||
done
|
||||
@@ -228,37 +267,12 @@ jobs:
|
||||
mapfile -t assets < "${RUNNER_TEMP}/managed-github-missing"
|
||||
if (( ${#assets[@]} )); then gh release upload "${RELEASE_TAG}" "${assets[@]}"; fi
|
||||
|
||||
- name: 'Install the verified OSS uploader'
|
||||
if: "${{ github.event_name == 'release' || !inputs.dry_run }}"
|
||||
env:
|
||||
OSSUTIL_URL: 'https://gosspublic.alicdn.com/ossutil/1.7.19/ossutil-v1.7.19-linux-amd64.zip'
|
||||
OSSUTIL_SHA256: 'dcc512e4a893e16bbee63bc769339d8e56b21744fd83c8212a9d8baf28767343'
|
||||
run: |-
|
||||
set -euo pipefail
|
||||
work=$(mktemp -d)
|
||||
trap 'rm -rf "${work}"' EXIT
|
||||
curl -fsSL --connect-timeout 15 --max-time 300 "${OSSUTIL_URL}" -o "${work}/ossutil.zip"
|
||||
echo "${OSSUTIL_SHA256} ${work}/ossutil.zip" | sha256sum -c -
|
||||
unzip -q "${work}/ossutil.zip" -d "${work}"
|
||||
binary=$(find "${work}" -type f -name 'ossutil64' -print -quit)
|
||||
test -n "${binary}"
|
||||
mkdir -p "${HOME}/.local/bin"
|
||||
install -m 0755 "${binary}" "${HOME}/.local/bin/ossutil"
|
||||
echo "${HOME}/.local/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: 'Upload only missing OSS release assets'
|
||||
if: "${{ github.event_name == 'release' || !inputs.dry_run }}"
|
||||
env:
|
||||
ALIYUN_OSS_ACCESS_KEY_ID: '${{ secrets.ALIYUN_OSS_ACCESS_KEY_ID }}'
|
||||
ALIYUN_OSS_ACCESS_KEY_SECRET: '${{ secrets.ALIYUN_OSS_ACCESS_KEY_SECRET }}'
|
||||
ALIYUN_OSS_ENDPOINT: "${{ vars.ALIYUN_OSS_ENDPOINT || 'https://oss-cn-hangzhou.aliyuncs.com' }}"
|
||||
ALIYUN_OSS_BUCKET: "${{ vars.ALIYUN_OSS_BUCKET || 'qwen-code-assets' }}"
|
||||
run: |-
|
||||
set -euo pipefail
|
||||
mapfile -t assets < "${RUNNER_TEMP}/managed-oss-missing"
|
||||
if (( ${#assets[@]} == 0 )); then exit 0; fi
|
||||
[[ -n "${ALIYUN_OSS_ACCESS_KEY_ID}" && -n "${ALIYUN_OSS_ACCESS_KEY_SECRET}" ]]
|
||||
ossutil config -e "${ALIYUN_OSS_ENDPOINT}" -i "${ALIYUN_OSS_ACCESS_KEY_ID}" -k "${ALIYUN_OSS_ACCESS_KEY_SECRET}" -L EN -c "${RUNNER_TEMP}/managed-ossutilconfig"
|
||||
node scripts/upload-aliyun-oss-assets.js --bucket "${ALIYUN_OSS_BUCKET}" --config "${RUNNER_TEMP}/managed-ossutilconfig" --prefix "releases/qwen-code/${RELEASE_TAG}" "${assets[@]}"
|
||||
|
||||
- name: 'Verify both public download channels'
|
||||
@@ -268,6 +282,6 @@ jobs:
|
||||
jar="qwen-managed-agent-server-${CLI_VERSION}.jar"
|
||||
for base in "https://github.com/QwenLM/qwen-code/releases/download/${RELEASE_TAG}" "${OSS_PUBLIC_BASE_URL}/releases/qwen-code/${RELEASE_TAG}"; do
|
||||
for asset in "${jar}" "${jar}.sha256"; do
|
||||
[[ "$(node scripts/managed-runtime-release.js --check-url "${base}/${asset}" --file "dist/managed-runtime/${asset}")" == 'present' ]]
|
||||
[[ "$(node .qwen/managed-release-tools/scripts/managed-runtime-release.js --check-url "${base}/${asset}" --file "dist/managed-runtime/${asset}")" == 'present' ]]
|
||||
done
|
||||
done
|
||||
@@ -0,0 +1,66 @@
|
||||
# Authenticated OSS preflight for Managed runtime releases
|
||||
|
||||
[English](managed-runtime-oss-preflight.md) | [简体中文](managed-runtime-oss-preflight.zh-CN.md)
|
||||
|
||||
## Problem and scope
|
||||
|
||||
Anonymous OSS GET can return `403 AccessDenied` for an absent object under the
|
||||
bucket ACL. Treating that response as missing would also permit overwriting an
|
||||
inaccessible existing release. Only the Managed runtime release preflight changes;
|
||||
Java artifacts, product code, public verification, and publication immutability
|
||||
checks remain unchanged.
|
||||
|
||||
## Decision
|
||||
|
||||
Keep the artifact checkout pinned to the release's `SOURCE_SHA`. Separately check
|
||||
out `scripts` from `github.workflow_sha` into the already ignored
|
||||
`.qwen/managed-release-tools` directory, without persisted credentials. Run the
|
||||
release helper there so backfills of old tags can use the new preflight. Its
|
||||
imports resolve through the root installation; the source checkout, Git metadata,
|
||||
version, and build timestamp remain unchanged.
|
||||
|
||||
Install the existing SHA-256-pinned ossutil 1.7.19 and configure its existing
|
||||
production-release credentials before preflight. Use the same bucket, object key,
|
||||
and configuration for preflight and upload. Create the configuration with
|
||||
`umask 077`; dry runs skip both setup and authenticated reads.
|
||||
|
||||
Add `--check-oss OSS_URL --config PATH --file PATH` to the release helper. Accept
|
||||
only an `oss://bucket/object` URL without credentials, port, query, or fragment.
|
||||
Run `ossutil cat` without a shell, writing stdout into a private temporary file,
|
||||
with a 300-second timeout and bounded stderr. This command cannot write remotely.
|
||||
|
||||
The pinned [cat implementation](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/cat.go)
|
||||
calls `GetObject`, copies raw bytes to stdout, and appends one newline. The
|
||||
[command wrapper](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/command_manager.go#L57)
|
||||
then prints `\n%.6f(s) elapsed\n`. Read at most 64 trailer bytes after the expected
|
||||
artifact size and require precisely `\n\n<seconds>.<6 digits>(s) elapsed\n`.
|
||||
Remove that trailer and compare SHA-256 using the existing helper. This also detects truncated output even though
|
||||
this version of cat ignores the `io.Copy` error. Original terminal newlines are
|
||||
preserved. Always remove the temporary download.
|
||||
|
||||
`cp` is unsuitable here: its [download precheck](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/cp.go#L2489)
|
||||
uses [GetObjectDetailedMeta](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/command.go#L658),
|
||||
a HEAD request, whose empty error body cannot reliably distinguish missing keys.
|
||||
The CLI [prints errors on stdout and exits 1](https://github.com/aliyun/ossutil/blob/v1.7.19/ossutil.go).
|
||||
Only exit 1 with the complete canonical line beginning
|
||||
`Error: oss: service returned error: StatusCode=404, ErrorCode=NoSuchKey, ErrorMessage=`
|
||||
is missing. The format comes from its pinned
|
||||
[OSS SDK v3.0.2](https://github.com/aliyun/aliyun-oss-go-sdk/blob/v3.0.2/oss/error.go).
|
||||
Read at most 16 KiB of error output. AccessDenied, NoSuchBucket, unstructured 404,
|
||||
network errors, subprocess errors, timeouts, and mismatched bytes fail closed.
|
||||
Do not echo subprocess diagnostics or credentials.
|
||||
|
||||
## Verification and limits
|
||||
|
||||
Real subprocess shims cover identical binary bytes, terminal newlines, exact
|
||||
NoSuchKey/404, other error classes, mismatches, timeout, spawn failure, argument
|
||||
validation, and download cleanup. Workflow tests enforce setup order, pinned
|
||||
uploader, credential/configuration reuse, dry-run guards, and final public HTTPS
|
||||
verification. A local old-tag fixture checks that the workflow-selected helper runs
|
||||
with root dependencies while the old helper remains unchanged. The new behavioral tests fail against the original script because
|
||||
`--check-oss` is unsupported, then pass with the fix.
|
||||
|
||||
No real credentials, authenticated remote requests, workflow dispatch, or
|
||||
publication are part of local validation. Actual OSS access still depends on the
|
||||
existing credential permissions; insufficient access deliberately fails the
|
||||
release. Upgrading ossutil requires rechecking its stdout and error contracts.
|
||||
@@ -0,0 +1,56 @@
|
||||
# Managed runtime 发布的 OSS 认证预检
|
||||
|
||||
[English](managed-runtime-oss-preflight.md) | [简体中文](managed-runtime-oss-preflight.zh-CN.md)
|
||||
|
||||
## 问题与范围
|
||||
|
||||
受 bucket ACL 影响,匿名 OSS GET 对不存在的对象也可能返回 `403 AccessDenied`。
|
||||
把此响应视为缺失,会允许覆盖无权访问的现有发布。本次只修改 Managed runtime
|
||||
发布预检;Java 产物、产品代码、公开访问验证及发布不可变性检查保持不变。
|
||||
|
||||
## 方案
|
||||
|
||||
产物源码继续检出发布的 `SOURCE_SHA`。另将 `github.workflow_sha` 的 `scripts` 检出到
|
||||
已有忽略规则覆盖的 `.qwen/managed-release-tools` 目录,不持久化凭据。使用此处的
|
||||
发布辅助脚本,使旧标签补发也能运行新增预检。依赖通过根目录安装解析;源码检出、
|
||||
Git 元数据、版本和构建时间戳保持不变。
|
||||
|
||||
在预检前安装现有 SHA-256 固定的 ossutil 1.7.19,并配置现有 production-release
|
||||
凭据。预检与上传使用相同 bucket、对象路径和配置。以 `umask 077` 创建配置文件;
|
||||
dry run 跳过配置与认证读取。
|
||||
|
||||
发布脚本增加 `--check-oss OSS_URL --config PATH --file PATH`。仅接受不含凭据、
|
||||
端口、查询参数或片段的 `oss://bucket/object` URL。直接启动 `ossutil cat`,不经过
|
||||
shell,将 stdout 写入私有临时文件,设置 300 秒超时并限制 stderr 大小。
|
||||
此命令不能写入远端。
|
||||
|
||||
固定版本的 [cat 实现](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/cat.go)
|
||||
调用 `GetObject`,把原始字节复制到 stdout,再追加一个换行符。
|
||||
[命令包装层](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/command_manager.go#L57)
|
||||
随后输出 `\n%.6f(s) elapsed\n`。从预期产物大小处最多读取 64 字节,要求尾部精确符合
|
||||
`\n\n<秒数>.<6 位小数>(s) elapsed\n`,移除此尾部后用现有辅助函数比较 SHA-256。即使此版本 cat
|
||||
忽略 `io.Copy` 的错误,这也能检测输出截断。保留产物自身的末尾换行符,并始终清理
|
||||
临时下载文件。
|
||||
|
||||
`cp` 不适合此场景:其[下载预检](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/cp.go#L2489)
|
||||
使用 [GetObjectDetailedMeta](https://github.com/aliyun/ossutil/blob/v1.7.19/lib/command.go#L658),
|
||||
这是 HEAD 请求,空错误响应体不能可靠区分缺失对象。
|
||||
CLI [把错误打印到 stdout 并以状态 1 退出](https://github.com/aliyun/ossutil/blob/v1.7.19/ossutil.go)。
|
||||
只有退出状态为 1,且输出为完整规范错误行、开头是
|
||||
`Error: oss: service returned error: StatusCode=404, ErrorCode=NoSuchKey, ErrorMessage=`,
|
||||
才认定对象缺失。此格式来自固定依赖
|
||||
[OSS SDK v3.0.2](https://github.com/aliyun/aliyun-oss-go-sdk/blob/v3.0.2/oss/error.go)。
|
||||
错误输出最多读取 16 KiB。AccessDenied、NoSuchBucket、非结构化 404、网络错误、
|
||||
子进程错误、超时和字节不一致均阻止发布。不回显子进程诊断或凭据。
|
||||
|
||||
## 验证与限制
|
||||
|
||||
真实子进程替身覆盖相同二进制字节、末尾换行符、精确 NoSuchKey/404、其他错误类别、
|
||||
字节不一致、超时、启动失败、参数校验和下载清理。工作流测试约束配置顺序、固定
|
||||
上传器、凭据与配置复用、dry-run 条件及最终公开 HTTPS 验证。本地旧标签夹具验证
|
||||
工作流选择的辅助脚本能解析根目录依赖,同时旧脚本保持不变。新增行为测试在原始
|
||||
脚本上因不支持 `--check-oss` 而失败,修复后通过。
|
||||
|
||||
本地验证不访问真实凭据,不执行认证远端请求、工作流触发或发布。真实 OSS 访问仍
|
||||
依赖现有凭据权限;权限不足时应阻止发布。升级 ossutil 时须重新核对其 stdout 和
|
||||
错误格式契约。
|
||||
@@ -5,6 +5,8 @@
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import os from 'node:os';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { Readable, Transform } from 'node:stream';
|
||||
@@ -82,6 +84,90 @@ export async function checkManagedRuntimeAsset(url, file, fetchImpl = fetch) {
|
||||
return 'present';
|
||||
}
|
||||
|
||||
export async function checkManagedRuntimeOssAsset(
|
||||
url,
|
||||
file,
|
||||
config,
|
||||
timeout = 300_000,
|
||||
) {
|
||||
const parsed = new URL(url);
|
||||
if (
|
||||
parsed.protocol !== 'oss:' ||
|
||||
!parsed.hostname ||
|
||||
parsed.username ||
|
||||
parsed.password ||
|
||||
parsed.port ||
|
||||
parsed.pathname.length <= 1 ||
|
||||
parsed.search ||
|
||||
parsed.hash
|
||||
) {
|
||||
throw new Error(
|
||||
'Release artifact verification requires an OSS object URL.',
|
||||
);
|
||||
}
|
||||
const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'managed-oss-'));
|
||||
const download = path.join(temporary, 'artifact');
|
||||
try {
|
||||
const output = fs.openSync(download, 'w');
|
||||
let result;
|
||||
try {
|
||||
// Unlike cp's HEAD precheck, cat uses GET and preserves OSS error codes.
|
||||
result = spawnSync('ossutil', ['cat', url, '-c', path.resolve(config)], {
|
||||
stdio: ['ignore', output, 'pipe'],
|
||||
timeout,
|
||||
killSignal: 'SIGKILL',
|
||||
maxBuffer: 64 * 1024,
|
||||
});
|
||||
} finally {
|
||||
fs.closeSync(output);
|
||||
}
|
||||
if (result.error || result.status !== 0) {
|
||||
if (
|
||||
!result.error &&
|
||||
result.status === 1 &&
|
||||
fs.statSync(download).size <= 16 * 1024 &&
|
||||
/^Error: oss: service returned error: StatusCode=404, ErrorCode=NoSuchKey, ErrorMessage="[^\r\n]*", RequestId=[^\r\n]+\r?\n?$/.test(
|
||||
fs.readFileSync(download, 'utf8'),
|
||||
)
|
||||
) {
|
||||
return 'missing';
|
||||
}
|
||||
throw new Error(
|
||||
'Cannot verify existing OSS artifact; refusing publication.',
|
||||
);
|
||||
}
|
||||
const size = fs.statSync(file).size;
|
||||
// cat appends a newline, then the 1.7.19 command wrapper prints elapsed time.
|
||||
const trailer = Buffer.alloc(64);
|
||||
const input = fs.openSync(download, 'r');
|
||||
let length;
|
||||
try {
|
||||
length = fs.readSync(input, trailer, 0, trailer.length, size);
|
||||
} finally {
|
||||
fs.closeSync(input);
|
||||
}
|
||||
if (
|
||||
fs.statSync(download).size !== size + length ||
|
||||
!/^\n\n\d+\.\d{6}\(s\) elapsed\n$/.test(
|
||||
trailer.toString('utf8', 0, length),
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
'Existing release artifact has different bytes; refusing overwrite.',
|
||||
);
|
||||
}
|
||||
fs.truncateSync(download, size);
|
||||
if ((await sha256File(download)) !== (await sha256File(file))) {
|
||||
throw new Error(
|
||||
'Existing release artifact has different bytes; refusing overwrite.',
|
||||
);
|
||||
}
|
||||
return 'present';
|
||||
} finally {
|
||||
fs.rmSync(temporary, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
if (isMainModule(import.meta.url)) {
|
||||
try {
|
||||
const args = parseArgs(process.argv.slice(2), {
|
||||
@@ -89,25 +175,43 @@ if (isMainModule(import.meta.url)) {
|
||||
'--version': { key: 'version', type: 'value' },
|
||||
'--out-dir': { key: 'outDir', type: 'value' },
|
||||
'--check-url': { key: 'url', type: 'value' },
|
||||
'--check-oss': { key: 'oss', type: 'value' },
|
||||
'--config': { key: 'config', type: 'value' },
|
||||
'--file': { key: 'file', type: 'value' },
|
||||
});
|
||||
if (args.help) {
|
||||
console.log(
|
||||
'Stage: --jar PATH --version VERSION --out-dir DIR\nVerify without replacing: --check-url HTTPS_URL --file PATH (prints present or missing).',
|
||||
'Stage: --jar PATH --version VERSION --out-dir DIR\nVerify without replacing: --check-url HTTPS_URL --file PATH (prints present or missing).\nAuthenticated OSS: --check-oss OSS_URL --config PATH --file PATH.',
|
||||
);
|
||||
} else if (
|
||||
args.url &&
|
||||
!args.oss &&
|
||||
!args.config &&
|
||||
args.file &&
|
||||
!args.jar &&
|
||||
!args.version &&
|
||||
!args.outDir
|
||||
) {
|
||||
console.log(await checkManagedRuntimeAsset(args.url, args.file));
|
||||
} else if (
|
||||
args.oss &&
|
||||
args.config &&
|
||||
args.file &&
|
||||
!args.url &&
|
||||
!args.jar &&
|
||||
!args.version &&
|
||||
!args.outDir
|
||||
) {
|
||||
console.log(
|
||||
await checkManagedRuntimeOssAsset(args.oss, args.file, args.config),
|
||||
);
|
||||
} else if (
|
||||
args.jar &&
|
||||
args.version &&
|
||||
args.outDir &&
|
||||
!args.url &&
|
||||
!args.oss &&
|
||||
!args.config &&
|
||||
!args.file
|
||||
) {
|
||||
console.log(JSON.stringify(await stageManagedRuntimeRelease(args)));
|
||||
|
||||
@@ -14,6 +14,7 @@ import { parse } from 'yaml';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
checkManagedRuntimeAsset,
|
||||
checkManagedRuntimeOssAsset,
|
||||
stageManagedRuntimeRelease,
|
||||
} from '../managed-runtime-release.js';
|
||||
|
||||
@@ -38,7 +39,10 @@ beforeEach(() => {
|
||||
jar = path.join(temporary, 'server.jar');
|
||||
fs.writeFileSync(jar, bytes);
|
||||
});
|
||||
afterEach(() => fs.rmSync(temporary, { recursive: true, force: true }));
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
fs.rmSync(temporary, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('stages the exact server bytes and the checksum consumed by the first-use installer', async () => {
|
||||
const outDir = path.join(temporary, 'release');
|
||||
@@ -150,6 +154,213 @@ it.each([
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe.skipIf(process.platform === 'win32')(
|
||||
'authenticated OSS verification',
|
||||
() => {
|
||||
function check({
|
||||
content = bytes,
|
||||
error = '',
|
||||
status = 0,
|
||||
url = 'oss://release-bucket/releases/server.jar',
|
||||
extra = [],
|
||||
script = path.join(root, 'scripts/managed-runtime-release.js'),
|
||||
} = {}) {
|
||||
const bin = path.join(temporary, 'bin');
|
||||
fs.mkdirSync(bin, { recursive: true });
|
||||
fs.writeFileSync(path.join(bin, 'package.json'), '{"type":"commonjs"}');
|
||||
fs.writeFileSync(
|
||||
path.join(bin, 'ossutil'),
|
||||
`#!${process.execPath}
|
||||
const fs = require('node:fs');
|
||||
fs.writeFileSync(process.env.OSS_ARGS, JSON.stringify(process.argv.slice(2)));
|
||||
process.stdout.write(Buffer.from(${JSON.stringify(content.toString('base64'))}, 'base64'));
|
||||
process.stdout.write(${JSON.stringify(error || '\n\n0.001234(s) elapsed\n')});
|
||||
process.exit(${status});
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
fs.realpathSync(script),
|
||||
'--check-oss',
|
||||
url,
|
||||
'--config',
|
||||
path.join(temporary, 'config'),
|
||||
'--file',
|
||||
jar,
|
||||
...extra,
|
||||
],
|
||||
{
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${bin}${path.delimiter}${process.env.PATH}`,
|
||||
OSS_ARGS: path.join(temporary, 'args'),
|
||||
TMPDIR: temporary,
|
||||
},
|
||||
encoding: 'utf8',
|
||||
},
|
||||
);
|
||||
expect(
|
||||
fs
|
||||
.readdirSync(temporary)
|
||||
.filter((name) => name.startsWith('managed-oss-')),
|
||||
).toEqual([]);
|
||||
return result;
|
||||
}
|
||||
|
||||
it('downloads authenticated bytes using only the configured read command', () => {
|
||||
const result = check();
|
||||
expect(result.stderr).toBe('');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toBe('present\n');
|
||||
expect(
|
||||
JSON.parse(fs.readFileSync(path.join(temporary, 'args'), 'utf8')),
|
||||
).toEqual([
|
||||
'cat',
|
||||
'oss://release-bucket/releases/server.jar',
|
||||
'-c',
|
||||
path.join(temporary, 'config'),
|
||||
]);
|
||||
expect(fs.readFileSync(jar)).toEqual(bytes);
|
||||
});
|
||||
|
||||
it('runs current workflow tooling alongside an unchanged old-tag source tree', () => {
|
||||
const command = workflow.jobs.release.steps
|
||||
.find((item) => item.id === 'publication')
|
||||
.run.match(/node (\S+) --check-oss/)[1];
|
||||
const script = path.join(temporary, command);
|
||||
fs.mkdirSync(path.dirname(script), { recursive: true });
|
||||
for (const name of [
|
||||
'managed-runtime-release.js',
|
||||
'release-script-utils.js',
|
||||
]) {
|
||||
fs.copyFileSync(
|
||||
path.join(root, 'scripts', name),
|
||||
path.join(path.dirname(script), name),
|
||||
);
|
||||
}
|
||||
fs.writeFileSync(
|
||||
path.join(temporary, 'package.json'),
|
||||
'{"type":"module"}',
|
||||
);
|
||||
fs.symlinkSync(
|
||||
path.join(root, 'node_modules'),
|
||||
path.join(temporary, 'node_modules'),
|
||||
'dir',
|
||||
);
|
||||
const oldSource = path.join(
|
||||
temporary,
|
||||
'scripts/managed-runtime-release.js',
|
||||
);
|
||||
fs.mkdirSync(path.dirname(oldSource), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
oldSource,
|
||||
'throw new Error("old tag has no --check-oss");',
|
||||
);
|
||||
const result = check({ script });
|
||||
expect(result.stderr).toBe('');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toBe('present\n');
|
||||
expect(fs.readFileSync(oldSource, 'utf8')).toBe(
|
||||
'throw new Error("old tag has no --check-oss");',
|
||||
);
|
||||
});
|
||||
|
||||
it('preserves an artifact terminal newline when removing the CLI trailer', () => {
|
||||
const content = Buffer.concat([bytes, Buffer.from('\n')]);
|
||||
fs.writeFileSync(jar, content);
|
||||
expect(check({ content }).stdout).toBe('present\n');
|
||||
});
|
||||
|
||||
it('refuses unexpected successful CLI output instead of discarding arbitrary bytes', () => {
|
||||
const result = check({ error: '\n' });
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('refusing overwrite');
|
||||
});
|
||||
|
||||
it('recognizes only the canonical structured NoSuchKey 404 as missing', () => {
|
||||
const result = check({
|
||||
content: Buffer.alloc(0),
|
||||
status: 1,
|
||||
error:
|
||||
'Error: oss: service returned error: StatusCode=404, ErrorCode=NoSuchKey, ErrorMessage="The specified key does not exist.", RequestId=request-id\n',
|
||||
});
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toBe('missing\n');
|
||||
});
|
||||
|
||||
it.each([
|
||||
'Error: oss: service returned error: StatusCode=403, ErrorCode=AccessDenied, ErrorMessage="Access denied", RequestId=id\n',
|
||||
'Error: oss: service returned error: StatusCode=404, ErrorCode=NoSuchBucket, ErrorMessage="Missing bucket", RequestId=id\n',
|
||||
'Error: oss: service returned error: StatusCode=403, ErrorCode=NoSuchKey, ErrorMessage="Denied", RequestId=id\n',
|
||||
'Error: oss: service returned error: StatusCode=404, ErrorCode=AccessDenied, ErrorMessage="NoSuchKey", RequestId=id\n',
|
||||
'Error: HTTP 404 NoSuchKey\n',
|
||||
'Error: network unavailable\n',
|
||||
])('fails closed for inaccessible or unconfirmed absence: %s', (error) => {
|
||||
const result = check({ content: Buffer.alloc(0), status: 1, error });
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stdout).toBe('');
|
||||
expect(result.stderr).toContain('Cannot verify existing OSS artifact');
|
||||
});
|
||||
|
||||
it.each([
|
||||
Buffer.alloc(bytes.length, 1),
|
||||
bytes.subarray(0, 4),
|
||||
Buffer.concat([bytes, Buffer.from('extra')]),
|
||||
])('refuses different bytes and cleans its download', (content) => {
|
||||
const result = check({ content });
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('refusing overwrite');
|
||||
});
|
||||
|
||||
it.each([
|
||||
'https://example.com/object',
|
||||
'oss://user:password@bucket/object',
|
||||
'oss://bucket/',
|
||||
'oss://bucket/key?versionId=old',
|
||||
])('rejects invalid OSS sources before invoking ossutil: %s', (url) => {
|
||||
expect(check({ url }).status).toBe(1);
|
||||
expect(fs.existsSync(path.join(temporary, 'args'))).toBe(false);
|
||||
});
|
||||
|
||||
it('kills a stalled reader and removes its partial download', async () => {
|
||||
fs.writeFileSync(
|
||||
path.join(temporary, 'ossutil'),
|
||||
`#!${process.execPath}
|
||||
setInterval(() => {}, 1000);
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
vi.stubEnv('PATH', temporary);
|
||||
vi.stubEnv('TMPDIR', temporary);
|
||||
await expect(
|
||||
checkManagedRuntimeOssAsset('oss://bucket/key', jar, 'config', 100),
|
||||
).rejects.toThrow('Cannot verify existing OSS artifact');
|
||||
expect(
|
||||
fs
|
||||
.readdirSync(temporary)
|
||||
.filter((name) => name.startsWith('managed-oss-')),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it('fails closed when ossutil cannot start', async () => {
|
||||
vi.stubEnv('PATH', temporary);
|
||||
await expect(
|
||||
checkManagedRuntimeOssAsset('oss://bucket/key', jar, 'config'),
|
||||
).rejects.toThrow('Cannot verify existing OSS artifact');
|
||||
});
|
||||
|
||||
it('rejects mixed public and authenticated verification arguments', () => {
|
||||
expect(
|
||||
check({ extra: ['--check-url', 'https://example.com/server.jar'] })
|
||||
.status,
|
||||
).toBe(1);
|
||||
expect(fs.existsSync(path.join(temporary, 'args'))).toBe(false);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe('Managed runtime publication gates', () => {
|
||||
const steps = workflow.jobs.release.steps;
|
||||
const step = (name) => steps.find((item) => item.name === name);
|
||||
@@ -170,6 +381,7 @@ describe('Managed runtime publication gates', () => {
|
||||
'Publish the signed server artifact to Central',
|
||||
'Attach only missing GitHub release assets',
|
||||
'Install the verified OSS uploader',
|
||||
'Configure authenticated OSS access',
|
||||
'Upload only missing OSS release assets',
|
||||
]) {
|
||||
expect(step(name).if).toContain(
|
||||
@@ -190,9 +402,73 @@ describe('Managed runtime publication gates', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('configures the pinned OSS reader before preflight and reuses the upload destination and credentials', () => {
|
||||
const install = step('Install the verified OSS uploader');
|
||||
const configure = step('Configure authenticated OSS access');
|
||||
const preflight = step(
|
||||
'Recheck the immutable tag and existing publication bytes',
|
||||
);
|
||||
const upload = step('Upload only missing OSS release assets');
|
||||
expect(steps.indexOf(install)).toBeLessThan(steps.indexOf(configure));
|
||||
expect(steps.indexOf(configure)).toBeLessThan(steps.indexOf(preflight));
|
||||
expect(steps.indexOf(preflight)).toBeLessThan(
|
||||
steps.indexOf(step('Publish the signed server artifact to Central')),
|
||||
);
|
||||
expect(install.env.OSSUTIL_URL).toContain('/1.7.19/');
|
||||
expect(install.env.OSSUTIL_SHA256).toBe(
|
||||
'dcc512e4a893e16bbee63bc769339d8e56b21744fd83c8212a9d8baf28767343',
|
||||
);
|
||||
expect(configure.env.ALIYUN_OSS_ACCESS_KEY_ID).toBe(
|
||||
'${{ secrets.ALIYUN_OSS_ACCESS_KEY_ID }}',
|
||||
);
|
||||
expect(configure.env.ALIYUN_OSS_ACCESS_KEY_SECRET).toBe(
|
||||
'${{ secrets.ALIYUN_OSS_ACCESS_KEY_SECRET }}',
|
||||
);
|
||||
expect(configure.run).toContain('umask 077');
|
||||
expect(configure.run).toContain(
|
||||
'-L EN -c "${RUNNER_TEMP}/managed-ossutilconfig"',
|
||||
);
|
||||
expect(workflow.jobs.release.env.ALIYUN_OSS_BUCKET).toContain(
|
||||
'vars.ALIYUN_OSS_BUCKET',
|
||||
);
|
||||
expect(preflight.run).toContain(
|
||||
'oss="oss://${ALIYUN_OSS_BUCKET}/releases/qwen-code/${RELEASE_TAG}/${asset}"',
|
||||
);
|
||||
expect(preflight.run).toContain(
|
||||
'--check-oss "${oss}" --config "${RUNNER_TEMP}/managed-ossutilconfig"',
|
||||
);
|
||||
expect(upload.run).toContain(
|
||||
'--bucket "${ALIYUN_OSS_BUCKET}" --config "${RUNNER_TEMP}/managed-ossutilconfig"',
|
||||
);
|
||||
expect(upload.env).toBeUndefined();
|
||||
expect(step('Verify both public download channels').run).toContain(
|
||||
'--check-url "${base}/${asset}"',
|
||||
);
|
||||
expect(step('Verify both public download channels').run).toContain(
|
||||
'OSS_PUBLIC_BASE_URL',
|
||||
);
|
||||
});
|
||||
|
||||
it('pins checkout and reproducible timestamps, and verifies the SDK before packaging the server', () => {
|
||||
expect(steps[0].with.ref).toBe('${{ needs.prepare.outputs.sha }}');
|
||||
expect(steps[0].with['persist-credentials']).toBe(false);
|
||||
const tools = step('Checkout release tooling from the workflow revision');
|
||||
expect(tools.with).toEqual({
|
||||
ref: '${{ github.workflow_sha }}',
|
||||
path: '.qwen/managed-release-tools',
|
||||
'sparse-checkout': 'scripts',
|
||||
'persist-credentials': false,
|
||||
});
|
||||
expect(tools.uses).toBe(steps[0].uses);
|
||||
for (const item of steps.filter((item) =>
|
||||
item.run?.includes('managed-runtime-release.js'),
|
||||
)) {
|
||||
expect(item.run).not.toContain('node scripts/managed-runtime-release.js');
|
||||
expect(item.run).toContain(
|
||||
'node .qwen/managed-release-tools/scripts/managed-runtime-release.js',
|
||||
);
|
||||
}
|
||||
|
||||
expect(
|
||||
step('Verify and install the SDK from the same source').run,
|
||||
).toContain('clean install');
|
||||
@@ -271,6 +547,30 @@ describe.skipIf(process.platform === 'win32')('release gate execution', () => {
|
||||
(item) => item.id === 'publication',
|
||||
).run;
|
||||
|
||||
it('requires both credentials before creating a private shared OSS configuration', () => {
|
||||
const configure = workflow.jobs.release.steps.find(
|
||||
(item) => item.name === 'Configure authenticated OSS access',
|
||||
).run;
|
||||
executable(
|
||||
'ossutil',
|
||||
'touch oss-called; for value do :; done; : > "$value"',
|
||||
);
|
||||
const env = {
|
||||
ALIYUN_OSS_ACCESS_KEY_ID: 'fixture-id',
|
||||
ALIYUN_OSS_ACCESS_KEY_SECRET: '',
|
||||
ALIYUN_OSS_ENDPOINT: 'https://example.invalid',
|
||||
};
|
||||
expect(run(configure, env).status).not.toBe(0);
|
||||
expect(fs.existsSync(path.join(temporary, 'oss-called'))).toBe(false);
|
||||
expect(
|
||||
run(configure, { ...env, ALIYUN_OSS_ACCESS_KEY_SECRET: 'fixture-secret' })
|
||||
.status,
|
||||
).toBe(0);
|
||||
expect(
|
||||
fs.statSync(path.join(temporary, 'managed-ossutilconfig')).mode & 0o777,
|
||||
).toBe(0o600);
|
||||
});
|
||||
|
||||
it('rejects a non-main manual run before consulting the release', () => {
|
||||
executable('gh', 'touch gh-called; exit 1');
|
||||
expect(run(gate, { WORKFLOW_REF: 'refs/heads/untrusted' }).status).not.toBe(
|
||||
@@ -312,6 +612,7 @@ describe.skipIf(process.platform === 'win32')('release gate execution', () => {
|
||||
SOURCE_SHA: 'original-sha',
|
||||
CLI_VERSION: '0.25.4',
|
||||
OSS_PUBLIC_BASE_URL: 'https://example.com',
|
||||
ALIYUN_OSS_BUCKET: 'release-bucket',
|
||||
});
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain('mismatch');
|
||||
|
||||
Reference in new issue
Block a user