diff --git a/.github/scripts/apt-mirror-probe.sh b/.github/scripts/apt-mirror-probe.sh new file mode 100644 index 00000000000..67fdd77e749 --- /dev/null +++ b/.github/scripts/apt-mirror-probe.sh @@ -0,0 +1,86 @@ +# shellcheck shell=bash +# Sourced by run-e2e-tests.sh and run-release-docker-integration.sh — both build +# the sandbox image with `npm run build:sandbox`, and both need the same apt +# source selection. Defines apt_mirror_probe_rate / select_apt_mirror; executes +# nothing on its own. +# +# The Dockerfile's built-in apt source is deb.debian.org. On the pool's +# mainland-China hosts that CDN path has been measured at 11-45 kB/s, which +# turns its two apt layers into a 2-3 hour build against the job's execution +# cap, so the leg is cancelled before any test runs (#16214). The same path +# runs at MB/s from the Hong Kong hosts, and it recovers on the mainland hosts +# at off-peak hours — so pinning either a mirror or a host list is wrong. +# Probe every build instead: the fastest endpoint wins, a healthy default is +# left alone, and no fleet map needs upkeep. + +apt_mirror_probe_rate() { + # A 512 KiB range read of a file every Debian mirror carries. Any large + # immutable index would do. The window matters more than the target: a + # shorter read cannot tell a warm CDN edge from a throttled one, while this + # one still finishes in well under a second wherever the path is healthy, + # so a slow candidate costs at most the 4-second cap. + local reply code rate + reply="$(curl -s -o /dev/null --max-time 4 --range 0-524287 \ + -w '%{http_code} %{speed_download}' \ + "http://$1/debian/dists/bookworm/main/binary-arm64/Packages.gz" 2>/dev/null)" || true + code="${reply%% *}" + rate="${reply##* }" + case "${code}" in + 200 | 206) printf '%s' "${rate%%.*}" ;; + *) printf '0' ;; + esac +} + +select_apt_mirror() { + command -v curl > /dev/null 2>&1 || return 0 + # Two gates, both on the mirror side. An absolute floor stops a "slow vs + # slower" override (two crawling endpoints must not move a host onto a + # mirror it never needed); the 4x factor keeps a genuinely fast default — + # the Hong Kong hosts — on the built-in source, since a mirror there is + # rarely 4x quicker. There is deliberately no "default looks healthy" gate: + # the mainland paths are bimodal, a 512 KiB probe can catch a fast burst + # (372 kB/s on hz4-2) while the sustained 200 MB transfer then throttles to + # 45 kB/s and times out (#16214 recurrence, run 41743539611). Trusting the + # mirror's own measured speed, not the default's, is what survives that. + local fast_mirror=1000000 + local default_rate best_rate=0 best='' mirror rate + default_rate="$(apt_mirror_probe_rate deb.debian.org)" + default_rate="${default_rate:-0}" + # mirrors.cloud.aliyuncs.com is the Alibaba Cloud VPC intranet mirror: it + # answers from inside the VPC and does not consume the instance's capped + # 100 Mbps public egress that throttles the public mirrors under concurrent + # load — the same reason configure-maven-mirror.sh prefers it for Central. + # Hosted runners and the Hong Kong hosts cannot reach it, so the probe just + # measures 0 there and it loses to a reachable public mirror; no host map is + # needed. Listed first because where it answers it is the fastest path. + for mirror in mirrors.cloud.aliyuncs.com mirrors.aliyun.com mirrors.tuna.tsinghua.edu.cn; do + rate="$(apt_mirror_probe_rate "${mirror}")" + rate="${rate:-0}" + if [ "${rate}" -gt "${best_rate}" ]; then + best="${mirror}" + best_rate="${rate}" + fi + done + # Print the measurements: the next person debugging a slow image build + # should not have to re-run this probe by hand (#16214). + echo "apt probe: deb.debian.org=${default_rate} B/s best=${best_rate} B/s (${best:-none})" >&2 + if [ -n "${best}" ] && [ "${best_rate}" -gt "${fast_mirror}" ] && + [ "${best_rate}" -gt $((default_rate * 4)) ]; then + printf '%s' "${best}" + fi +} + +# Applies the probe to the sandbox build: exports BUILD_SANDBOX_FLAGS with the +# selected APT_MIRROR build arg (preserving any flags already set, e.g. the CI +# sandbox label) and logs the decision. A no-op selection leaves the flags +# untouched so the Dockerfile keeps its built-in deb.debian.org source. +apply_apt_mirror() { + local apt_mirror + apt_mirror="$(select_apt_mirror)" + if [ -n "${apt_mirror}" ]; then + export BUILD_SANDBOX_FLAGS="${BUILD_SANDBOX_FLAGS:-} --build-arg APT_MIRROR=${apt_mirror}" + echo "apt mirror: ${apt_mirror} (probe measured deb.debian.org as slower)" + else + echo "apt mirror: deb.debian.org (probe found no faster mirror)" + fi +} diff --git a/.github/scripts/run-e2e-tests.sh b/.github/scripts/run-e2e-tests.sh index 4107f2e814f..40e6d3cb3a9 100644 --- a/.github/scripts/run-e2e-tests.sh +++ b/.github/scripts/run-e2e-tests.sh @@ -2,6 +2,9 @@ set -euo pipefail +# shellcheck source=.github/scripts/apt-mirror-probe.sh +source "$(dirname "${BASH_SOURCE[0]}")/apt-mirror-probe.sh" + sandbox="${1:?usage: run-e2e-tests.sh }" shard="${2:?usage: run-e2e-tests.sh }" @@ -31,71 +34,6 @@ if [ "$sandbox" = 'sandbox:docker' ]; then npm run build:sandbox -- -s --no-prune -i "$sandbox_image" 7>&- 8>&- 9>&- } - # The Dockerfile's built-in apt source is deb.debian.org. On the pool's - # mainland-China hosts that CDN path has been measured at 11-20 kB/s, which - # turns its two apt layers into a 2-3 hour build against this job's 60-minute - # cap, so the leg is cancelled before any test runs (#16214). The same path - # runs at MB/s from the Hong Kong hosts, and it recovers on the mainland hosts - # at off-peak hours — so pinning either a mirror or a host list is wrong. - # Probe every build instead: the fastest endpoint wins, a healthy default is - # left alone, and no fleet map needs upkeep. - apt_mirror_probe_rate() { - # A 512 KiB range read of a file every Debian mirror carries. Any large - # immutable index would do. The window matters more than the target: a - # shorter read cannot tell a warm CDN edge from a throttled one, while this - # one still finishes in well under a second wherever the path is healthy, - # so a slow candidate costs at most the 4-second cap. - local reply code rate - reply="$(curl -s -o /dev/null --max-time 4 --range 0-524287 \ - -w '%{http_code} %{speed_download}' \ - "http://$1/debian/dists/bookworm/main/binary-arm64/Packages.gz" 2>/dev/null)" || true - code="${reply%% *}" - rate="${reply##* }" - case "${code}" in - 200 | 206) printf '%s' "${rate%%.*}" ;; - *) printf '0' ;; - esac - } - - select_apt_mirror() { - command -v curl > /dev/null 2>&1 || return 0 - # Two gates, both on the mirror side. An absolute floor stops a "slow vs - # slower" override (two crawling endpoints must not move a host onto a - # mirror it never needed); the 4x factor keeps a genuinely fast default — - # the Hong Kong hosts — on the built-in source, since a mirror there is - # rarely 4x quicker. There is deliberately no "default looks healthy" gate: - # the mainland paths are bimodal, a 512 KiB probe can catch a fast burst - # (372 kB/s on hz4-2) while the sustained 200 MB transfer then throttles to - # 45 kB/s and times out (#16214 recurrence, run 41743539611). Trusting the - # mirror's own measured speed, not the default's, is what survives that. - local fast_mirror=1000000 - local default_rate best_rate=0 best='' mirror rate - default_rate="$(apt_mirror_probe_rate deb.debian.org)" - default_rate="${default_rate:-0}" - # mirrors.cloud.aliyuncs.com is the Alibaba Cloud VPC intranet mirror: it - # answers from inside the VPC and does not consume the instance's capped - # 100 Mbps public egress that throttles the public mirrors under concurrent - # load — the same reason configure-maven-mirror.sh prefers it for Central. - # Hosted runners and the Hong Kong hosts cannot reach it, so the probe just - # measures 0 there and it loses to a reachable public mirror; no host map is - # needed. Listed first because where it answers it is the fastest path. - for mirror in mirrors.cloud.aliyuncs.com mirrors.aliyun.com mirrors.tuna.tsinghua.edu.cn; do - rate="$(apt_mirror_probe_rate "${mirror}")" - rate="${rate:-0}" - if [ "${rate}" -gt "${best_rate}" ]; then - best="${mirror}" - best_rate="${rate}" - fi - done - # Print the measurements: the next person debugging a slow image build - # should not have to re-run this probe by hand (#16214). - echo "apt probe: deb.debian.org=${default_rate} B/s best=${best_rate} B/s (${best:-none})" >&2 - if [ -n "${best}" ] && [ "${best_rate}" -gt "${fast_mirror}" ] && - [ "${best_rate}" -gt $((default_rate * 4)) ]; then - printf '%s' "${best}" - fi - } - if [ "$RUNNER_ENVIRONMENT" = 'self-hosted' ]; then mkdir -p "${HOME}/.cache/qwen-code-ci" # Host daemon lock, shared for the whole step and never @@ -136,13 +74,7 @@ if [ "$sandbox" = 'sandbox:docker' ]; then # Label- and age-filtered, so it cannot touch the image a # concurrent shard is testing. timeout 20m docker image prune --all --force --filter 'label=org.qwen-code.ci.sandbox=true' --filter 'until=24h' || echo "::warning::old CI sandbox image cleanup failed on ${RUNNER_NAME:-this runner}" - apt_mirror="$(select_apt_mirror)" - if [ -n "${apt_mirror}" ]; then - export BUILD_SANDBOX_FLAGS="${BUILD_SANDBOX_FLAGS:-} --build-arg APT_MIRROR=${apt_mirror}" - echo "apt mirror: ${apt_mirror} (probe measured deb.debian.org as slower)" - else - echo "apt mirror: deb.debian.org (probe found no faster mirror)" - fi + apply_apt_mirror if ! build_image; then echo "::warning::sandbox image build failed; retrying once" build_image diff --git a/.github/scripts/run-release-docker-integration.sh b/.github/scripts/run-release-docker-integration.sh index 024c046076b..2e8880d56dc 100755 --- a/.github/scripts/run-release-docker-integration.sh +++ b/.github/scripts/run-release-docker-integration.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -euo pipefail +# shellcheck source=.github/scripts/apt-mirror-probe.sh +source "$(dirname "${BASH_SOURCE[0]}")/apt-mirror-probe.sh" + cleanup_release_containers() { container_ids="$(timeout 30 docker ps -aq --filter "label=org.qwen-code.ci.owner=${RELEASE_CONTAINER_OWNER}" 2>/dev/null)" || container_ids='' if [ -n "$container_ids" ]; then @@ -139,6 +142,11 @@ if [ "$image_inspect_status" -ne 0 ]; then # bounding that side too is a cross-lane change tracked separately. timeout 20m docker image prune --all --force --filter 'label=org.qwen-code.ci.sandbox=true' --filter 'until=24h' || echo "::warning::old CI sandbox image cleanup failed on ${RUNNER_NAME:-this runner}" check_docker_data_root_floor "${DISK_FLOOR_DOCKER_MIN_FREE_KB:-8388608}" + # This lane prunes the builder cache above, so the apt layers are always cold + # here — the same probe the E2E leg uses (#16214) selects the VPC intranet + # mirror on mainland hosts instead of rebuilding against the throttled + # default source (#16271). + apply_apt_mirror # See e2e.yml: closing the lock descriptors in the child keeps a descendant # that outlives this job from holding the lock. npm run build:sandbox -- -s --no-prune -i "$sandbox_image" 7>&- 8>&- 9>&- diff --git a/scripts/tests/e2e-workflow.test.js b/scripts/tests/e2e-workflow.test.js index e830a5d9a81..7059157b9bf 100644 --- a/scripts/tests/e2e-workflow.test.js +++ b/scripts/tests/e2e-workflow.test.js @@ -1096,18 +1096,25 @@ describe('e2e workflow', () => { it('probes the apt mirror instead of pinning a region', () => { // #16214: deb.debian.org serves this pool at MB/s from Hong Kong and - // 11-20 kB/s from the mainland hosts, where the two apt layers then - // outlast the job's 60-minute cap. So neither a single mirror nor a + // 11-45 kB/s from the mainland hosts, where the two apt layers then + // outlast the job's execution cap. So neither a single mirror nor a // host-name map is correct — the build probes and overrides only on a - // clear win, which leaves healthy hosts on the built-in source. - // The VPC intranet mirror is the durable fix: it bypasses the capped - // public egress that throttles the public mirrors (the maven script's - // rationale). It must stay a probe candidate — unreachable off-VPC, it - // measures 0 and loses, so no host map is needed. - expect(e2eRunScript).toContain('mirrors.cloud.aliyuncs.com'); - expect(e2eRunScript).toContain('mirrors.aliyun.com'); - expect(e2eRunScript).toContain('mirrors.tuna.tsinghua.edu.cn'); - expect(e2eRunScript).toContain('--build-arg APT_MIRROR='); + // clear win, which leaves healthy hosts on the built-in source. The + // probe lives in a shared script so the release docker lane (#16271) + // selects the same source; the VPC intranet mirror bypasses the capped + // public egress (the maven script's rationale) and, unreachable off-VPC, + // measures 0 and loses — no host map needed. + const aptProbeScript = readFileSync( + '.github/scripts/apt-mirror-probe.sh', + 'utf8', + ); + // Pin the candidate loop itself, not a substring: the mirrors are also + // named in the comments, so a toContain would survive removing a + // candidate from the actual `for` list. + expect(aptProbeScript).toMatch( + /for mirror in mirrors\.cloud\.aliyuncs\.com mirrors\.aliyun\.com mirrors\.tuna\.tsinghua\.edu\.cn; do/, + ); + expect(aptProbeScript).toContain('--build-arg APT_MIRROR='); // Both stages need the ARG: a stage cannot inherit an ARG declared // before its own FROM, so one declaration per stage is the contract. // Counting (not toMatch) catches deleting either one — R1-1 witnessed @@ -1116,16 +1123,15 @@ describe('e2e workflow', () => { readFileSync('Dockerfile', 'utf8').match(/^ARG APT_MIRROR=$/gm), ).toHaveLength(2); - const selectIndex = e2eRunScript.indexOf('select_apt_mirror()'); - expect(selectIndex).toBeGreaterThan(-1); - const callIndex = e2eRunScript.indexOf( - 'apt_mirror="$(select_apt_mirror)"', - ); - expect(callIndex).toBeGreaterThan(selectIndex); - // The probe has to run before the build consumes the flag, otherwise the - // mirror is decided after the layer it was meant to speed up. + // The E2E leg sources the shared probe and applies it before the build + // consumes the flag, otherwise the mirror is decided after the layer it + // was meant to speed up. Match the source statement, not a substring — + // the shellcheck directive above it also names the file. + expect(e2eRunScript).toMatch(/^source .*apt-mirror-probe\.sh"$/m); + const applyIndex = e2eRunScript.indexOf('apply_apt_mirror'); + expect(applyIndex).toBeGreaterThan(-1); expect(e2eRunScript.indexOf('if ! build_image;')).toBeGreaterThan( - callIndex, + applyIndex, ); // Two gates hold the override back, both on the mirror side: an absolute // floor (no "slow vs slower" override) and a 4x factor (a genuinely fast @@ -1133,16 +1139,16 @@ describe('e2e workflow', () => { // pinned, not just the names: a 10x mutation of the floor flips the // both-slow shape onto a still-slow mirror. \b, not toContain, so the // 10x-up mutation 'fast_mirror=10000000' cannot satisfy the pin. - expect(e2eRunScript).toMatch(/fast_mirror=1000000\b/); - expect(e2eRunScript).toMatch(/-gt "\$\{fast_mirror\}" \]/); - expect(e2eRunScript).toMatch(/-gt \$\(\(default_rate \* 4\)\)/); + expect(aptProbeScript).toMatch(/fast_mirror=1000000\b/); + expect(aptProbeScript).toMatch(/-gt "\$\{fast_mirror\}" \]/); + expect(aptProbeScript).toMatch(/-gt \$\(\(default_rate \* 4\)\)/); // There must be NO "default looks healthy" gate: the mainland paths are // bimodal, so a short probe of the default cannot veto the override // (#16214 recurrence on hz4-2, run 41743539611). Pin its absence. - expect(e2eRunScript).not.toContain('slow_default'); + expect(aptProbeScript).not.toContain('slow_default'); // The measurements have to reach the log, or the next slow build is // another manual investigation. - expect(e2eRunScript).toContain('apt probe:'); + expect(aptProbeScript).toContain('apt probe:'); }); it('keeps one bounded retry without pruning the shared daemon', () => { diff --git a/scripts/tests/release-workflow.test.js b/scripts/tests/release-workflow.test.js index eb78c4259b1..56b56b211bd 100644 --- a/scripts/tests/release-workflow.test.js +++ b/scripts/tests/release-workflow.test.js @@ -2516,6 +2516,23 @@ describe('release workflow', () => { ); }); + it('selects the apt mirror before the always-cold sandbox build', () => { + // #16271: this lane prunes the builder cache, so its apt layers are always + // cold — the same defect #16214 fixed for the E2E leg. It must source the + // shared probe and apply it before `build:sandbox`, so a mainland host + // builds against the VPC intranet mirror rather than the throttled default. + // Match the source statement, not a substring — the shellcheck directive + // above it also names the file. + expect(dockerIntegrationScript).toMatch( + /^source .*apt-mirror-probe\.sh"$/m, + ); + const applyIndex = dockerIntegrationScript.indexOf('apply_apt_mirror'); + expect(applyIndex).toBeGreaterThan(-1); + expect( + dockerIntegrationScript.indexOf('npm run build:sandbox'), + ).toBeGreaterThan(applyIndex); + }); + it('prunes BuildKit cache and gates the docker data root before the build and the vitest phase', () => { // #13479: run 37374675168 passed the job-start disk floor gate and the // runner still died on ENOSPC 24 minutes into the docker test step. The