diff --git a/.github/scripts/run-e2e-tests.sh b/.github/scripts/run-e2e-tests.sh index dcdb4e25e89..9e1d103d854 100644 --- a/.github/scripts/run-e2e-tests.sh +++ b/.github/scripts/run-e2e-tests.sh @@ -31,6 +31,64 @@ if [ "$sandbox" = 'sandbox:docker' ]; then npm run build:sandbox -- -s --no-prune -i "$sandbox_image" 7>&- 8>&- 9>&- } + # The Dockerfile's built-in apt source is deb.debian.org. On the pool's + # mainland-China hosts that CDN path has been measured at 11-20 kB/s, which + # turns its two apt layers into a 2-3 hour build against this job's 60-minute + # cap, so the leg is cancelled before any test runs (#16214). The same path + # runs at MB/s from the Hong Kong hosts, and it recovers on the mainland hosts + # at off-peak hours — so pinning either a mirror or a host list is wrong. + # Probe every build instead: the fastest endpoint wins, a healthy default is + # left alone, and no fleet map needs upkeep. + apt_mirror_probe_rate() { + # A 512 KiB range read of a file every Debian mirror carries. Any large + # immutable index would do. The window matters more than the target: a + # shorter read cannot tell a warm CDN edge from a throttled one, while this + # one still finishes in well under a second wherever the path is healthy, + # so a slow candidate costs at most the 4-second cap. + local reply code rate + reply="$(curl -s -o /dev/null --max-time 4 --range 0-524287 \ + -w '%{http_code} %{speed_download}' \ + "http://$1/debian/dists/bookworm/main/binary-arm64/Packages.gz" 2>/dev/null)" || true + code="${reply%% *}" + rate="${reply##* }" + case "${code}" in + 200 | 206) printf '%s' "${rate%%.*}" ;; + *) printf '0' ;; + esac + } + + select_apt_mirror() { + command -v curl > /dev/null 2>&1 || return 0 + # Below this rate the two apt layers (~200 MB) would eat more than ~15 of + # this job's 60 minutes, so a faster mirror is worth the extra cache key. + # Above it the built-in source is healthy and is left untouched — which is + # what keeps a fixed mirror from slowing the Hong Kong hosts down. + local slow_default=250000 + # A mirror also has to be fast in absolute terms, not merely faster than a + # broken default: without this floor, two slow endpoints would still + # "win" and move a host onto a mirror it never needed. + local fast_mirror=1000000 + local default_rate best_rate=0 best='' mirror rate + default_rate="$(apt_mirror_probe_rate deb.debian.org)" + default_rate="${default_rate:-0}" + for mirror in mirrors.aliyun.com mirrors.tuna.tsinghua.edu.cn; do + rate="$(apt_mirror_probe_rate "${mirror}")" + rate="${rate:-0}" + if [ "${rate}" -gt "${best_rate}" ]; then + best="${mirror}" + best_rate="${rate}" + fi + done + # Print the measurements: the next person debugging a slow image build + # should not have to re-run this probe by hand (#16214). + echo "apt probe: deb.debian.org=${default_rate} B/s best=${best_rate} B/s (${best:-none})" >&2 + if [ -n "${best}" ] && [ "${default_rate}" -lt "${slow_default}" ] && + [ "${best_rate}" -gt "${fast_mirror}" ] && + [ "${best_rate}" -gt $((default_rate * 4)) ]; then + printf '%s' "${best}" + fi + } + if [ "$RUNNER_ENVIRONMENT" = 'self-hosted' ]; then mkdir -p "${HOME}/.cache/qwen-code-ci" # Host daemon lock, shared for the whole step and never @@ -71,6 +129,13 @@ if [ "$sandbox" = 'sandbox:docker' ]; then # Label- and age-filtered, so it cannot touch the image a # concurrent shard is testing. timeout 20m docker image prune --all --force --filter 'label=org.qwen-code.ci.sandbox=true' --filter 'until=24h' || echo "::warning::old CI sandbox image cleanup failed on ${RUNNER_NAME:-this runner}" + apt_mirror="$(select_apt_mirror)" + if [ -n "${apt_mirror}" ]; then + export BUILD_SANDBOX_FLAGS="${BUILD_SANDBOX_FLAGS:-} --build-arg APT_MIRROR=${apt_mirror}" + echo "apt mirror: ${apt_mirror} (probe measured deb.debian.org as slower)" + else + echo "apt mirror: deb.debian.org (probe found no faster mirror)" + fi if ! build_image; then echo "::warning::sandbox image build failed; retrying once" build_image diff --git a/Dockerfile b/Dockerfile index 542dda20b17..14333b3ada3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,20 @@ # ratchet:docker.io/library/node:22-slim FROM docker.io/library/node:22-slim@sha256:83f487e0a63425e5b4d146fb5e5be574bcbe1b7b843d3ebafdd95eaf7767a7e5 AS builder +# Optional apt mirror for the layer below. Empty keeps deb.debian.org, which is +# what release builds and local sandbox builds use; the E2E leg passes a +# regional mirror only on hosts where its probe found the default CDN path to be +# far slower (#16214). +ARG APT_MIRROR= + # Install build dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN set -e; \ + if [ -n "${APT_MIRROR}" ]; then \ + for sources in /etc/apt/sources.list /etc/apt/sources.list.d/debian.sources; do \ + if [ -f "${sources}" ]; then sed -i "s|deb.debian.org|${APT_MIRROR}|g" "${sources}"; fi; \ + done; \ + fi; \ + apt-get update && apt-get install -y --no-install-recommends \ python3 \ make \ g++ \ @@ -40,11 +52,20 @@ FROM docker.io/library/node:22-slim@sha256:83f487e0a63425e5b4d146fb5e5be574bcbe1 ARG SANDBOX_NAME="qwen-code-sandbox" ARG CLI_VERSION_ARG +# Same optional override as the builder stage: a stage cannot inherit an ARG +# declared before its own FROM. +ARG APT_MIRROR= ENV SANDBOX="$SANDBOX_NAME" ENV CLI_VERSION=$CLI_VERSION_ARG # Install runtime dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN set -e; \ + if [ -n "${APT_MIRROR}" ]; then \ + for sources in /etc/apt/sources.list /etc/apt/sources.list.d/debian.sources; do \ + if [ -f "${sources}" ]; then sed -i "s|deb.debian.org|${APT_MIRROR}|g" "${sources}"; fi; \ + done; \ + fi; \ + apt-get update && apt-get install -y --no-install-recommends \ python3 \ man-db \ curl \ diff --git a/scripts/tests/e2e-workflow.test.js b/scripts/tests/e2e-workflow.test.js index 27b21431b76..e8c67467512 100644 --- a/scripts/tests/e2e-workflow.test.js +++ b/scripts/tests/e2e-workflow.test.js @@ -1094,6 +1094,52 @@ describe('e2e workflow', () => { ); }); + it('probes the apt mirror instead of pinning a region', () => { + // #16214: deb.debian.org serves this pool at MB/s from Hong Kong and + // 11-20 kB/s from the mainland hosts, where the two apt layers then + // outlast the job's 60-minute cap. So neither a single mirror nor a + // host-name map is correct — the build probes and overrides only on a + // clear win, which leaves healthy hosts on the built-in source. + expect(e2eRunScript).toContain('mirrors.aliyun.com'); + expect(e2eRunScript).toContain('mirrors.tuna.tsinghua.edu.cn'); + expect(e2eRunScript).toContain('--build-arg APT_MIRROR='); + // Both stages need the ARG: a stage cannot inherit an ARG declared + // before its own FROM, so one declaration per stage is the contract. + // Counting (not toMatch) catches deleting either one — R1-1 witnessed + // the runtime-stage removal surviving a single-match assertion. + expect( + readFileSync('Dockerfile', 'utf8').match(/^ARG APT_MIRROR=$/gm), + ).toHaveLength(2); + + const selectIndex = e2eRunScript.indexOf('select_apt_mirror()'); + expect(selectIndex).toBeGreaterThan(-1); + const callIndex = e2eRunScript.indexOf( + 'apt_mirror="$(select_apt_mirror)"', + ); + expect(callIndex).toBeGreaterThan(selectIndex); + // The probe has to run before the build consumes the flag, otherwise the + // mirror is decided after the layer it was meant to speed up. + expect(e2eRunScript.indexOf('if ! build_image;')).toBeGreaterThan( + callIndex, + ); + // Three conditions hold the override back, and dropping any one is a real + // regression: a healthy default host keeps its source, a mirror has to be + // fast in absolute terms rather than merely faster than a broken default, + // and it has to beat the default decisively. The literals are pinned, not + // just the variable names: a 10x mutation of either threshold leaves the + // structure intact but flips the both-slow shape onto a still-slow mirror. + // \b, not toContain: 'slow_default=250000' is a substring of the 10x + // mutation 'slow_default=2500000', which would keep a naive pin green. + expect(e2eRunScript).toMatch(/slow_default=250000\b/); + expect(e2eRunScript).toMatch(/-lt "\$\{slow_default\}" \]/); + expect(e2eRunScript).toMatch(/fast_mirror=1000000\b/); + expect(e2eRunScript).toMatch(/-gt "\$\{fast_mirror\}" \]/); + expect(e2eRunScript).toMatch(/-gt \$\(\(default_rate \* 4\)\)/); + // The measurements have to reach the log, or the next slow build is + // another manual investigation. + expect(e2eRunScript).toContain('apt probe:'); + }); + it('keeps one bounded retry without pruning the shared daemon', () => { expect(e2eRunScript.match(/build_image/g)).toHaveLength(3); expect(e2eRunScript).toContain(