From cfd95f6abbfed3bf17cd274cb57c3b7b4fafa01b Mon Sep 17 00:00:00 2001 From: qwen-code-dev-bot Date: Thu, 22 Oct 2026 21:44:45 +0800 Subject: [PATCH] fix(ci): scope the sdk-java host lock to the cache write-back (#15128) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ci): scope the sdk-java host lock to the cache write-back (#15117) Run 39627609266's fast-tier job arrived at 17:38, stayed head of the sdk-java host-lock queue, and failed at 19:15 without starting Maven. The MariaDB integration lane held the same lock across its install and full MySQL clean verify; under pool contention that hold ran from 15:54 to 18:57 (run 39628985638), past the fast tier's 90-minute budget. Both MariaDB Maven runs already resolve from a job-local repository, so they touch no host-shared state. Seed that repository before the lock, run both Maven commands unlocked, and take the host lock only for the cache write-back, the sole remaining host-shared mutation. Drop fd 9 on the write-back children so a cancelled job's survivor cannot retain the lock. Raise the lane ceiling to 300 minutes for the ~3-hour verify plus one lock wait and setup, and pin the new order and fd closes. Fixes #15117 * test(ci): exempt the OpenTUI parity suites from the no-AK gate (#15117) The main merge brought three interactive suites from #14964. The no-AK gate requires every runnable integration test to be selected or explicitly exempted, so the Test job fails until they are listed. Co-authored-by: Qwen-Coder --------- Co-authored-by: Qwen Autofix Co-authored-by: Qwen-Coder Co-authored-by: jinjing.zzj Co-authored-by: 易良 <1204183885@qq.com> --- .github/workflows/.size-baseline | 2 +- .github/workflows/sdk-java.yml | 103 ++++++------ .../2026-10-20-sdk-java-fair-lock-queue.md | 28 ++- ...26-10-20-sdk-java-fair-lock-queue.zh-CN.md | 22 ++- scripts/tests/sdk-java-workflow.test.js | 159 +++++++++++++++--- 5 files changed, 232 insertions(+), 82 deletions(-) diff --git a/.github/workflows/.size-baseline b/.github/workflows/.size-baseline index 26679459c65..740707b4737 100644 --- a/.github/workflows/.size-baseline +++ b/.github/workflows/.size-baseline @@ -59,7 +59,7 @@ 1079 scorecard-monthly.yml 9751 sdk-java-contract-open-prs.yml 9555 sdk-java-flyway-open-prs.yml -114180 sdk-java.yml +114901 sdk-java.yml 3886 sdk-python.yml 5595 security-checks.yml 6777 serve-ab-publish.yml diff --git a/.github/workflows/sdk-java.yml b/.github/workflows/sdk-java.yml index 92a37df35bb..43ad42bd308 100644 --- a/.github/workflows/sdk-java.yml +++ b/.github/workflows/sdk-java.yml @@ -450,7 +450,8 @@ jobs: # Pool routing per the flyway job (#13245): hosted queue starvation fails # a zero-step job outright (#13623), the pool does not. The job is # pool-safe: the database is a per-leg container on a random host port, - # and every Maven run holds the test job's per-host sdk-java lock. + # both Maven runs use a job-local repository, and only the shared cache + # write-back holds the test job's per-host sdk-java lock. # Both legs run the same non-Hosted *IT family, as before #14936 removed # the MariaDB service. The MariaDB leg is what keeps the published # MariaDB-only V12 repair (RTRIM, error 1901) and the schema catalog's @@ -471,17 +472,16 @@ jobs: image: 'mariadb:10.11.18' client: 'mariadb' engine_args: '--innodb-use-native-aio=0' - # Each leg takes the per-host sdk-java lock once, around the single step - # that runs both Maven commands. Run 39353663670's verify - # acquisition waited the full 90 minutes after the install step released - # the lock, so a per-step lock let this job lose its place in the host - # queue between installing and verifying. The ceiling must cover that - # 90-minute wait, up to ~100 minutes from acquisition to the end of - # verify (job 118848599935: a 71-minute seed, 26 of Maven), the 5-minute - # write-back backstop and ~30 minutes of setup; 280 leaves margin. Run - # 39563394993 hit the old 200. Hosted lanes never take the lock - # and inherit this ceiling as a bound, not as an expected duration. - timeout-minutes: 280 + # On the pool this job takes the per-host sdk-java lock only for its + # shared Maven cache write-back: both Maven runs resolve from a job-local + # repository, so holding the lock across the whole integration suite + # serialized the fast tier behind a multi-hour hold (run 39628985638 held + # it 15:54-18:57 while run 39627609266's fast-tier job burned its full + # 90-minute wait). The ceiling absorbs that wait, the ~3-hour Maven run, + # the 5-minute write-back backstop and 30 minutes of uncapped setup: 310. + # Hosted lanes never take the lock and inherit this ceiling as a bound, + # not as an expected duration. + timeout-minutes: 310 env: MYSQL_CONTAINER: 'qwen-code-sdk-java-${{ matrix.database }}-${{ github.run_id }}-${{ github.run_attempt }}' DATABASE_IMAGE: '${{ matrix.image }}' @@ -732,33 +732,17 @@ jobs: MAVEN_ARGS: '--settings ${{ runner.temp }}/setup-java-m2/settings.xml --toolchains ${{ runner.temp }}/setup-java-m2/toolchains.xml -Dmaven.repo.local=${{ runner.temp }}/m2-repo' run: |- set -o pipefail - # One acquisition spans both Maven runs: run 39353663670's verify - # step waited the full 90 minutes after the install step released - # the lock, so a per-step lock let this job lose its place in the - # host queue between installing and verifying. The ticketed queue - # in sdk-java-lock.sh orders that one wait (fast PR legs overtake - # it until it parks or is promoted) instead of a wake-up race - # against sibling holds (qwen-code#14743). - if [ -f "${GITHUB_WORKSPACE}/.github/scripts/sdk-java-lock.sh" ]; then - source "${GITHUB_WORKSPACE}/.github/scripts/sdk-java-lock.sh" - sdk_java_lock_acquire 5400 - else - # release-sdk-java.yml resumes an existing tag by checking out - # that immutable commit, which may predate this helper. Fall - # back to the bare lock this queue replaced rather than dying - # before Maven and wedging the release. - mkdir -p "${HOME}/.cache/qwen-code-ci" - exec 9>"${HOME}/.cache/qwen-code-ci/sdk-java-tests.lock" - if ! flock --wait 5400 9; then - echo "::error::sdk-java host lock not acquired within 90 minutes" - exit 1 - fi - fi # Seed from the host-shared repo (cache-restored on hosted runners) # so it and the egress proxy keep paying off for the job-local copy. + # The seed only reads the host cache and is best-effort, so it takes + # no lock and cannot delay the lane. cp -aln "${HOME}/.m2/repository" "${RUNNER_TEMP}/m2-repo" 2>/dev/null || true - # The shell owns the lock; Maven children must not extend its lifetime. - mvn --batch-mode --no-transfer-progress -f packages/sdk-java/qwencode/pom.xml -DskipTests -Dgpg.skip=true -Dmaven.javadoc.skip=true install 9>&- + # Both Maven runs resolve from the job-local copy above, so they do + # not touch host-shared state and must not hold the per-host lock: + # a MySQL verify can run for hours under pool contention, and the + # fast tier's 90-minute wait expires behind it (run 39627609266 + # waited 17:38-19:15 behind run 39628985638's integration hold). + mvn --batch-mode --no-transfer-progress -f packages/sdk-java/qwencode/pom.xml -DskipTests -Dgpg.skip=true -Dmaven.javadoc.skip=true install cd packages/managed-agent-server # Keep mysql.url for release runs that check out an older tag. mvn --batch-mode --no-transfer-progress -Pmysql-integration \ @@ -766,15 +750,38 @@ jobs: -Dmysql.url="jdbc:mysql://127.0.0.1:${MYSQL_PORT_1}/managed_agent_test?createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true&useSSL=false" \ -Dmysql.url.1="jdbc:mysql://127.0.0.1:${MYSQL_PORT_1}/managed_agent_test?createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true&useSSL=false" \ -Dmysql.url.2="jdbc:mysql://127.0.0.1:${MYSQL_PORT_2}/managed_agent_test?createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true&useSSL=false" \ - -Dmysql.user=root -Dmysql.password=runtime-broker clean verify checkstyle:check 9>&- - # Close the cache loop inside the same lock: the one-way seed - # above alone does not recapture the Server dependency tree after - # a pom key rotation. The excludes name the two in-house - # fixed release coordinates rather than the whole com/alibaba - # group, so third-party artifacts under it (druid) still - # recapture into the host-shared ~/.m2. Best-effort like the - # seed: the warning arms keep a copy failure from reddening the - # gate after mvn has already passed. + -Dmysql.user=root -Dmysql.password=runtime-broker clean verify checkstyle:check + # Only the write-back mutates host-shared state, so only it takes + # the per-host lock. One acquisition spans the write-back: run + # 39353663670's verify step waited the full 90 minutes after the + # install step released the lock, so a per-step lock let this job + # lose its place in the host queue between installing and verifying. + # The ticketed queue in sdk-java-lock.sh orders that one wait (fast + # PR legs overtake it until it parks or is promoted) instead of a + # wake-up race against sibling holds (qwen-code#14743). + if [ -f "${GITHUB_WORKSPACE}/.github/scripts/sdk-java-lock.sh" ]; then + source "${GITHUB_WORKSPACE}/.github/scripts/sdk-java-lock.sh" + sdk_java_lock_acquire 5400 + else + # release-sdk-java.yml resumes an existing tag by checking out + # that immutable commit, which may predate this helper. Fall + # back to the bare lock this queue replaced rather than dying + # before the write-back and wedging the release. + mkdir -p "${HOME}/.cache/qwen-code-ci" + exec 9>"${HOME}/.cache/qwen-code-ci/sdk-java-tests.lock" + if ! flock --wait 5400 9; then + echo "::error::sdk-java host lock not acquired within 90 minutes" + exit 1 + fi + fi + # Close the cache loop inside that lock: the one-way seed above + # alone does not recapture the Server dependency tree after a pom + # key rotation. The excludes name the two in-house fixed release + # coordinates rather than the whole com/alibaba group, so + # third-party artifacts under it (druid) still recapture into the + # host-shared ~/.m2. Best-effort like the seed: the warning arms + # keep a copy failure from reddening the gate after mvn has already + # passed. # On a fresh ECS host nothing creates ~/.m2/repository anymore # (the hosted-only cache no longer restores it), so create it # before the write-back or every job re-resolves through the @@ -784,10 +791,12 @@ jobs: # took up to 61 minutes (run 39563394993). Each link is atomic and # -n never replaces a host file, so the backstop can leave an # artifact missing for the next job to re-resolve, never truncated. - mkdir -p "${HOME}/.m2/repository" + # No write-back child inherits fd 9, or a cancelled job's survivor + # would hold the host lock until it exits. + mkdir -p "${HOME}/.m2/repository" 9>&- cd "${RUNNER_TEMP}/m2-repo" write_back=0 - timeout 300 find . \( -path ./com/alibaba/qwencode-sdk -o -path ./com/alibaba/qwen-managed-agent-server \) -prune -o -type f -links 1 -exec cp -ln --parents -t "${HOME}/.m2/repository" {} + || write_back=$? + timeout 300 find . \( -path ./com/alibaba/qwencode-sdk -o -path ./com/alibaba/qwen-managed-agent-server \) -prune -o -type f -links 1 -exec cp -ln --parents -t "${HOME}/.m2/repository" {} + 9>&- || write_back=$? if [ "${write_back}" -eq 124 ]; then echo "::warning::Maven cache write-back stopped after 300 s; the gate result above is unaffected" elif [ "${write_back}" -ne 0 ]; then diff --git a/docs/design/2026-10-20-sdk-java-fair-lock-queue.md b/docs/design/2026-10-20-sdk-java-fair-lock-queue.md index 546f035f8c7..4a89961c1b9 100644 --- a/docs/design/2026-10-20-sdk-java-fair-lock-queue.md +++ b/docs/design/2026-10-20-sdk-java-fair-lock-queue.md @@ -32,7 +32,7 @@ hold at a time — no constant budget bounds the queue. - Non-goal: reducing total occupancy. Sharding the lock by suite, capping registrations per host, or adding hosts are orthogonal and can follow; fairness makes their remaining work measurable instead of urgent. -- Non-goal: changing the 5400 s budget or the 150/280-minute job ceilings. +- Non-goal: changing the 5400 s budget or the 150/310-minute job ceilings. ## Design @@ -82,6 +82,25 @@ A new sourced helper, `.github/scripts/sdk-java-lock.sh`, replaces the inline `flock --wait 5400 9` lock they replaced when the helper file is absent, rather than dying before Maven and wedging the release permanently. +### Hold scope: the write-back, not the suite + +A fair queue still cannot bound a wait that outlasts its budget. The MariaDB +lane held the lock across its install and its full MySQL `clean verify`; under +pool contention that verify ran for about three hours (run 39628985638 held it +15:54-18:57) while the fast tier owns the same lock with a 90-minute wait. Run +39627609266's fast-tier job arrived at 17:38, stayed head of the queue, and +failed at 19:15 without starting Maven — the hold, not the queue order, was +the bound. + +Both Maven runs already resolve from a job-local repository, so they touch no +host-shared state. Only the cache write-back mutates `~/.m2`, so only the +write-back takes the lock. The seed (`cp -aln`) is read-only and best-effort +and runs before the acquisition; the Maven runs run unlocked; the write-back +holds one acquisition, and its `mkdir` and bounded `find`/`cp` children +drop fd 9 so a survivor +cannot retain the lock. A hold now measures the copy instead of the suite, and +the fast tier's 90-minute budget is again far above one hold. + ### Fast-tier priority The FIFO queue bounds a waiter by its position, but it still puts a fast-tier @@ -189,9 +208,10 @@ order this section exists to prevent. ordering stubs park `flock --wait` until release, as the kernel does, so a head cannot loop and be overtaken. - The workflow pins move with the mechanism: - `scripts/tests/sdk-java-workflow.test.js` now pins the sourced helper and - the `sdk_java_lock_acquire 5400` budget at each acquisition site, and the - `.size-baseline` records the workflow's new size. + `scripts/tests/sdk-java-workflow.test.js` pins the sourced helper and the + `sdk_java_lock_acquire 5400` budget at each acquisition site, the MariaDB + lane's Maven runs above its acquisition and its write-back children's fd 9 + close, and the `.size-baseline` records the workflow's new size. - Real-host soak is CI itself: the first pool runs after merge either acquire in ticket order or time out with an attributable ticket number. diff --git a/docs/design/2026-10-20-sdk-java-fair-lock-queue.zh-CN.md b/docs/design/2026-10-20-sdk-java-fair-lock-queue.zh-CN.md index 0f9b98332c5..aa7fe4e205e 100644 --- a/docs/design/2026-10-20-sdk-java-fair-lock-queue.zh-CN.md +++ b/docs/design/2026-10-20-sdk-java-fair-lock-queue.zh-CN.md @@ -29,7 +29,7 @@ runner 注册抢同一把锁，需求随到达率增长而容量一次只能放 - 非目标：降低总占用。按套件分片锁、限制每台宿主机注册数、加机器， 都是正交工作，可以后续再做；有了公平性，这些工作的剩余价值才 变得可衡量，而不是紧急。 -- 非目标：修改 5400 秒预算或 150/280 分钟的 job 上限。 +- 非目标：修改 5400 秒预算或 150/310 分钟的 job 上限。 ## 设计 @@ -72,6 +72,21 @@ runner 注册抢同一把锁，需求随到达率增长而容量一次只能放 `flock --wait 5400 9`，而不是在 Maven 之前直接失败并让发布永久 卡死。 +### 持锁范围：只包住写回，而不是整套测试 + +公平队列也无法约束超过预算的等待。MariaDB 通道曾把锁持在整个 install 和 +完整的 MySQL `clean verify` 之上；在池竞争下那次 verify 跑了约三小时 +（run 39628985638 在 15:54-18:57 持锁），而 fast 通道用的是同一把锁、只有 +90 分钟等待预算。run 39627609266 的 fast 通道 job 17:38 到达，一直是队首， +19:15 失败，Maven 一次都没启动 —— 约束它的是持锁时长，不是排队顺序。 + +两次 Maven 运行本来就都从 job 本地仓库解析，不触碰宿主机共享状态。只有 +缓存写回会修改 `~/.m2`，因此只有写回取锁。种子复制（`cp -aln`）是只读且 +尽力而为的，放在取锁之前；两次 Maven 运行不持锁；写回只取一次锁，且它的 +`mkdir` 和有界 `find`/`cp` 子进程会关闭 fd 9，避免幸存进程继续持有锁。这样一次持锁 +只量一次复制，而不是一整套测试，fast 通道的 90 分钟预算又远大于一次 +持锁时长。 + ### fast 档优先 FIFO 队列把 waiter 的等待限制在它的排位内，但 fast 档 job 仍然排在所有 @@ -161,8 +176,9 @@ normal 票在内核中占到比之后到达的 fast 票更靠前的位置，而 会像内核一样让 `flock --wait` 一直阻塞到释放，因此队首不会循环、 也不会被超过。 - 工作流的 pin 随机制迁移：`scripts/tests/sdk-java-workflow.test.js` - 现在 pin 的是 source 该辅助脚本和两处 `sdk_java_lock_acquire 5400` - 预算，`.size-baseline` 记录工作流的新尺寸。 + pin 的是 source 该辅助脚本和两处 `sdk_java_lock_acquire 5400` 预算、 + MariaDB 通道的 Maven 运行在取锁之前、写回子进程关闭 fd 9，以及 + `.size-baseline` 记录的工作流新尺寸。 - 真实宿主机上的浸泡验证就是 CI 本身：合并后第一批池上运行要么 按票序拿到锁，要么带着可归因的票据号超时。 diff --git a/scripts/tests/sdk-java-workflow.test.js b/scripts/tests/sdk-java-workflow.test.js index 302ed6b8033..186e4fa8469 100644 --- a/scripts/tests/sdk-java-workflow.test.js +++ b/scripts/tests/sdk-java-workflow.test.js @@ -155,7 +155,7 @@ describe('SDK Java self-hosted workflow guards', () => { ); }); - it('keeps the host lock out of every Maven child', () => { + it('keeps the host lock out of every locked child', () => { const jobs = parse(workflow).jobs; const lockedSteps = ['test', 'mysql-integration'].flatMap((name) => jobs[name].steps.filter((s) => s.run?.includes('sdk-java-tests.lock')), @@ -163,25 +163,40 @@ describe('SDK Java self-hosted workflow guards', () => { // The MariaDB lane's two Maven steps are one step now (#14903), so the // lock-bearing steps are the test job's and this job's single one. expect(lockedSteps).toHaveLength(2); - for (const s of lockedSteps) { - const commands = s.run.replace(/\\\n/g, ' ').match(/^\s*mvn .+$/gm); - expect(commands?.length, s.name).toBeGreaterThan(0); - for (const command of commands) { - expect(command.trim(), s.name).toMatch(/ 9>&-$/); - } + // The test job runs Maven under the lock, so every Maven child must drop + // fd 9 or a survivor keeps the host lock after the step shell exits. + const maven = lockedSteps[0].run.match(/^\s*mvn .+$/gm); + expect(maven?.length, lockedSteps[0].name).toBeGreaterThan(0); + for (const command of maven) { + expect(command.trim(), lockedSteps[0].name).toMatch(/ 9>&-$/); } + // The MariaDB lane locks only its write-back: both Maven runs are above + // the acquisition, and the write-back children drop fd 9 so a cancelled + // job's survivor cannot retain the lock. + const integration = lockedSteps[1].run; + const lockAt = integration.indexOf( + 'source "${GITHUB_WORKSPACE}/.github/scripts/sdk-java-lock.sh"', + ); + expect( + integration.indexOf('-f packages/sdk-java/qwencode/pom.xml'), + ).toBeLessThan(lockAt); + expect(integration.indexOf('-Pmysql-integration')).toBeLessThan(lockAt); + expect(integration).toContain('mkdir -p "${HOME}/.m2/repository" 9>&-'); + expect(integration).toMatch( + /timeout 300 find \..*-exec cp -ln --parents -t "\$\{HOME\}\/\.m2\/repository" \{\} \+ 9>&- \|\| write_back=\$\?/, + ); }); it.skipIf(process.platform !== 'linux')( 'retains serialization while Maven runs and releases it with a surviving child', () => { const jobs = parse(workflow).jobs; - const commands = ['test', 'mysql-integration'].flatMap((name) => - jobs[name].steps.flatMap((s) => - s.run?.includes('sdk-java-tests.lock') - ? (s.run.replace(/\\\n/g, ' ').match(/^\s*mvn .+$/gm) ?? []) - : [], - ), + // Only the test job's Maven runs are inside the lock now; the MariaDB + // lane's Maven runs resolve from a job-local repository above it. + const commands = jobs.test.steps.flatMap((s) => + s.run?.includes('sdk-java-tests.lock') + ? (s.run.replace(/\\\n/g, ' ').match(/^\s*mvn .+$/gm) ?? []) + : [], ); const dir = mkdtempSync(join(tmpdir(), 'sdk-java-lock-')); try { @@ -221,6 +236,94 @@ describe('SDK Java self-hosted workflow guards', () => { }, ); + it.skipIf(process.platform !== 'linux')( + 'releases the host lock when a write-back child survives the step', + () => { + const jobs = parse(workflow).jobs; + const integration = jobs['mysql-integration'].steps.find((s) => + s.run?.includes('sdk-java-tests.lock'), + ).run; + const start = integration.indexOf('mkdir -p "${HOME}/.m2/repository"'); + expect(start).toBeGreaterThan(-1); + const writeBack = integration.slice(start); + const dir = mkdtempSync(join(tmpdir(), 'sdk-java-write-back-')); + const shims = join(dir, 'bin'); + const home = join(dir, 'home'); + const runnerTemp = join(dir, 'runner-temp'); + const pidFile = join(dir, 'children.pid'); + mkdirSync(shims); + mkdirSync(join(home, '.m2', 'repository'), { recursive: true }); + mkdirSync(join(runnerTemp, 'm2-repo'), { recursive: true }); + // A child that outlives the step: if it inherits fd 9, the host lock + // stays held after the step shell exits. `mkdir` and `timeout` are the + // write-back's only children, so each gets one. + for (const name of ['mkdir', 'timeout']) { + writeFileSync( + join(shims, name), + '#!/bin/sh\n' + + 'sleep 30 /dev/null 2>&1 &\n' + + 'printf \'%s\\n\' "$!" >> "$CHILD_PID_FILE"\n' + + 'exit 0\n', + { mode: 0o755 }, + ); + } + try { + const result = spawnSync( + 'bash', + [ + '-e', + '-c', + ` + exec 9>"$LOCK_FILE" + flock --nonblock 9 + ${writeBack} + `, + ], + { + encoding: 'utf8', + timeout: 5000, + env: { + ...process.env, + PATH: `${shims}:${process.env.PATH}`, + HOME: home, + RUNNER_TEMP: runnerTemp, + CHILD_PID_FILE: pidFile, + LOCK_FILE: join(dir, 'tests.lock'), + }, + }, + ); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + const children = readFileSync(pidFile, 'utf8') + .trim() + .split('\n') + .filter(Boolean); + expect(children.length).toBeGreaterThanOrEqual(2); + // The step shell has exited. A survivor that inherited fd 9 would + // still hold the lock; the closures are what keep it free. + const free = spawnSync( + 'flock', + ['--nonblock', join(dir, 'tests.lock'), 'true'], + { encoding: 'utf8' }, + ); + expect(free.status, free.stderr).toBe(0); + } finally { + if (existsSync(pidFile)) { + for (const pid of readFileSync(pidFile, 'utf8').split('\n')) { + if (pid.trim()) { + try { + process.kill(Number(pid), 'SIGKILL'); + } catch { + // Already gone. + } + } + } + } + rmSync(dir, { recursive: true, force: true }); + } + }, + ); + it('runs Broker tests from the combined Server after installing the SDK on self-hosted Java 21', () => { const block = step(job('test'), 'Run Java SDK tests (self-hosted)'); expect(block).toContain("working-directory: 'packages/sdk-java/qwencode'"); @@ -1101,13 +1204,12 @@ describe('SDK Java MySQL lane on the ECS pool', () => { } }); - it('holds the per-host sdk-java lock once around both Maven runs', () => { - // Derived from the parsed steps, not a string count: the install and - // verify commands share one acquisition — the shared path pinned for the - // test job above is what makes the two jobs mutually exclusive on one ECS - // host — and wait on it with the pinned literal. Run 39353663670's second - // acquisition lost the queue between the two steps and exhausted the wait - // without starting Maven. + it('holds the per-host sdk-java lock once around the cache write-back', () => { + // The Maven runs resolve from the job-local repository, so the lock now + // guards only the host-shared cache write-back. Run 39627609266's + // fast-tier job burned its full 90-minute wait while the previous + // whole-suite hold ran from 15:54 to 18:57 (run 39628985638); a + // multi-hour verify must not sit between the fast tier and its hold. const parsed = parse(workflow); const mavenSteps = parsed.jobs['mysql-integration'].steps.filter( (s) => typeof s.run === 'string' && s.run.includes('mvn '), @@ -1153,21 +1255,24 @@ describe('SDK Java MySQL lane on the ECS pool', () => { // verify it runs `clean verify` in the wrong module. const cdAt = step.run.indexOf('cd packages/managed-agent-server'); const verifyAt = step.run.indexOf('-Pmysql-integration'); + const writeBackAt = step.run.indexOf('cd "${RUNNER_TEMP}/m2-repo"'); expect(lockAt).toBeGreaterThan(-1); - expect(installAt).toBeGreaterThan(lockAt); + expect(installAt).toBeGreaterThan(-1); + expect(installAt).toBeLessThan(lockAt); expect(cdAt).toBeGreaterThan(installAt); expect(verifyAt).toBeGreaterThan(cdAt); - // The ceiling must absorb one full lock wait plus up to ~100 minutes - // from acquisition to the end of verify (a 71-minute seed and 26 minutes - // of Maven in job 118848599935), the write-back backstop and ~30 minutes - // of uncapped setup; run 39563394993 was cancelled at the old 200. + expect(verifyAt).toBeLessThan(lockAt); + expect(writeBackAt).toBeGreaterThan(lockAt); + // The ceiling must absorb one full lock wait, the ~3-hour Maven run, + // the write-back backstop and ~30 minutes of uncapped setup; 310 leaves + // a few minutes of margin. const writeBackBound = Number( step.run.match(/timeout (\d+) find \. /)?.[1], ); expect(writeBackBound).toBe(300); expect( parsed.jobs['mysql-integration']['timeout-minutes'] * 60, - ).toBeGreaterThanOrEqual(seconds + 100 * 60 + writeBackBound + 30 * 60); + ).toBeGreaterThanOrEqual(seconds + 3 * 60 * 60 + writeBackBound + 30 * 60); // The sibling test job shares the same per-host lock: its ceiling must // cover one full wait plus its own measured hold (the sdk-java suite plus // the managed-agent H2 suite, 925 s) and ~11 minutes of setup. @@ -1208,7 +1313,7 @@ describe('SDK Java MySQL lane on the ECS pool', () => { // artifact in the shared cache the next seed hardlinks from. expect(step.run).toContain('cd "${RUNNER_TEMP}/m2-repo"'); expect(step.run).toContain( - 'timeout 300 find . \\( -path ./com/alibaba/qwencode-sdk -o -path ./com/alibaba/qwen-managed-agent-server \\) -prune -o -type f -links 1 -exec cp -ln --parents -t "${HOME}/.m2/repository" {} + || write_back=$?', + 'timeout 300 find . \\( -path ./com/alibaba/qwencode-sdk -o -path ./com/alibaba/qwen-managed-agent-server \\) -prune -o -type f -links 1 -exec cp -ln --parents -t "${HOME}/.m2/repository" {} + 9>&- || write_back=$?', ); expect(step.run).not.toMatch(/tar [^\n]*-xf/); expect(step.run).toContain(