fix(ci): hold the MariaDB lane's sdk-java lock once per job (#14896) (#14903)

* fix(ci): hold the MariaDB lane's sdk-java lock once per job (#14896)

Co-authored-by: Qwen-Coder <[email protected]>

* test(ci): pin the merged MariaDB step's order and SpotBugs witness

The single-step merge dropped guarantees that the two-step layout got for
free from step boundaries, and one witness became satisfiable by prose:

- spotbugs-gate matched `verify` anywhere in the step body, and the merged
  step's own comments explain the incident with the word 'verify' twice.
  A later narrowing to `clean test` would stay green. Match `clean verify`
  on command lines only.
- seed-before-write-back was a step-order invariant; both strings now live
  in one body with presence assertions only, and the write-back's warning
  arm swallows the ordering failure. Pin seed < install < verify <
  write-back.
- `working-directory:` applied to the whole step and could not be
  misplaced; its `cd` replacement is positional inside the body. Pin it
  between the two Maven runs.
- Drop the finder arm that matched the deleted `working-directory:`
  property: no step in either inspected job carries it any more.

---------

Co-authored-by: Qwen-Coder <[email protected]>
Co-authored-by: 易良 <[email protected]>
Co-authored-by: Shaojin Wen <[email protected]>

scripts/tests/hosted-process-ci.test.js

@@ -323,7 +323,7 @@ describe('Hosted real-process gates', () => {
     const mariadbRun = java.jobs['mysql-integration'].steps.find(
       (step) =>
         step.name ===
-        'Run Managed Agent tests, Checkstyle, and MySQL integration',
+        'Install and verify Managed Agent dependencies, tests, Checkstyle, and MySQL integration',
     );
     expect(mariadbRun.run).toContain('-Pmysql-integration');
     expect(mariadbRun.run).not.toContain('-Dit.test');
@@ -341,7 +341,12 @@ describe('Hosted real-process gates', () => {
       expect(profile(id), id).toMatch(skipsUnits);
     }
     expect(profile('mysql-integration')).not.toContain('maven-surefire-plugin');
-    expect(mariadbRun.run).not.toMatch(/-D(skipTests|maven\.test\.skip)/);
+    // The step stages the SDK with -DskipTests before the MySQL run; only
+    // the integration command itself must keep its tests.
+    const mariadbVerify = mariadbRun.run.slice(
+      mariadbRun.run.indexOf('-Pmysql-integration'),
+    );
+    expect(mariadbVerify).not.toMatch(/-D(skipTests|maven\.test\.skip)/);
     // Those checks read only the command lines, so each job also compares
     // the reports of its run with the source tree, right after its last
     // integration-test step.

scripts/tests/sdk-java-workflow.test.js

@@ -50,8 +50,8 @@ describe('SDK Java self-hosted workflow guards', () => {
       );
       const verify = steps.findIndex(
         (s) =>
-          (s['working-directory'] === 'packages/managed-agent-server' ||
-            s.run?.includes('-f packages/managed-agent-server/pom.xml')) &&
+          (s.run?.includes('-f packages/managed-agent-server/pom.xml') ||
+            s.run?.includes('cd packages/managed-agent-server')) &&
           s.run?.includes('clean verify'),
       );
       expect(install, name).toBeGreaterThan(-1);
@@ -282,7 +282,7 @@ describe('SDK Java self-hosted workflow guards', () => {
       "settings-path: '${{ runner.temp }}/setup-java-m2'",
     );
     // Two tail-anchored forms, no prefix match: mysql-integration appends
-    // a job-local -Dmaven.repo.local to two of its three MAVEN_ARGS
+    // a job-local -Dmaven.repo.local to its single Maven-bearing step
     // (pinned by the 'installs and verifies against one job-local Maven
     // repository' test), the EventTransport lanes to both of theirs; the
     // other jobs must carry no suffix at all — an appended -DskipTests
@@ -296,7 +296,7 @@ describe('SDK Java self-hosted workflow guards', () => {
     expect((exact?.length ?? 0) + (jobLocal?.length ?? 0)).toBe(
       {
         test: 6,
-        'mysql-integration': 2,
+        'mysql-integration': 1,
         'daemon-e2e': 1,
         'event-transport-broker': 2,
         'event-transport-multi-node': 2,
@@ -709,68 +709,70 @@ describe('SDK Java MariaDB lane on the ECS pool', () => {
     );
   });
 
-  it('holds the per-host sdk-java lock around every Maven run', () => {
-    // Derived from the parsed steps, not a string count: every Maven-bearing
-    // step must open the test job's per-host lock — the shared path pinned
-    // for the test job above is what makes the two jobs mutually exclusive
-    // on one ECS host — and wait on it with the pinned literal.
+  it('holds the per-host sdk-java lock once around both Maven runs', () => {
+    // Derived from the parsed steps, not a string count: the install and
+    // verify commands share one acquisition — the shared path pinned for the
+    // test job above is what makes the two jobs mutually exclusive on one ECS
+    // host — and wait on it with the pinned literal. Run 39353663670's second
+    // acquisition lost the queue between the two steps and exhausted the wait
+    // without starting Maven.
     const parsed = parse(workflow);
     const mavenSteps = parsed.jobs['mysql-integration'].steps.filter(
       (s) => typeof s.run === 'string' && s.run.includes('mvn '),
     );
     expect(mavenSteps.map((s) => s.name)).toEqual([
-      'Install Managed Agent dependencies',
-      'Run Managed Agent tests, Checkstyle, and MySQL integration',
+      'Install and verify Managed Agent dependencies, tests, Checkstyle, and MySQL integration',
     ]);
-    const lockWaits = mavenSteps.map((s) => {
-      expect(s.run, s.name).toContain(
-        'exec 9>"${HOME}/.cache/qwen-code-ci/sdk-java-tests.lock"',
-      );
-      const seconds = Number(s.run.match(/flock --wait (\d+) 9/)?.[1]);
-      // The fast tier owns the same lock and waits behind these two steps, so
-      // one acquisition budget applies to every lane; run 39229303984 died at
-      // the old 60-minute wait before Maven started.
-      expect(seconds, s.name).toBe(90 * 60);
-      return seconds;
-    });
-    // The ceiling must absorb one full lock wait per acquisition — this job
-    // takes the lock once per Maven step — plus the ~50 minutes of measured
-    // Maven work and ~30 minutes of uncapped setup. Derive it from the step
-    // count so another locked step cannot silently overdraw it.
+    const [step] = mavenSteps;
+    expect(step.run).toContain(
+      'exec 9>"${HOME}/.cache/qwen-code-ci/sdk-java-tests.lock"',
+    );
+    const seconds = Number(step.run.match(/flock --wait (\d+) 9/)?.[1]);
+    // The fast tier owns the same lock and waits behind this job, so one
+    // acquisition budget applies to every lane; run 39229303984 died at the
+    // old 60-minute wait before Maven started.
+    expect(seconds).toBe(90 * 60);
+    const lockWait = step.run.match(/flock --wait \d+ 9/)?.[0];
+    expect(lockWait).toBeDefined();
+    const lockAt = step.run.indexOf(lockWait);
+    const installAt = step.run.indexOf('-f packages/sdk-java/qwencode/pom.xml');
+    // The deleted `working-directory:` applied to the whole step; its `cd`
+    // replacement only applies from where it sits, so pin it between the two
+    // Maven runs: above the install it breaks the SDK `-f` path, below the
+    // verify it runs `clean verify` in the wrong module.
+    const cdAt = step.run.indexOf('cd packages/managed-agent-server');
+    const verifyAt = step.run.indexOf('-Pmysql-integration');
+    expect(lockAt).toBeGreaterThan(-1);
+    expect(installAt).toBeGreaterThan(lockAt);
+    expect(cdAt).toBeGreaterThan(installAt);
+    expect(verifyAt).toBeGreaterThan(cdAt);
+    // The ceiling must absorb one full lock wait plus the ~60 minutes of
+    // measured Maven work and ~30 minutes of uncapped setup.
     expect(
       parsed.jobs['mysql-integration']['timeout-minutes'] * 60,
-    ).toBeGreaterThanOrEqual(
-      lockWaits.reduce((total, seconds) => total + seconds, 0) +
-        50 * 60 +
-        30 * 60,
-    );
+    ).toBeGreaterThanOrEqual(seconds + 60 * 60 + 30 * 60);
     // The sibling test job shares the same per-host lock: its ceiling must
     // cover one full wait plus its own measured hold (the sdk-java suite plus
     // the managed-agent H2 suite, 925 s) and ~11 minutes of setup.
     expect(parsed.jobs.test['timeout-minutes'] * 60).toBeGreaterThanOrEqual(
-      lockWaits[0] + 925 + 11 * 60,
+      seconds + 925 + 11 * 60,
     );
   });
 
   it('installs and verifies against one job-local Maven repository', () => {
-    // The per-step lock releases at each step boundary, so the fixed
-    // 0.1.0-alpha release coordinates must not round-trip through the
-    // host-shared ~/.m2: the install step and the verify step resolve from
-    // the same job-local repo, seeded from the shared cache inside the lock.
+    // The fixed 0.1.0-alpha release coordinates must not resolve from the
+    // host-shared ~/.m2: the job-local repo is seeded from that cache, and
+    // both Maven runs resolve from the same copy.
     const steps = parse(workflow).jobs['mysql-integration'].steps;
-    const install = steps.find(
-      (s) => s.name === 'Install Managed Agent dependencies',
-    );
-    const verify = steps.find(
+    const step = steps.find(
       (s) =>
-        s.name === 'Run Managed Agent tests, Checkstyle, and MySQL integration',
+        s.name ===
+        'Install and verify Managed Agent dependencies, tests, Checkstyle, and MySQL integration',
     );
-    for (const s of [install, verify]) {
-      expect(s?.env?.MAVEN_ARGS, s?.name).toContain(
-        '-Dmaven.repo.local=${{ runner.temp }}/m2-repo',
-      );
-    }
-    expect(install.run).toContain(
+    expect(step?.env?.MAVEN_ARGS).toContain(
+      '-Dmaven.repo.local=${{ runner.temp }}/m2-repo',
+    );
+    expect(step.run).toContain(
       'cp -aln "${HOME}/.m2/repository" "${RUNNER_TEMP}/m2-repo"',
     );
     // The write-back closes the host-shared ~/.m2 loop — the seed above only
@@ -782,17 +784,29 @@ describe('SDK Java MariaDB lane on the ECS pool', () => {
     // warning arm keeps a copy failure from reddening the gate after mvn
     // has already passed, and pipefail keeps a failing left-hand tar from
     // being masked by the extractor's exit 0.
-    expect(verify.run).toContain('set -o pipefail');
-    expect(verify.run).toContain('mkdir -p "${HOME}/.m2/repository"');
-    expect(verify.run).toContain(
+    expect(step.run).toContain('set -o pipefail');
+    expect(step.run).toContain('mkdir -p "${HOME}/.m2/repository"');
+    expect(step.run).toContain(
       'tar -C "${RUNNER_TEMP}/m2-repo" --exclude=./com/alibaba/qwencode-sdk --exclude=./com/alibaba/qwen-managed-agent-server -cf - .',
     );
-    expect(verify.run).toContain(
+    expect(step.run).toContain(
       'tar -C "${HOME}/.m2/repository" --skip-old-files -xf -',
     );
-    expect(verify.run).toContain(
+    expect(step.run).toContain(
       '|| echo "::warning::Maven cache write-back failed; the gate result above is unaffected"',
     );
+    // One body replaced the two-step split, so only these positions still
+    // order the phases: the write-back runs against a directory the seed has
+    // not created yet if hoisted above it, and its warning arm swallows the
+    // failure instead of reddening the gate.
+    const seedAt = step.run.indexOf('cp -aln "${HOME}/.m2/repository"');
+    const installAt = step.run.indexOf('-f packages/sdk-java/qwencode/pom.xml');
+    const verifyAt = step.run.indexOf('-Pmysql-integration');
+    const writeBackAt = step.run.indexOf('tar -C "${RUNNER_TEMP}/m2-repo"');
+    expect(seedAt).toBeGreaterThan(-1);
+    expect(installAt).toBeGreaterThan(seedAt);
+    expect(verifyAt).toBeGreaterThan(installAt);
+    expect(writeBackAt).toBeGreaterThan(verifyAt);
   });
 });
 

scripts/tests/spotbugs-gate.test.js

@@ -46,10 +46,16 @@ describe('SpotBugs gate witness', () => {
     );
     const byName = (name) => steps.filter((step) => step.name === name);
 
-    // Hosted MySQL, MariaDB and both EventTransport gate jobs each stage the
-    // sibling modules before their own verify; the Hosted failover job
-    // stages them before packaging the jar its arms run.
-    const staging = byName('Install Managed Agent dependencies');
+    // Hosted MySQL, the MariaDB job's combined install-and-verify step and
+    // both EventTransport gate jobs each stage the sibling modules before
+    // their own verify; the Hosted failover job stages them before packaging
+    // the jar its arms run.
+    const mergedStep =
+      'Install and verify Managed Agent dependencies, tests, Checkstyle, and MySQL integration';
+    const staging = [
+      ...byName('Install Managed Agent dependencies'),
+      ...byName(mergedStep),
+    ];
     expect(staging.length).toBe(5);
     for (const step of staging) {
       expect(step.run).toContain('packages/sdk-java/qwencode/pom.xml');
@@ -57,14 +63,22 @@ describe('SpotBugs gate witness', () => {
     }
 
     const enforcing = [
-      ...byName('Run Managed Agent tests, Checkstyle, and MySQL integration'),
+      ...byName(mergedStep),
       ...byName('Verify Hosted Java, Spring and MySQL processes'),
       ...byName('Verify the EventTransport broker gates'),
       ...byName('Verify the EventTransport multi-node gates'),
     ];
     expect(enforcing.length).toBe(4);
     for (const step of enforcing) {
-      expect(step.run).toContain('verify');
+      // Match the command line only: the MariaDB job's merged step explains
+      // its own incident in comments that say 'verify', so a bare
+      // toContain('verify') would let prose witness the phase after a real
+      // narrowing to `clean test`.
+      const commands = step.run
+        .split('\n')
+        .filter((line) => !line.trimStart().startsWith('#'))
+        .join('\n');
+      expect(commands).toMatch(/\bclean verify\b/);
       expect(step.run).not.toContain('spotbugs.skip');
     }