Merge pull request #14939 from esengine/fx/notarize-staple-retry

fix(release): retry stapling when the connection to Apple drops

scripts/notarize-desktop.mjs

@@ -8,7 +8,8 @@ import { pathToFileURL } from "node:url";
 export function notarizeDesktop({ archive, target, kind, diagnosticsDir, env = process.env,
   run = (command, args) => spawnSync(command, args, {
     encoding: "utf8", stdio: ["ignore", "pipe", "inherit"], maxBuffer: 16 * 1024 * 1024,
-  }), warn = console.warn }) {
+  }), warn = console.warn, stapleAttempts = 4, stapleDelayMs = 20_000,
+  sleep = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms) }) {
   if (!["app", "dmg"].includes(kind)) throw new Error("Expected notarization kind app or dmg");
   for (const name of ["APPLE_API_KEY_PATH", "APPLE_API_KEY_ID", "APPLE_API_ISSUER_ID"]) {
     if (!env[name]) throw new Error(`Missing ${name}`);
@@ -55,8 +56,19 @@ export function notarizeDesktop({ archive, target, kind, diagnosticsDir, env = p
   if (submission.status !== 0 || submission.error || !id || status !== "Accepted") {
     throw new Error(`Notarization ${status ?? "unknown"} (${id ?? "no submission ID"}); see ${kind}-submission.json and the notary log`);
   }
-  checked("xcrun", ["stapler", "staple", target]);
-  checked("xcrun", ["stapler", "validate", target]);
+  // Stapling fetches the accepted ticket from Apple's CDN, so a dropped
+  // connection says nothing about the submission; only that pair is retried.
+  for (let attempt = 1; ; attempt++) {
+    try {
+      checked("xcrun", ["stapler", "staple", target]);
+      checked("xcrun", ["stapler", "validate", target]);
+      break;
+    } catch (error) {
+      if (attempt >= stapleAttempts) throw error;
+      warn(`${error.message} (attempt ${attempt} of ${stapleAttempts}); retrying in ${stapleDelayMs / 1000}s`);
+      sleep(stapleDelayMs);
+    }
+  }
   // Gatekeeper requires notarization; it is not a pre-submission signature test.
   checked("spctl", ["--assess", "--verbose=4", "--type", kind === "app" ? "exec" : "open",
     ...(kind === "dmg" ? ["--context", "context:primary-signature"] : []), target]);

scripts/notarize-desktop.test.mjs

@@ -13,11 +13,11 @@ const ok = (value = {}) => ({ status: 0, stdout: JSON.stringify(value) });
 function fixture(t, { kind = "app", status = "Accepted", submit, log, fail } = {}) {
   const diagnosticsDir = mkdtempSync(join(tmpdir(), "reasonix-notary-test-"));
   t.after(() => rmSync(diagnosticsDir, { recursive: true, force: true }));
-  const calls = [], warnings = [];
+  const calls = [], warnings = [], sleeps = [];
   const execute = () => notarizeDesktop({
     archive: kind === "app" ? "upload.zip" : "Reasonix.dmg",
     target: kind === "app" ? "Reasonix.app" : "Reasonix.dmg",
-    kind, diagnosticsDir, env, warn: (message) => warnings.push(message),
+    kind, diagnosticsDir, env, warn: (message) => warnings.push(message), sleep: (ms) => sleeps.push(ms),
     run: (command, args) => {
       calls.push([command, ...args]);
       if (fail?.(command, args)) return { status: 65, stdout: "" };
@@ -27,7 +27,7 @@ function fixture(t, { kind = "app", status = "Accepted", submit, log, fail } = {
     },
   });
   const report = (name) => JSON.parse(readFileSync(join(diagnosticsDir, `${kind}-${name}.json`), "utf8"));
-  return { execute, calls, warnings, report, diagnosticsDir };
+  return { execute, calls, warnings, sleeps, report, diagnosticsDir };
 }
 
 for (const kind of ["app", "dmg"]) {
@@ -118,7 +118,9 @@ for (const stage of ["codesign", "staple", "validate", "spctl"]) {
   test(`${stage} failure stops the pipeline`, (t) => {
     const f = fixture(t, { fail: (command, args) => command === stage || args[1] === stage });
     assert.throws(f.execute, /failed/);
-    assert.equal(f.calls.length, { codesign: 1, staple: 4, validate: 5, spctl: 6 }[stage]);
+    const stapling = stage === "staple" || stage === "validate";
+    assert.equal(f.calls.length, { codesign: 1, staple: 3 + 4, validate: 3 + 4 * 2, spctl: 6 }[stage]);
+    assert.equal(f.sleeps.length, stapling ? 3 : 0);
   });
 }
 
@@ -190,3 +192,14 @@ else fs.writeFileSync(args.at(-1), JSON.stringify({jobId: args[2], status: 'Inva
     }
   });
 }
+
+test("a dropped connection while stapling is retried and the pipeline completes", (t) => {
+  let dropped = 0;
+  const f = fixture(t, { fail: (command, args) => args[1] === "staple" && ++dropped < 3 });
+  f.execute();
+  assert.equal(dropped, 3);
+  assert.equal(f.sleeps.length, 2);
+  assert.equal(f.calls.filter((call) => call[2] === "staple").length, 3);
+  assert.deepEqual(f.calls.at(-1).slice(0, 2), ["spctl", "--assess"]);
+  assert.equal(f.calls.filter((call) => call[1] === "notarytool" && call[2] === "submit").length, 1);
+});