[build-tools] Send an Idempotency-Key header when creating artifact upload sessions

.changeset/artifact-upload-idempotency-key.md

@@ -0,0 +1,5 @@
+---
+'@expo/build-tools': patch
+---
+
+Send an `Idempotency-Key` header when creating device run session and sandbox artifact upload sessions, reuse it when an upload is retried, and stop retrying an upload that www rejects.

packages/build-tools/src/steps/utils/__tests__/agentDeviceArtifacts.test.ts

@@ -5,14 +5,20 @@ import timers from 'node:timers/promises';
 
 import { CustomBuildContext } from '../../../customBuildContext';
 import { Sentry } from '../../../sentry';
-import { uploadDeviceRunSessionArtifactAsync } from '../deviceRunSessionArtifacts';
+import {
+  DeviceRunSessionArtifactRejectedError,
+  uploadDeviceRunSessionArtifactAsync,
+} from '../deviceRunSessionArtifacts';
 import {
   listAgentDeviceArtifactsAsync,
   pollAgentDeviceArtifactsForUploadAsync,
   uploadAgentDeviceArtifactAsync,
 } from '../agentDeviceArtifacts';
 
-jest.mock('../deviceRunSessionArtifacts');
+jest.mock('../deviceRunSessionArtifacts', () => ({
+  ...jest.requireActual('../deviceRunSessionArtifacts'),
+  uploadDeviceRunSessionArtifactAsync: jest.fn(),
+}));
 jest.mock('../../../sentry');
 jest.mock('node-fetch');
 
@@ -163,6 +169,7 @@ describe(uploadAgentDeviceArtifactAsync, () => {
       deviceRunSessionId: 'drs-id',
       daemonUrl: 'http://127.0.0.1:1234',
       daemonToken: 'daemon-token',
+      idempotencyKey: 'idempotency-key',
       logger,
       artifact: {
         id: 'artifact-id',
@@ -182,6 +189,7 @@ describe(uploadAgentDeviceArtifactAsync, () => {
       kind: 'agent-device-test-report',
       size: data.length,
       openStream: expect.any(Function),
+      idempotencyKey: 'idempotency-key',
     });
   });
 
@@ -201,6 +209,7 @@ describe(uploadAgentDeviceArtifactAsync, () => {
       deviceRunSessionId: 'drs-id',
       daemonUrl: 'http://127.0.0.1:1234',
       daemonToken: 'daemon-token',
+      idempotencyKey: 'idempotency-key',
       logger,
       artifact: {
         id: 'artifact-id',
@@ -236,6 +245,7 @@ describe(uploadAgentDeviceArtifactAsync, () => {
       deviceRunSessionId: 'drs-id',
       daemonUrl: 'http://127.0.0.1:1234',
       daemonToken: 'daemon-token',
+      idempotencyKey: 'idempotency-key',
       logger,
       artifact: {
         id: 'artifact-id',
@@ -276,6 +286,7 @@ describe(uploadAgentDeviceArtifactAsync, () => {
       deviceRunSessionId: 'drs-id',
       daemonUrl: 'http://127.0.0.1:1234',
       daemonToken: 'daemon-token',
+      idempotencyKey: 'idempotency-key',
       logger,
       artifact: { id: 'artifact-id', artifactType: 'screenshot', filename: 'shot.png' },
     });
@@ -305,6 +316,7 @@ describe(uploadAgentDeviceArtifactAsync, () => {
       deviceRunSessionId: 'drs-id',
       daemonUrl: 'http://127.0.0.1:1234',
       daemonToken: 'daemon-token',
+      idempotencyKey: 'idempotency-key',
       logger: createLoggerMock(),
       artifact: { id: 'artifact-id', artifactType: 'screen-recording', filename: 'session.mp4' },
     });
@@ -627,7 +639,7 @@ describe(pollAgentDeviceArtifactsForUploadAsync, () => {
     expect(Sentry.capture).toHaveBeenCalledTimes(2);
   });
 
-  it('retries an artifact after a failed upload', async () => {
+  it('retries an artifact with its first idempotency key and gives each artifact its own key', async () => {
     const data = Buffer.from('artifact-data');
     const logger = createLoggerMock();
     const ctx = {} as unknown as CustomBuildContext;
@@ -635,18 +647,24 @@ describe(pollAgentDeviceArtifactsForUploadAsync, () => {
       id: 'artifact-id',
       filename: 'report.json',
     };
-    const listResponse = () => new Response(JSON.stringify({ artifacts: [artifact] }));
+    const otherArtifact = {
+      id: 'other-artifact-id',
+      filename: 'other.json',
+    };
+    const listResponse = () =>
+      new Response(JSON.stringify({ artifacts: [artifact, otherArtifact] }));
 
     jest
       .mocked(fetch)
       .mockResolvedValueOnce(listResponse())
       .mockResolvedValueOnce(new Response(Readable.from([data])))
+      .mockResolvedValueOnce(new Response(Readable.from([data])))
       .mockResolvedValueOnce(listResponse())
       .mockResolvedValueOnce(new Response(Readable.from([data])));
     jest
       .mocked(uploadDeviceRunSessionArtifactAsync)
       .mockRejectedValueOnce(new Error('upload failed'))
-      .mockImplementationOnce(async (_ctx, { openStream }) => {
+      .mockImplementation(async (_ctx, { openStream }) => {
         await readStreamAsync(openStream());
       });
 
@@ -658,12 +676,53 @@ describe(pollAgentDeviceArtifactsForUploadAsync, () => {
     });
 
     await flushPromisesAsync();
-    expect(jest.mocked(uploadDeviceRunSessionArtifactAsync)).toHaveBeenCalledTimes(1);
+    expect(jest.mocked(uploadDeviceRunSessionArtifactAsync)).toHaveBeenCalledTimes(2);
 
     await jest.advanceTimersByTimeAsync(5_000);
     await flushPromisesAsync();
 
-    expect(jest.mocked(uploadDeviceRunSessionArtifactAsync)).toHaveBeenCalledTimes(2);
+    const calls = jest
+      .mocked(uploadDeviceRunSessionArtifactAsync)
+      .mock.calls.map(([, { artifactId, idempotencyKey }]) => ({ artifactId, idempotencyKey }));
+    expect(calls.map(call => call.artifactId)).toEqual([
+      'artifact-id',
+      'other-artifact-id',
+      'artifact-id',
+    ]);
+    expect(calls[0].idempotencyKey).toEqual(expect.any(String));
+    expect(calls[2].idempotencyKey).toBe(calls[0].idempotencyKey);
+    expect(calls[1].idempotencyKey).not.toBe(calls[0].idempotencyKey);
+  });
+
+  it('does not retry an artifact that www rejects', async () => {
+    const data = Buffer.from('artifact-data');
+    const logger = createLoggerMock();
+    const ctx = {} as unknown as CustomBuildContext;
+    const listResponse = () =>
+      new Response(JSON.stringify({ artifacts: [{ id: 'artifact-id', filename: 'report.json' }] }));
+
+    jest
+      .mocked(fetch)
+      .mockResolvedValueOnce(listResponse())
+      .mockResolvedValueOnce(new Response(Readable.from([data])))
+      .mockResolvedValueOnce(listResponse());
+    jest
+      .mocked(uploadDeviceRunSessionArtifactAsync)
+      .mockRejectedValueOnce(new DeviceRunSessionArtifactRejectedError('Rejected.'));
+
+    void pollAgentDeviceArtifactsForUploadAsync(ctx, {
+      deviceRunSessionId: 'drs-id',
+      daemonUrl: 'http://127.0.0.1:1234',
+      daemonToken: 'daemon-token',
+      logger,
+    });
+
+    await flushPromisesAsync();
+    await jest.advanceTimersByTimeAsync(5_000);
+    await flushPromisesAsync();
+
+    expect(jest.mocked(fetch)).toHaveBeenCalledTimes(3);
+    expect(jest.mocked(uploadDeviceRunSessionArtifactAsync)).toHaveBeenCalledTimes(1);
   });
 
   it('stops polling when the daemon does not expose artifact inventory', async () => {

packages/build-tools/src/steps/utils/__tests__/deviceRunSessionArtifacts.test.ts

@@ -1,9 +1,13 @@
+import { SystemError } from '@expo/eas-build-job';
 import fetch from 'node-fetch';
 import { Readable } from 'node:stream';
 
 import { CustomBuildContext } from '../../../customBuildContext';
 import { turtleFetch } from '../../../utils/turtleFetch';
-import { uploadDeviceRunSessionArtifactAsync } from '../deviceRunSessionArtifacts';
+import {
+  DeviceRunSessionArtifactRejectedError,
+  uploadDeviceRunSessionArtifactAsync,
+} from '../deviceRunSessionArtifacts';
 import { DEVICE_RUN_SESSION_TOKEN_HEADER } from '../deviceRunSessionWorkerApi';
 
 jest.mock('node-fetch');
@@ -13,13 +17,26 @@ const { Response } = jest.requireActual('node-fetch') as typeof import('node-fet
 
 const TOKEN = 'device-run-session-token-value';
 
-function jsonResponse(body: unknown): InstanceType<typeof Response> {
+function jsonResponse(body: unknown, status = 200): InstanceType<typeof Response> {
   return new Response(JSON.stringify(body), {
-    status: 200,
+    status,
     headers: { 'Content-Type': 'application/json' },
   });
 }
 
+const uploadSessionResponse = () =>
+  jsonResponse({ data: { uploadSession: { url: 'https://uploads.expo.test/a', headers: {} } } });
+
+const artifact = {
+  deviceRunSessionId: 'drs-id',
+  artifactId: 'artifact-id',
+  name: 'Screenshot',
+  filename: 'screenshot.png',
+  kind: 'screenshot',
+  size: 0,
+  openStream: () => Readable.from([]),
+};
+
 function createCtx(): CustomBuildContext {
   return {
     env: { EAS_DEVICE_RUN_SESSION_TOKEN: TOKEN },
@@ -60,13 +77,17 @@ describe(uploadDeviceRunSessionArtifactAsync, () => {
       metadata: { firstFrameRecordAt: 'test-time' },
       size: reportedSize,
       openStream: () => stream,
+      idempotencyKey: 'idempotency-key',
     });
 
     expect(jest.mocked(turtleFetch)).toHaveBeenCalledWith(
       'https://api.expo.test/v2/device-run-session-worker/artifact-upload-sessions',
       'POST',
       expect.objectContaining({
-        headers: { [DEVICE_RUN_SESSION_TOKEN_HEADER]: TOKEN },
+        headers: {
+          [DEVICE_RUN_SESSION_TOKEN_HEADER]: TOKEN,
+          'Idempotency-Key': 'idempotency-key',
+        },
         json: {
           name: 'Artifact report.json (artifact-id)',
           filename: 'report.json',
@@ -116,4 +137,66 @@ describe(uploadDeviceRunSessionArtifactAsync, () => {
     expect(allocationSignal).toBeInstanceOf(AbortSignal);
     expect(allocationSignal?.aborted).toBe(true);
   });
+
+  it('sends a new idempotency key for each upload without one', async () => {
+    jest
+      .mocked(turtleFetch)
+      .mockResolvedValueOnce(uploadSessionResponse())
+      .mockResolvedValueOnce(uploadSessionResponse());
+    jest.mocked(fetch).mockResolvedValue(new Response('', { status: 200 }));
+
+    await uploadDeviceRunSessionArtifactAsync(createCtx(), artifact);
+    await uploadDeviceRunSessionArtifactAsync(createCtx(), artifact);
+
+    const [first, second] = jest
+      .mocked(turtleFetch)
+      .mock.calls.map(
+        ([, , options]) => (options.headers as Record<string, string>)['Idempotency-Key']
+      );
+    expect(first).toEqual(expect.any(String));
+    expect(second).toEqual(expect.any(String));
+    expect(second).not.toBe(first);
+  });
+
+  it('reports an artifact name conflict as a rejected upload session', async () => {
+    jest.mocked(turtleFetch).mockResolvedValueOnce(
+      jsonResponse(
+        {
+          errors: [
+            {
+              code: 'ARTIFACT_NAME_CONFLICT',
+              message:
+                'Device run session already has an artifact named "Screenshot". Use a different name for this artifact.',
+            },
+          ],
+        },
+        409
+      )
+    );
+
+    await expect(uploadDeviceRunSessionArtifactAsync(createCtx(), artifact)).rejects.toBeInstanceOf(
+      DeviceRunSessionArtifactRejectedError
+    );
+    expect(fetch).not.toHaveBeenCalled();
+  });
+
+  it.each([
+    { status: 409, code: 'OTHER_CONFLICT' },
+    { status: 400, code: 'VALIDATION_ERROR' },
+    { status: 500, code: 'UNEXPECTED_ERROR' },
+  ])(
+    'does not report HTTP $status $code as a rejected upload session',
+    async ({ status, code }) => {
+      jest
+        .mocked(turtleFetch)
+        .mockResolvedValueOnce(jsonResponse({ errors: [{ code, message: 'Failed.' }] }, status));
+
+      const error = await uploadDeviceRunSessionArtifactAsync(createCtx(), artifact).catch(
+        (err: unknown) => err
+      );
+
+      expect(error).toBeInstanceOf(SystemError);
+      expect(error).not.toBeInstanceOf(DeviceRunSessionArtifactRejectedError);
+    }
+  );
 });

packages/build-tools/src/steps/utils/__tests__/deviceRunSessionScreenshots.test.ts

@@ -7,14 +7,20 @@ import path from 'node:path';
 
 import type { CustomBuildContext } from '../../../customBuildContext';
 import { Sentry } from '../../../sentry';
-import { uploadDeviceRunSessionArtifactAsync } from '../deviceRunSessionArtifacts';
+import {
+  DeviceRunSessionArtifactRejectedError,
+  uploadDeviceRunSessionArtifactAsync,
+} from '../deviceRunSessionArtifacts';
 import { createScreenshotMetadataAsync } from '../deviceRunSessionScreenshotMetadata';
 import {
   startDeviceRunSessionScreenshotsAsync,
   uploadDeviceRunSessionScreenshotsAsync,
 } from '../deviceRunSessionScreenshots';
 
-jest.mock('../deviceRunSessionArtifacts');
+jest.mock('../deviceRunSessionArtifacts', () => ({
+  ...jest.requireActual('../deviceRunSessionArtifacts'),
+  uploadDeviceRunSessionArtifactAsync: jest.fn(),
+}));
 jest.mock('../deviceRunSessionScreenshotMetadata');
 jest.mock('../../../sentry');
 const ctx = {} as CustomBuildContext;
@@ -22,7 +28,7 @@ const logger = { info: jest.fn(), warn: jest.fn() } as unknown as bunyan;
 const options = { deviceRunSessionId: 'session-id', logger };
 let directory: string;
 let uploaded: Buffer[];
-let failedUploads: Map<string, { attempts: number; lastError: Error }>;
+let failedUploads: Map<string, { attempts: number; lastError: Error; idempotencyKey: string }>;
 
 beforeEach(async () => {
   jest.clearAllMocks();
@@ -91,7 +97,11 @@ it('uploads captures that never failed first, oldest first, before retrying fail
   for (const name of [newer, failed, older]) {
     await writeFile(path.join(directory, name), name);
   }
-  failedUploads.set(failed, { attempts: 1, lastError: new Error('stalled') });
+  failedUploads.set(failed, {
+    attempts: 1,
+    lastError: new Error('stalled'),
+    idempotencyKey: 'failed-key',
+  });
   await flush();
   expect(
     jest
@@ -124,6 +134,41 @@ it('keeps failed uploads and retries the same artifact ID, logging the failed at
   expect(Sentry.capture).not.toHaveBeenCalled();
 });
 
+it('retries a screenshot with its first idempotency key and gives each screenshot its own key', async () => {
+  const first = 'screenshot-2026-09-24T08-45-00-000Z-aaaaaaaaaaaa.png';
+  const second = 'screenshot-2026-09-24T08-45-30-000Z-bbbbbbbbbbbb.png';
+  await writeFile(path.join(directory, first), 'png');
+  await writeFile(path.join(directory, second), 'png');
+  jest.mocked(uploadDeviceRunSessionArtifactAsync).mockRejectedValueOnce(new Error('offline'));
+  await flush();
+  await flush();
+  const calls = jest
+    .mocked(uploadDeviceRunSessionArtifactAsync)
+    .mock.calls.map(([, { artifactId, idempotencyKey }]) => ({ artifactId, idempotencyKey }));
+  expect(calls.map(call => call.artifactId)).toEqual(
+    [first, second, first].map(name => name.slice(0, -4))
+  );
+  expect(calls[0].idempotencyKey).toEqual(expect.any(String));
+  expect(calls[2].idempotencyKey).toBe(calls[0].idempotencyKey);
+  expect(calls[1].idempotencyKey).not.toBe(calls[0].idempotencyKey);
+});
+
+it('keeps a screenshot that www rejects without retrying it', async () => {
+  const filename = 'screenshot-2026-09-24T08-45-59-123Z-ffffffffffff.png';
+  await writeFile(path.join(directory, filename), 'png');
+  jest
+    .mocked(uploadDeviceRunSessionArtifactAsync)
+    .mockRejectedValueOnce(new DeviceRunSessionArtifactRejectedError('Rejected.'));
+  await flush();
+  await flush();
+  expect(uploadDeviceRunSessionArtifactAsync).toHaveBeenCalledTimes(1);
+  expect(await readdir(directory)).toEqual([filename]);
+  expect(logger.warn).toHaveBeenCalledWith(
+    expect.objectContaining({ attempt: 1 }),
+    'www rejected the preview screenshot upload. Keeping it without retrying.'
+  );
+});
+
 it('uploads atomic captures immediately, including a capture arriving during another upload', async () => {
   jest.useFakeTimers();
   const collector = await startDeviceRunSessionScreenshotsAsync(ctx, options);
@@ -244,6 +289,7 @@ it('warns for the first three attempts and every tenth after, without reporting
   expect(failedUploads.get(filename)).toEqual({
     attempts: 12,
     lastError: expect.objectContaining({ message: 'offline' }),
+    idempotencyKey: expect.any(String),
   });
   expect(Sentry.capture).not.toHaveBeenCalled();
 });

packages/build-tools/src/steps/utils/__tests__/deviceRunSessionScreenshotsHttp.test.ts

@@ -210,7 +210,10 @@ it('keeps the signed upload URL out of the error when the upload connection drop
   handle = request => request.socket.destroy();
   const filename = 'screenshot-2026-09-24T08-45-59-123Z-a1b2c3d4e5f6.png';
   await writeFile(path.join(directory, filename), 'png');
-  const failedUploads = new Map<string, { attempts: number; lastError: Error }>();
+  const failedUploads = new Map<
+    string,
+    { attempts: number; lastError: Error; idempotencyKey: string }
+  >();
   const logger = { info: jest.fn(), warn: jest.fn() } as unknown as bunyan;
   await uploadDeviceRunSessionScreenshotsAsync(ctx, {
     directory,

packages/build-tools/src/steps/utils/__tests__/deviceRunSessionWorkerApi.test.ts

@@ -145,9 +145,11 @@ describe(DeviceRunSessionWorkerApi, () => {
           kind: 'screenshot',
           size: 12,
           metadata: { __eas_type: 'screenshot' },
+          idempotencyKey: 'idempotency-key',
         }),
       method: 'POST',
       route: 'artifact-upload-sessions',
+      headers: { 'Idempotency-Key': 'idempotency-key' },
       json: {
         name: 'Screenshot',
         filename: 'screenshot.png',
@@ -228,7 +230,7 @@ describe(DeviceRunSessionWorkerApi, () => {
     },
   ])(
     '$name calls $method /v2/device-run-session-worker/$route with the session token header',
-    async ({ call, method, route, json, data, expected }) => {
+    async ({ call, method, route, headers, json, data, expected }) => {
       jest.mocked(turtleFetch).mockResolvedValueOnce(jsonResponse({ data }));
 
       await expect(call(createApi())).resolves.toEqual(expected);
@@ -237,7 +239,7 @@ describe(DeviceRunSessionWorkerApi, () => {
       const [url, calledMethod, options] = jest.mocked(turtleFetch).mock.calls[0];
       expect(url).toBe(`https://api.expo.test/v2/device-run-session-worker/${route}`);
       expect(calledMethod).toBe(method);
-      expect(options.headers).toEqual({ [DEVICE_RUN_SESSION_TOKEN_HEADER]: TOKEN });
+      expect(options.headers).toEqual({ ...headers, [DEVICE_RUN_SESSION_TOKEN_HEADER]: TOKEN });
       expect(options.json).toEqual(json);
       expect(options).toEqual(expect.objectContaining({ retries: 0, shouldThrowOnNotOk: false }));
     }

packages/build-tools/src/steps/utils/__tests__/deviceRunSessionWorkerApiCallers.test.ts

@@ -330,6 +330,10 @@ describe('with a device run session token', () => {
     });
 
     expectWorkerApiCall('artifact-upload-sessions', 'POST', {
+      headers: {
+        [DEVICE_RUN_SESSION_TOKEN_HEADER]: TOKEN,
+        'Idempotency-Key': expect.any(String),
+      },
       json: {
         name: 'Report',
         filename: 'report.json',

packages/build-tools/src/steps/utils/__tests__/sandboxArtifacts.test.ts

@@ -23,7 +23,7 @@ import { setTimeout as delay } from 'node:timers/promises';
 import { inspect } from 'node:util';
 
 import { SandboxArtifactUploadManager, startSandboxArtifactUploadAsync } from '../sandboxArtifacts';
-import { type SandboxWorkerApi } from '../sandboxWorkerApi';
+import { SandboxWorkerApi } from '../sandboxWorkerApi';
 import { WorkerApiError } from '../workerApi';
 
 const { AbortError, FetchError, Response } = jest.requireActual(
@@ -46,7 +46,7 @@ describe('sandbox artifact uploads', () => {
     createArtifactUploadSessionAsync: jest.Mock;
     finalizeArtifactAsync: jest.Mock;
   };
-  let logger: { info: jest.Mock; error: jest.Mock };
+  let logger: { info: jest.Mock; warn: jest.Mock; error: jest.Mock };
   let uploadArtifactAsync: (options: {
     filePath: string;
     name: string;
@@ -62,7 +62,7 @@ describe('sandbox artifact uploads', () => {
       createArtifactUploadSessionAsync: jest.fn(),
       finalizeArtifactAsync: jest.fn(),
     };
-    logger = { info: jest.fn(), error: jest.fn() };
+    logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() };
     uploadArtifactAsync = async options =>
       await startSandboxArtifactUploadAsync({
         workerApi: workerApi as unknown as SandboxWorkerApi,
@@ -218,13 +218,28 @@ describe('sandbox artifact uploads', () => {
   });
 
   it.each([
-    { status: 400, code: 'VALIDATION_ERROR' },
-    { status: 403, code: 'UNAUTHORIZED' },
+    {
+      status: 409,
+      code: 'ARTIFACT_NAME_CONFLICT',
+      apiMessage:
+        'Sandbox already has an artifact named "Crash log". Use a different name for this artifact.',
+    },
+    {
+      status: 400,
+      code: 'VALIDATION_ERROR',
+      apiMessage:
+        'Sandbox is stopped, so it cannot upload artifacts. Upload artifacts while the sandbox is running.',
+    },
+    {
+      status: 403,
+      code: 'UNAUTHORIZED',
+      apiMessage:
+        'You are not authorized to upload artifacts for this sandbox. Only the user who created the sandbox can upload its artifacts.',
+    },
   ])(
     'rejects a $code upload session error as a bad request with the API message',
-    async ({ status, code }) => {
-      const message =
-        'POST /v2/sandbox-worker/artifact-upload-sessions failed with status 400: Sandbox already has an artifact named "Crash log". Use a different name for this artifact.';
+    async ({ status, code, apiMessage }) => {
+      const message = `POST /v2/sandbox-worker/artifact-upload-sessions failed with status ${status}: ${apiMessage}`;
       workerApi.createArtifactUploadSessionAsync.mockRejectedValueOnce(
         new WorkerApiError(message, status, undefined, code)
       );
@@ -246,6 +261,7 @@ describe('sandbox artifact uploads', () => {
   it.each([
     { status: 500, code: 'UNEXPECTED_ERROR' },
     { status: 401, code: 'AUTHENTICATION_ERROR' },
+    { status: 409, code: 'OTHER_CONFLICT' },
   ])('rethrows a $code upload session error unchanged', async ({ status, code }) => {
     const error = new WorkerApiError('Request failed.', status, undefined, code);
     workerApi.createArtifactUploadSessionAsync.mockRejectedValueOnce(error);
@@ -387,6 +403,79 @@ describe('sandbox artifact uploads', () => {
     await manager.stoppedPromise;
   });
 
+  describe('with the sandbox worker API', () => {
+    const uploadSessionsUrl = 'https://api.expo.test/v2/sandbox-worker/artifact-upload-sessions';
+
+    afterEach(() => {
+      jest.useRealTimers();
+    });
+
+    it('retries a failed upload session request with the same idempotency key', async () => {
+      jest.useFakeTimers({ doNotFake: ['nextTick', 'setImmediate', 'queueMicrotask'] });
+      jest
+        .mocked(fetch)
+        .mockResolvedValueOnce(new Response('unavailable', { status: 503 }))
+        .mockResolvedValueOnce(jsonResponse({ data: createdSession }))
+        .mockResolvedValueOnce(new Response('', { status: 200 }))
+        .mockResolvedValueOnce(
+          jsonResponse({ data: { id: ARTIFACT_ID, downloadUrl: DOWNLOAD_URL } })
+        );
+
+      const uploadPromise = uploadWithWorkerApiAsync();
+      while (uploadSessionRequests().length === 0) {
+        await new Promise(resolve => setImmediate(resolve));
+      }
+      await jest.advanceTimersByTimeAsync(1000);
+      const upload = await uploadPromise;
+
+      await expect(upload.completed).resolves.toEqual({
+        status: 'uploaded',
+        downloadUrl: DOWNLOAD_URL,
+      });
+      const keys = uploadSessionRequests().map(
+        ([, init]) => (init?.headers as Record<string, string>)['Idempotency-Key']
+      );
+      expect(keys).toEqual([expect.any(String), keys[0]]);
+    });
+
+    it('does not retry an artifact name conflict', async () => {
+      const apiMessage =
+        'Sandbox already has an artifact named "Crash log". Use a different name for this artifact.';
+      jest
+        .mocked(fetch)
+        .mockResolvedValueOnce(
+          jsonResponse({ errors: [{ code: 'ARTIFACT_NAME_CONFLICT', message: apiMessage }] }, 409)
+        );
+
+      await expect(uploadWithWorkerApiAsync()).rejects.toMatchObject({
+        code: SandboxDaemonErrorCode.BAD_REQUEST,
+        message: `POST /v2/sandbox-worker/artifact-upload-sessions failed with status 409: ${apiMessage}`,
+      });
+      expect(fetch).toHaveBeenCalledTimes(1);
+    });
+
+    async function uploadWithWorkerApiAsync(): ReturnType<typeof startSandboxArtifactUploadAsync> {
+      return await startSandboxArtifactUploadAsync({
+        workerApi: new SandboxWorkerApi('https://api.expo.test/v2/', 'sandbox-worker-token'),
+        logger: logger as unknown as bunyan,
+        filePath,
+        name: 'Crash log',
+        signal: new AbortController().signal,
+      });
+    }
+
+    function uploadSessionRequests(): Parameters<typeof fetch>[] {
+      return jest.mocked(fetch).mock.calls.filter(([url]) => url === uploadSessionsUrl);
+    }
+
+    function jsonResponse(body: unknown, status = 200): InstanceType<typeof Response> {
+      return new Response(JSON.stringify(body), {
+        status,
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+  });
+
   function createManager(signal: AbortSignal): SandboxArtifactUploadManager {
     return new SandboxArtifactUploadManager({
       workerApi: workerApi as unknown as SandboxWorkerApi,

packages/build-tools/src/steps/utils/__tests__/sandboxWorkerApi.test.ts

@@ -85,6 +85,7 @@ describe(SandboxWorkerApi, () => {
       call: (api: SandboxWorkerApi) =>
         api.createArtifactUploadSessionAsync({ name: 'Crash log', filename: 'crash.log', size: 5 }),
       route: 'artifact-upload-sessions',
+      headers: { 'Idempotency-Key': expect.any(String) },
       json: { name: 'Crash log', filename: 'crash.log', size: 5 },
       data: {
         artifact: { id: ARTIFACT_ID },
@@ -105,7 +106,7 @@ describe(SandboxWorkerApi, () => {
     },
   ])(
     '$name calls POST /v2/sandbox-worker/$route with the sandbox worker token header',
-    async ({ call, route, json, data, expected }) => {
+    async ({ call, route, headers, json, data, expected }) => {
       jest.mocked(turtleFetch).mockResolvedValueOnce(jsonResponse({ data }));
 
       await expect(call(new SandboxWorkerApi(BASE_URL, TOKEN))).resolves.toEqual(expected);
@@ -113,11 +114,35 @@ describe(SandboxWorkerApi, () => {
       const [url, method, options] = jest.mocked(turtleFetch).mock.calls[0];
       expect(url).toBe(`https://api.expo.test/v2/sandbox-worker/${route}`);
       expect(method).toBe('POST');
-      expect(options.headers).toEqual({ 'expo-sandbox-worker-token': TOKEN });
+      expect(options.headers).toEqual({ ...headers, 'expo-sandbox-worker-token': TOKEN });
       expect(options.json).toEqual(json);
     }
   );
 
+  it('sends a new idempotency key for each artifact upload session', async () => {
+    jest.mocked(turtleFetch).mockImplementation(async () =>
+      jsonResponse({
+        data: {
+          artifact: { id: ARTIFACT_ID },
+          uploadSession: { url: 'https://uploads.expo.test/a', headers: {} },
+        },
+      })
+    );
+    const api = new SandboxWorkerApi(BASE_URL, TOKEN);
+
+    await api.createArtifactUploadSessionAsync({ name: 'A', filename: 'a.log', size: 1 });
+    await api.createArtifactUploadSessionAsync({ name: 'B', filename: 'b.log', size: 1 });
+
+    const [first, second] = jest
+      .mocked(turtleFetch)
+      .mock.calls.map(
+        ([, , options]) => (options.headers as Record<string, string>)['Idempotency-Key']
+      );
+    expect(first).toEqual(expect.any(String));
+    expect(second).toEqual(expect.any(String));
+    expect(second).not.toBe(first);
+  });
+
   it('escapes the artifact ID in the finalize route', async () => {
     jest
       .mocked(turtleFetch)

packages/build-tools/src/steps/utils/__tests__/workerApi.test.ts

@@ -192,6 +192,29 @@ describe('worker API retries', () => {
     });
   });
 
+  it('sends the same idempotency key on every attempt', async () => {
+    jest
+      .mocked(turtleFetch)
+      .mockRejectedValueOnce(new Error('socket hang up'))
+      .mockResolvedValueOnce(new Response('unavailable', { status: 503 }))
+      .mockResolvedValueOnce(jsonResponse({ data: { id: 'thing-id' } }));
+    const assertion = expect(
+      new WorkerApi(BASE_URL, AUTH).requestAsync('POST', 'things', ThingSchema, {
+        json: {},
+        idempotencyKey: 'idempotency-key',
+        retries: 3,
+      })
+    ).resolves.toEqual({ id: 'thing-id' });
+    await jest.runAllTimersAsync();
+    await assertion;
+    expect(jest.mocked(turtleFetch).mock.calls.map(([, , options]) => options.headers)).toEqual(
+      Array.from({ length: 3 }, () => ({
+        'expo-test-worker-token': 'test-worker-token-value',
+        'Idempotency-Key': 'idempotency-key',
+      }))
+    );
+  });
+
   it('recovers from a network failure', async () => {
     jest
       .mocked(turtleFetch)

packages/build-tools/src/steps/utils/agentDeviceArtifacts.ts

@@ -1,5 +1,6 @@
 import { SystemError, formatBytes } from '@expo/eas-build-job';
 import { type bunyan } from '@expo/logger';
+import { randomUUID } from 'node:crypto';
 import { createReadStream, createWriteStream } from 'node:fs';
 import { mkdtemp, rm, stat } from 'node:fs/promises';
 import fetch from 'node-fetch';
@@ -11,7 +12,10 @@ import { z } from 'zod';
 
 import { CustomBuildContext } from '../../customBuildContext';
 import { Sentry } from '../../sentry';
-import { uploadDeviceRunSessionArtifactAsync } from './deviceRunSessionArtifacts';
+import {
+  DeviceRunSessionArtifactRejectedError,
+  uploadDeviceRunSessionArtifactAsync,
+} from './deviceRunSessionArtifacts';
 import { createScreenshotMetadataAsync } from './deviceRunSessionScreenshotMetadata';
 
 const AGENT_DEVICE_ARTIFACT_UPLOAD_POLL_INTERVAL_MS = 5_000;
@@ -58,7 +62,8 @@ export async function pollAgentDeviceArtifactsForUploadAsync(
   }
 ): Promise<void> {
   logger.info('Started polling agent-device daemon for artifacts.');
-  const uploadedArtifactIds = new Set<string>();
+  const finishedArtifactIds = new Set<string>();
+  const idempotencyKeys = new Map<string, string>();
   let listArtifactsErrorCount = 0;
 
   // Ending normal polling must not cancel a transfer that the session already produced.
@@ -92,23 +97,29 @@ export async function pollAgentDeviceArtifactsForUploadAsync(
         });
         listArtifactsErrorCount = 0;
         for (const artifact of artifacts) {
-          if (uploadedArtifactIds.has(artifact.id)) {
+          if (finishedArtifactIds.has(artifact.id)) {
             continue;
           }
+          const idempotencyKey = idempotencyKeys.get(artifact.id) ?? randomUUID();
+          idempotencyKeys.set(artifact.id, idempotencyKey);
           try {
             await uploadAgentDeviceArtifactAsync(ctx, {
               deviceRunSessionId,
               daemonUrl,
               daemonToken,
               artifact,
+              idempotencyKey,
               logger,
               ...(transferSignal ? { signal: transferSignal } : {}),
             });
-            uploadedArtifactIds.add(artifact.id);
+            finishedArtifactIds.add(artifact.id);
           } catch (err) {
             if (transferSignal?.aborted) {
               return;
             }
+            if (err instanceof DeviceRunSessionArtifactRejectedError) {
+              finishedArtifactIds.add(artifact.id);
+            }
             const error = err instanceof Error ? err : new Error(String(err));
             Sentry.capture('Could not upload agent-device remote session artifact', error);
             logger.warn({ err: error }, 'Could not upload agent-device remote session artifact.');
@@ -192,6 +203,7 @@ export async function uploadAgentDeviceArtifactAsync(
     daemonUrl,
     daemonToken,
     artifact,
+    idempotencyKey,
     logger,
     signal,
   }: {
@@ -199,6 +211,7 @@ export async function uploadAgentDeviceArtifactAsync(
     daemonUrl: string;
     daemonToken: string;
     artifact: AgentDeviceArtifact;
+    idempotencyKey: string;
     logger: bunyan;
     signal?: AbortSignal;
   }
@@ -237,6 +250,7 @@ export async function uploadAgentDeviceArtifactAsync(
       ...(metadata ? { metadata } : {}),
       size,
       openStream: () => createReadStream(temporaryArtifactPath),
+      idempotencyKey,
       ...(signal ? { signal } : {}),
     });
   } finally {

packages/build-tools/src/steps/utils/deviceRunSessionArtifacts.ts

@@ -1,4 +1,5 @@
 import { SystemError } from '@expo/eas-build-job';
+import { randomUUID } from 'node:crypto';
 import type { Readable } from 'node:stream';
 
 import { CustomBuildContext } from '../../customBuildContext';
@@ -10,6 +11,7 @@ import {
 } from '../../storage/uploadWithSignedUrl';
 import { withDeviceRunSessionTimeoutAsync } from './deviceRunSessionTimeout';
 import { requireDeviceRunSessionWorkerApi } from './deviceRunSessionWorkerApi';
+import { WorkerApiError } from './workerApi';
 
 export type ArtifactMetadata = {
   deviceRunSessionId: string;
@@ -27,12 +29,19 @@ export async function uploadDeviceRunSessionArtifactAsync(
     openStream: () => Readable;
     signal?: AbortSignal;
     attempts?: number;
+    idempotencyKey?: string;
   }
 ): Promise<void> {
+  const idempotencyKey = options.idempotencyKey ?? randomUUID();
   try {
     const uploadSession = await withDeviceRunSessionTimeoutAsync(
       { name: 'Artifact upload session creation', timeoutMs: 15_000, signal: options.signal },
-      async signal => await createDeviceRunSessionArtifactUploadSessionAsync(ctx, options, signal)
+      async signal =>
+        await createDeviceRunSessionArtifactUploadSessionAsync(
+          ctx,
+          { ...options, idempotencyKey },
+          signal
+        )
     );
     options.signal?.throwIfAborted();
     await uploadWithSignedUrl(
@@ -47,6 +56,8 @@ export async function uploadDeviceRunSessionArtifactAsync(
   }
 }
 
+export class DeviceRunSessionArtifactRejectedError extends SystemError {}
+
 class DeviceRunSessionArtifactUploadError extends SystemError {
   constructor(artifactId: string, reason: string) {
     super(`Failed to upload device run session artifact ${artifactId}: ${reason}.`);
@@ -55,7 +66,16 @@ class DeviceRunSessionArtifactUploadError extends SystemError {
 
 async function createDeviceRunSessionArtifactUploadSessionAsync(
   ctx: CustomBuildContext,
-  { deviceRunSessionId, artifactId, name, filename, kind, metadata, size }: ArtifactMetadata,
+  {
+    deviceRunSessionId,
+    artifactId,
+    name,
+    filename,
+    kind,
+    metadata,
+    size,
+    idempotencyKey,
+  }: ArtifactMetadata & { idempotencyKey: string },
   signal: AbortSignal
 ): Promise<SignedUrl> {
   const workerApi = requireDeviceRunSessionWorkerApi(ctx);
@@ -68,17 +88,19 @@ async function createDeviceRunSessionArtifactUploadSessionAsync(
         ...(kind !== undefined ? { kind } : {}),
         ...(metadata !== undefined ? { metadata } : {}),
         size,
+        idempotencyKey,
       },
       { signal }
     );
   } catch (err) {
     signal.throwIfAborted();
-    throw new SystemError(
-      `Failed to create upload session for device run session artifact ${artifactId}: ${
-        err instanceof Error ? err.message : String(err)
-      }`,
-      { cause: err }
-    );
+    const message = `Failed to create upload session for device run session artifact ${artifactId}: ${
+      err instanceof Error ? err.message : String(err)
+    }`;
+    if (err instanceof WorkerApiError && err.code === 'ARTIFACT_NAME_CONFLICT') {
+      throw new DeviceRunSessionArtifactRejectedError(message, { cause: err });
+    }
+    throw new SystemError(message, { cause: err });
   }
   signal.throwIfAborted();
   return uploadSession;

packages/build-tools/src/steps/utils/deviceRunSessionScreenshots.ts

@@ -1,5 +1,6 @@
 import { formatBytes } from '@expo/eas-build-job';
 import type { bunyan } from '@expo/logger';
+import { randomUUID } from 'node:crypto';
 import { type FSWatcher, createReadStream, watch } from 'node:fs';
 import { mkdtemp, readFile, readdir, rm, rmdir, stat } from 'node:fs/promises';
 import os from 'node:os';
@@ -8,7 +9,10 @@ import { z } from 'zod';
 
 import type { CustomBuildContext } from '../../customBuildContext';
 import { Sentry } from '../../sentry';
-import { uploadDeviceRunSessionArtifactAsync } from './deviceRunSessionArtifacts';
+import {
+  DeviceRunSessionArtifactRejectedError,
+  uploadDeviceRunSessionArtifactAsync,
+} from './deviceRunSessionArtifacts';
 import { createScreenshotMetadataAsync } from './deviceRunSessionScreenshotMetadata';
 import {
   type ScreenshotSession,
@@ -51,7 +55,7 @@ async function reportScreenshotFailureRecordAsync(file: string, logger: bunyan):
   await rm(file, { force: true });
 }
 
-type FailedUploads = Map<string, { attempts: number; lastError: Error }>;
+type FailedUploads = Map<string, { attempts: number; lastError: Error; idempotencyKey: string }>;
 
 type ScreenshotEntry = { kind: 'failureRecord' } | { kind: 'screenshot'; timestamp: string };
 
@@ -74,6 +78,7 @@ async function uploadScreenshotAsync(
     session,
     signal,
     failedUploads,
+    idempotencyKey,
     logger,
   }: {
     file: string;
@@ -84,6 +89,7 @@ async function uploadScreenshotAsync(
     session: Promise<ScreenshotSession | null>;
     signal: AbortSignal;
     failedUploads: FailedUploads;
+    idempotencyKey: string;
     logger: bunyan;
   }
 ): Promise<void> {
@@ -103,6 +109,7 @@ async function uploadScreenshotAsync(
     openStream: () => createReadStream(file),
     attempts: 3,
     signal,
+    idempotencyKey,
   });
   await rm(file);
   const previousFailures = failedUploads.get(name)?.attempts ?? 0;
@@ -127,6 +134,7 @@ function recordUploadFailure(
     size,
     signal,
     failedUploads,
+    idempotencyKey,
     logger,
   }: {
     file: string;
@@ -134,6 +142,7 @@ function recordUploadFailure(
     size: number | undefined;
     signal: AbortSignal;
     failedUploads: FailedUploads;
+    idempotencyKey: string;
     logger: bunyan;
   }
 ): 'aborted' | 'recorded' {
@@ -146,7 +155,14 @@ function recordUploadFailure(
   }
   const error = err instanceof Error ? err : new Error(String(err));
   const attempt = (failedUploads.get(name)?.attempts ?? 0) + 1;
-  failedUploads.set(name, { attempts: attempt, lastError: error });
+  failedUploads.set(name, { attempts: attempt, lastError: error, idempotencyKey });
+  if (error instanceof DeviceRunSessionArtifactRejectedError) {
+    logger.warn(
+      { err: error, file, attempt, size },
+      'www rejected the preview screenshot upload. Keeping it without retrying.'
+    );
+    return 'recorded';
+  }
   // A sustained outage would otherwise warn for every waiting file on each 30 s scan.
   if (attempt <= 3 || attempt % 10 === 0) {
     logger.warn(
@@ -202,6 +218,11 @@ export async function uploadDeviceRunSessionScreenshotsAsync(
       saveFailures++;
       continue;
     }
+    const failure = failedUploads.get(entry.name);
+    if (failure?.lastError instanceof DeviceRunSessionArtifactRejectedError) {
+      continue;
+    }
+    const idempotencyKey = failure?.idempotencyKey ?? randomUUID();
     let size: number | undefined;
     try {
       size = (await stat(file)).size;
@@ -214,6 +235,7 @@ export async function uploadDeviceRunSessionScreenshotsAsync(
         session,
         signal,
         failedUploads,
+        idempotencyKey,
         logger,
       });
       uploadedCount++;
@@ -227,6 +249,7 @@ export async function uploadDeviceRunSessionScreenshotsAsync(
         size,
         signal,
         failedUploads,
+        idempotencyKey,
         logger,
       });
       if (outcome === 'aborted') {

packages/build-tools/src/steps/utils/deviceRunSessionWorkerApi.ts

@@ -61,6 +61,7 @@ export type CreateDeviceRunSessionArtifactUploadSessionInput = {
   kind?: string | null;
   size: number;
   metadata?: Record<string, unknown> | null;
+  idempotencyKey: string;
 };
 
 /**
@@ -101,14 +102,14 @@ export class DeviceRunSessionWorkerApi {
   }
 
   public async createArtifactUploadSessionAsync(
-    input: CreateDeviceRunSessionArtifactUploadSessionInput,
+    { idempotencyKey, ...input }: CreateDeviceRunSessionArtifactUploadSessionInput,
     options: WorkerApiRequestOptions = {}
   ): Promise<DeviceRunSessionUploadSession> {
     const { uploadSession } = await this.client.requestAsync(
       'POST',
       'artifact-upload-sessions',
       UploadSessionSchema,
-      { ...options, json: input }
+      { ...options, json: input, idempotencyKey }
     );
     return uploadSession;
   }

packages/build-tools/src/steps/utils/sandboxArtifacts.ts

@@ -15,8 +15,13 @@ import { type SandboxArtifactUploadSession, type SandboxWorkerApi } from './sand
 import { WorkerApiError } from './workerApi';
 
 const WORKER_API_REQUEST_TIMEOUT_MS = 15_000;
+const UPLOAD_SESSION_RETRIES = 2;
 const FINALIZE_RETRIES = 2;
-const REJECTED_UPLOAD_SESSION_ERROR_CODES = new Set(['VALIDATION_ERROR', 'UNAUTHORIZED']);
+const REJECTED_UPLOAD_SESSION_ERROR_CODES = new Set([
+  'VALIDATION_ERROR',
+  'UNAUTHORIZED',
+  'ARTIFACT_NAME_CONFLICT',
+]);
 
 type SandboxArtifactUploadOutcome =
   | { status: 'uploaded'; downloadUrl: string }
@@ -124,7 +129,7 @@ export async function startSandboxArtifactUploadAsync({
   } = await createSandboxArtifactUploadSessionAsync(
     workerApi,
     { name, filename: path.basename(filePath), size },
-    signal
+    { signal, logger }
   );
   const completed = (async (): Promise<SandboxArtifactUploadOutcome> => {
     logger.info(`Uploading sandbox artifact "${name}" (${artifactId}, ${size} bytes).`);
@@ -158,12 +163,14 @@ export async function startSandboxArtifactUploadAsync({
 async function createSandboxArtifactUploadSessionAsync(
   workerApi: SandboxWorkerApi,
   input: { name: string; filename: string; size: number },
-  signal: AbortSignal
+  { signal, logger }: { signal: AbortSignal; logger: bunyan }
 ): Promise<SandboxArtifactUploadSession> {
   try {
     return await workerApi.createArtifactUploadSessionAsync(input, {
       signal,
       timeoutMs: WORKER_API_REQUEST_TIMEOUT_MS,
+      retries: UPLOAD_SESSION_RETRIES,
+      logger,
     });
   } catch (error) {
     signal.throwIfAborted();

packages/build-tools/src/steps/utils/sandboxWorkerApi.ts

@@ -1,4 +1,5 @@
 import { type Env, type EnvironmentSecret, type Job, SystemError } from '@expo/eas-build-job';
+import { randomUUID } from 'node:crypto';
 import { z } from 'zod';
 
 import {
@@ -65,7 +66,7 @@ export class SandboxWorkerApi {
       'POST',
       'artifact-upload-sessions',
       ArtifactUploadSessionSchema,
-      { ...options, json: input }
+      { ...options, json: input, idempotencyKey: randomUUID() }
     );
   }
 

packages/build-tools/src/steps/utils/workerApi.ts

@@ -26,6 +26,11 @@ export type WorkerApiRequestOptions = {
   logger?: bunyan;
 };
 
+type WorkerApiRequest = WorkerApiRequestOptions & {
+  json?: Record<string, unknown>;
+  idempotencyKey?: string;
+};
+
 const ApiV2ErrorResponseSchema = z.object({
   errors: z.array(z.object({ code: z.string().optional(), message: z.string() })).min(1),
 });
@@ -66,7 +71,7 @@ export class WorkerApi {
     method: 'GET' | 'POST',
     route: string,
     dataSchema: z.ZodType<T>,
-    { retries = 0, ...options }: WorkerApiRequestOptions & { json?: Record<string, unknown> } = {}
+    { retries = 0, ...options }: WorkerApiRequest = {}
   ): Promise<T> {
     const url = new URL(route, this.baseUrl);
     const routeDescription = `${method} ${url.pathname}`;
@@ -96,18 +101,14 @@ export class WorkerApi {
     url: string,
     routeDescription: string,
     dataSchema: z.ZodType<T>,
-    {
-      signal,
-      timeoutMs,
-      logger,
-      json,
-    }: Omit<WorkerApiRequestOptions, 'retries'> & {
-      json?: Record<string, unknown>;
-    }
+    { signal, timeoutMs, logger, json, idempotencyKey }: Omit<WorkerApiRequest, 'retries'>
   ): Promise<T> {
     const responseResult = await asyncResult(
       turtleFetch(url, method, {
-        headers: { [this.auth.authHeader]: this.auth.token },
+        headers: {
+          [this.auth.authHeader]: this.auth.token,
+          ...(idempotencyKey ? { 'Idempotency-Key': idempotencyKey } : {}),
+        },
         ...(json ? { json } : {}),
         ...(signal ? { signal } : {}),
         ...(timeoutMs !== undefined ? { timeout: timeoutMs } : {}),