[worker] Add a packaged startup wrapper

.changeset/download-build-network-retries.md

@@ -0,0 +1,5 @@
+---
+'@expo/build-tools': patch
+---
+
+Retry the build lookup and the archive download of `eas/download_build` on transient failures, and name the failure cause in the build log.

packages/build-tools/src/steps/functions/__tests__/downloadBuild.test.ts

@@ -11,6 +11,7 @@ import * as tar from 'tar';
 
 import { createGlobalContextMock } from '../../../__tests__/utils/context';
 import { createMockLogger } from '../../../__tests__/utils/logger';
+import { Datadog } from '../../../datadog';
 import * as files from '../../../utils/files';
 import { createDownloadBuildFunction, downloadBuildAsync } from '../downloadBuild';
 
@@ -30,11 +31,17 @@ const APPLICATION_ARCHIVE_URL = `https://expo.dev/artifacts/eas/${randomUUID()}.
 function createMockGraphqlClient({
   applicationArchiveUrl,
   error,
+  transientError,
 }: {
   applicationArchiveUrl?: string | null;
   error?: Error;
+  transientError?: Error;
 }): Client {
-  const toPromise = jest.fn().mockResolvedValue(
+  const toPromise = jest.fn();
+  if (transientError) {
+    toPromise.mockResolvedValueOnce({ error: transientError, data: undefined });
+  }
+  toPromise.mockResolvedValue(
     error
       ? { error, data: undefined }
       : {
@@ -62,7 +69,7 @@ function createSuccessfulResponse({
   url,
   headers,
 }: {
-  body: Buffer;
+  body: Buffer | Readable;
   url: string;
   headers?: Record<string, string>;
 }): Response {
@@ -72,7 +79,7 @@ function createSuccessfulResponse({
 
   return {
     ok: true,
-    body: Readable.from(body),
+    body: body instanceof Readable ? body : Readable.from(body),
     url,
     headers: {
       get: (name: string): string | null => normalizedHeaders[name.toLowerCase()] ?? null,
@@ -80,6 +87,47 @@ function createSuccessfulResponse({
   } as unknown as Response;
 }
 
+function createUndiciNetworkError(code: string): CombinedError {
+  return new CombinedError({
+    networkError: Object.assign(new TypeError('fetch failed'), {
+      cause: Object.assign(new Error(`read ${code}`), { code }),
+    }),
+  });
+}
+
+const { FetchError: ActualFetchError } = jest.requireActual(
+  'node-fetch'
+) as typeof import('node-fetch');
+
+function createSocketFetchError(url: string, code: string): Error {
+  return new ActualFetchError(`request to ${url} failed, reason: connect ${code}`, 'system', {
+    code,
+  } as any);
+}
+
+function useFakeBackoffTimers(): void {
+  // Streams and fs callbacks need the real nextTick and setImmediate; only the backoff timers are faked.
+  jest.useFakeTimers({ doNotFake: ['nextTick', 'setImmediate', 'queueMicrotask'] });
+}
+
+const MAX_IO_TURNS_BEFORE_BACKOFF = 1000;
+
+async function waitForBackoffTimerAsync(): Promise<void> {
+  for (let turns = 0; jest.getTimerCount() === 0; turns++) {
+    if (turns === MAX_IO_TURNS_BEFORE_BACKOFF) {
+      throw new Error('No backoff timer was scheduled.');
+    }
+    await new Promise(resolve => setImmediate(resolve));
+  }
+}
+
+async function fireBackoffTimersAsync(count: number): Promise<void> {
+  for (let i = 0; i < count; i++) {
+    await waitForBackoffTimerAsync();
+    await jest.advanceTimersToNextTimerAsync();
+  }
+}
+
 async function createFlatAppTarGzBufferAsync(): Promise<Buffer> {
   const sourceDirectory = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'flat-app-source-'));
   const archivePath = path.join(os.tmpdir(), `${randomUUID()}.tar.gz`);
@@ -108,6 +156,11 @@ async function createFlatAppTarGzBufferAsync(): Promise<Buffer> {
   }
 }
 
+afterEach(() => {
+  jest.useRealTimers();
+  jest.restoreAllMocks();
+});
+
 describe('downloadBuild', () => {
   it('downloads from applicationArchiveUrl returned by GraphQL', async () => {
     const buildId = randomUUID();
@@ -465,13 +518,13 @@ describe('downloadBuild', () => {
     expect(jest.mocked(fetch)).not.toHaveBeenCalled();
   });
 
-  it('throws SystemError when GraphQL request fails with a network error', async () => {
+  it('throws SystemError with the cause code after retrying a failing build lookup', async () => {
+    useFakeBackoffTimers();
     const buildId = randomUUID();
     const graphqlClient = createMockGraphqlClient({
-      error: new CombinedError({
-        networkError: new Error('Network request failed'),
-      }),
+      error: createUndiciNetworkError('ECONNRESET'),
     });
+    const datadogLog = jest.spyOn(Datadog, 'log');
 
     const promise = downloadBuildAsync({
       logger: createLogger({ name: 'test' }),
@@ -480,19 +533,81 @@ describe('downloadBuild', () => {
       robotAccessToken: null,
       extensions: ['app'],
     });
+    const rejection = expect(promise).rejects.toBeInstanceOf(SystemError);
+    await fireBackoffTimersAsync(5);
 
-    await expect(promise).rejects.toBeInstanceOf(SystemError);
+    await rejection;
     await expect(promise).rejects.toMatchObject({
-      message: `Could not fetch build ${buildId}: [Network] Network request failed`,
+      message: `Could not fetch build ${buildId}: [Network] fetch failed (ECONNRESET)`,
+    });
+    expect(graphqlClient.query).toHaveBeenCalledTimes(6);
+    expect(datadogLog).toHaveBeenCalledWith('download_build.build_lookup failed on attempt 6', {
+      event: 'network_retry',
+      operation: 'download_build.build_lookup',
+      outcome: 'failed',
+      attempts: '6',
+      code: 'ECONNRESET',
     });
-
     expect(jest.mocked(fetch)).not.toHaveBeenCalled();
   });
 
-  it('throws SystemError when GraphQL request returns 5xx', async () => {
+  it.each([
+    ['a network error', createUndiciNetworkError('ECONNRESET'), 'ECONNRESET'],
+    [
+      'a transient GraphQL error',
+      new CombinedError({
+        graphQLErrors: [
+          {
+            message: 'Too many requests',
+            extensions: { isTransient: true, errorCode: 'TOO_MANY_REQUESTS' },
+          },
+        ],
+        response: { status: 200 },
+      }),
+      'TOO_MANY_REQUESTS',
+    ],
+  ])('downloads after a build lookup that failed once with %s', async (_, transientError, code) => {
+    useFakeBackoffTimers();
     const buildId = randomUUID();
     const graphqlClient = createMockGraphqlClient({
-      error: Object.assign(new Error('Internal Server Error'), {
+      applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
+      transientError,
+    });
+    jest
+      .mocked(fetch)
+      .mockResolvedValue(
+        createSuccessfulResponse({ body: Buffer.from('hello'), url: 'https://cdn.test/app.apk' })
+      );
+
+    const datadogLog = jest.spyOn(Datadog, 'log');
+
+    const promise = downloadBuildAsync({
+      logger: createLogger({ name: 'test' }),
+      buildId,
+      graphqlClient,
+      robotAccessToken: null,
+      extensions: ['apk'],
+    });
+    await fireBackoffTimersAsync(1);
+
+    const { artifactPath } = await promise;
+    expect(path.basename(artifactPath)).toBe('app.apk');
+    expect(graphqlClient.query).toHaveBeenCalledTimes(2);
+    expect(datadogLog).toHaveBeenCalledWith('download_build.build_lookup recovered on attempt 2', {
+      event: 'network_retry',
+      operation: 'download_build.build_lookup',
+      outcome: 'recovered',
+      attempts: '2',
+      code,
+    });
+  });
+
+  it('throws SystemError when GraphQL request returns 5xx', async () => {
+    useFakeBackoffTimers();
+    const buildId = randomUUID();
+    const graphqlClient = createMockGraphqlClient({
+      error: new CombinedError({
+        graphQLErrors: [{ message: 'Internal Server Error' }],
         response: { status: 500 },
       }),
     });
@@ -504,12 +619,14 @@ describe('downloadBuild', () => {
       robotAccessToken: null,
       extensions: ['app'],
     });
+    const rejection = expect(promise).rejects.toBeInstanceOf(SystemError);
+    await fireBackoffTimersAsync(5);
 
-    await expect(promise).rejects.toBeInstanceOf(SystemError);
+    await rejection;
     await expect(promise).rejects.toMatchObject({
-      message: `Could not fetch build ${buildId}: Internal Server Error`,
+      message: `Could not fetch build ${buildId}: [GraphQL] Internal Server Error`,
     });
-
+    expect(graphqlClient.query).toHaveBeenCalledTimes(6);
     expect(jest.mocked(fetch)).not.toHaveBeenCalled();
   });
 
@@ -534,10 +651,193 @@ describe('downloadBuild', () => {
       errorCode: 'EAS_DOWNLOAD_BUILD_FETCH_FAILED',
       message: `Could not fetch build ${buildId}: [GraphQL] Build not found`,
     });
-
+    expect(graphqlClient.query).toHaveBeenCalledTimes(1);
     expect(jest.mocked(fetch)).not.toHaveBeenCalled();
   });
 
+  it('downloads after an archive request that failed once with a socket error', async () => {
+    useFakeBackoffTimers();
+    const graphqlClient = createMockGraphqlClient({});
+    jest
+      .mocked(fetch)
+      .mockRejectedValueOnce(createSocketFetchError(APPLICATION_ARCHIVE_URL, 'ECONNRESET'))
+      .mockResolvedValueOnce(
+        createSuccessfulResponse({ body: Buffer.from('hello'), url: 'https://cdn.test/app.apk' })
+      );
+
+    const promise = downloadBuildAsync({
+      logger: createLogger({ name: 'test' }),
+      applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
+      graphqlClient,
+      robotAccessToken: null,
+      extensions: ['apk'],
+    });
+    await fireBackoffTimersAsync(1);
+
+    const { artifactPath } = await promise;
+    expect(path.basename(artifactPath)).toBe('app.apk');
+    expect(jest.mocked(fetch)).toHaveBeenCalledTimes(2);
+  });
+
+  it('throws SystemError with the cause code after retrying a failing build archive request', async () => {
+    useFakeBackoffTimers();
+    const graphqlClient = createMockGraphqlClient({
+      applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
+    });
+    jest
+      .mocked(fetch)
+      .mockRejectedValue(createSocketFetchError(APPLICATION_ARCHIVE_URL, 'ETIMEDOUT'));
+    const datadogLog = jest.spyOn(Datadog, 'log');
+
+    const promise = downloadBuildAsync({
+      logger: createLogger({ name: 'test' }),
+      buildId: randomUUID(),
+      graphqlClient,
+      robotAccessToken: null,
+      extensions: ['apk'],
+    });
+    const rejection = expect(promise).rejects.toBeInstanceOf(SystemError);
+    await fireBackoffTimersAsync(5);
+
+    await rejection;
+    await expect(promise).rejects.toMatchObject({
+      message: `Could not download the application archive: request to ${APPLICATION_ARCHIVE_URL} failed, reason: connect ETIMEDOUT`,
+    });
+    expect(jest.mocked(fetch)).toHaveBeenCalledTimes(6);
+    expect(datadogLog).toHaveBeenCalledWith('download_build.archive_download failed on attempt 6', {
+      event: 'network_retry',
+      operation: 'download_build.archive_download',
+      outcome: 'failed',
+      attempts: '6',
+      code: 'ETIMEDOUT',
+    });
+  });
+
+  function createBrokenBody(code: string): Readable {
+    return Readable.from(
+      (async function* () {
+        yield Buffer.from('he');
+        throw Object.assign(new Error(`body failed with ${code}`), { code });
+      })()
+    );
+  }
+
+  it.each(['ERR_STREAM_PREMATURE_CLOSE', 'ECONNRESET'])(
+    'downloads again when the archive body breaks off with %s after the headers arrived',
+    async code => {
+      useFakeBackoffTimers();
+      const graphqlClient = createMockGraphqlClient({});
+      const brokenBody = createBrokenBody(code);
+      jest
+        .mocked(fetch)
+        .mockResolvedValueOnce(
+          createSuccessfulResponse({ body: brokenBody, url: 'https://cdn.test/app.apk' })
+        )
+        .mockResolvedValueOnce(
+          createSuccessfulResponse({ body: Buffer.from('hello'), url: 'https://cdn.test/app.apk' })
+        );
+      const datadogLog = jest.spyOn(Datadog, 'log');
+
+      const promise = downloadBuildAsync({
+        logger: createLogger({ name: 'test' }),
+        applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
+        graphqlClient,
+        robotAccessToken: null,
+        extensions: ['apk'],
+      });
+      await fireBackoffTimersAsync(1);
+
+      const { artifactPath } = await promise;
+      expect(await fs.promises.readFile(artifactPath, 'utf-8')).toBe('hello');
+      expect(jest.mocked(fetch)).toHaveBeenCalledTimes(2);
+      expect(datadogLog).toHaveBeenCalledWith(
+        'download_build.archive_download recovered on attempt 2',
+        expect.objectContaining({ outcome: 'recovered', code })
+      );
+    }
+  );
+
+  it('throws SystemError when the build archive body keeps breaking off', async () => {
+    useFakeBackoffTimers();
+    const graphqlClient = createMockGraphqlClient({
+      applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
+    });
+    jest.mocked(fetch).mockImplementation(async () =>
+      createSuccessfulResponse({
+        body: createBrokenBody('ECONNRESET'),
+        url: 'https://cdn.test/app.apk',
+      })
+    );
+    const datadogLog = jest.spyOn(Datadog, 'log');
+
+    const promise = downloadBuildAsync({
+      logger: createLogger({ name: 'test' }),
+      buildId: randomUUID(),
+      graphqlClient,
+      robotAccessToken: null,
+      extensions: ['apk'],
+    });
+    const rejection = expect(promise).rejects.toBeInstanceOf(SystemError);
+    await fireBackoffTimersAsync(5);
+
+    await rejection;
+    await expect(promise).rejects.toMatchObject({
+      message: 'Could not download the application archive: body failed with ECONNRESET',
+    });
+    expect(jest.mocked(fetch)).toHaveBeenCalledTimes(6);
+    expect(datadogLog).toHaveBeenCalledWith(
+      'download_build.archive_download failed on attempt 6',
+      expect.objectContaining({ outcome: 'failed', code: 'ECONNRESET' })
+    );
+  });
+
+  it('does not start another archive attempt after an abort during the backoff', async () => {
+    useFakeBackoffTimers();
+    const graphqlClient = createMockGraphqlClient({});
+    jest
+      .mocked(fetch)
+      .mockRejectedValueOnce(createSocketFetchError(APPLICATION_ARCHIVE_URL, 'ECONNRESET'));
+    const controller = new AbortController();
+    const reason = new Error('step timed out');
+
+    const promise = downloadBuildAsync({
+      logger: createLogger({ name: 'test' }),
+      applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
+      graphqlClient,
+      robotAccessToken: null,
+      extensions: ['apk'],
+      signal: controller.signal,
+    });
+    const rejection = expect(promise).rejects.toBe(reason);
+    await waitForBackoffTimerAsync();
+    controller.abort(reason);
+    await jest.advanceTimersToNextTimerAsync();
+
+    await rejection;
+    expect(jest.mocked(fetch)).toHaveBeenCalledTimes(1);
+  });
+
+  it('rethrows the raw error of a failing user-supplied archive URL after the retries', async () => {
+    useFakeBackoffTimers();
+    const graphqlClient = createMockGraphqlClient({});
+    jest
+      .mocked(fetch)
+      .mockRejectedValue(createSocketFetchError(APPLICATION_ARCHIVE_URL, 'ENOTFOUND'));
+
+    const promise = downloadBuildAsync({
+      logger: createLogger({ name: 'test' }),
+      applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
+      graphqlClient,
+      robotAccessToken: null,
+      extensions: ['apk'],
+    });
+    const rejection = expect(promise).rejects.toBeInstanceOf(ActualFetchError);
+    await fireBackoffTimersAsync(5);
+
+    await rejection;
+    expect(jest.mocked(fetch)).toHaveBeenCalledTimes(6);
+  });
+
   it('should throw an error if no matching files are found', async () => {
     const graphqlClient = createMockGraphqlClient({
       applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
@@ -605,6 +905,21 @@ describe('createDownloadBuildFunction', () => {
       };
     }
 
+    function createApplicationResponse(buildId: string): Response {
+      return new ActualResponse(
+        JSON.stringify({
+          data: {
+            build: {
+              id: buildId,
+              gitCommitHash: null,
+              applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
+            },
+          },
+        }),
+        { status: 200 }
+      );
+    }
+
     function createStep(ctx: any, callInputs: Record<string, unknown>) {
       return createDownloadBuildFunction(ctx).createBuildStepFromFunctionCall(
         createGlobalContextMock({
@@ -620,18 +935,7 @@ describe('createDownloadBuildFunction', () => {
       const graphqlClient = createMockGraphqlClient({});
       jest.mocked(fetch).mockImplementation(async url => {
         if (url === workerApiUrl) {
-          return new ActualResponse(
-            JSON.stringify({
-              data: {
-                build: {
-                  id: buildId,
-                  gitCommitHash: null,
-                  applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
-                },
-              },
-            }),
-            { status: 200 }
-          );
+          return createApplicationResponse(buildId);
         }
         return {
           ok: false,
@@ -661,6 +965,33 @@ describe('createDownloadBuildFunction', () => {
       expect(graphqlClient.query).not.toHaveBeenCalled();
     });
 
+    it('retries the worker API lookup after a 5xx response', async () => {
+      useFakeBackoffTimers();
+      const buildId = randomUUID();
+      const graphqlClient = createMockGraphqlClient({});
+      jest
+        .mocked(fetch)
+        .mockResolvedValueOnce(new ActualResponse('Service Unavailable', { status: 503 }))
+        .mockResolvedValueOnce(createApplicationResponse(buildId))
+        .mockResolvedValueOnce(
+          createSuccessfulResponse({ body: Buffer.from('hello'), url: 'https://cdn.test/app.apk' })
+        );
+
+      const execution = createStep(createSessionCtx(graphqlClient), {
+        build_id: buildId,
+        extensions: ['apk'],
+      }).executeAsync();
+      await fireBackoffTimersAsync(1);
+
+      await execution;
+      expect(jest.mocked(fetch)).toHaveBeenCalledTimes(3);
+      expect(jest.mocked(fetch).mock.calls.map(([url]) => url)).toEqual([
+        workerApiUrl,
+        workerApiUrl,
+        APPLICATION_ARCHIVE_URL,
+      ]);
+    });
+
     it('does not call the worker API for an application_archive_url input', async () => {
       const graphqlClient = createMockGraphqlClient({});
       jest.mocked(fetch).mockResolvedValue({

packages/build-tools/src/steps/functions/downloadBuild.ts

@@ -7,7 +7,7 @@ import {
   BuildStepInputValueTypeName,
   BuildStepOutput,
 } from '@expo/steps';
-import { Client } from '@urql/core';
+import { Client, CombinedError } from '@urql/core';
 import contentDisposition from 'content-disposition';
 import { glob } from 'fast-glob';
 import { graphql } from 'gql.tada';
@@ -26,10 +26,16 @@ import { CustomBuildContext } from '../../customBuildContext';
 import { formatBytes } from '../../utils/artifacts';
 import { decompressTarAsync, isFileTarGzAsync } from '../../utils/files';
 import { graphqlAbortContext } from '../../utils/graphqlAbort';
-import { retryOnDNSFailure } from '../../utils/retryOnDNSFailure';
+import {
+  describeError,
+  isTransferFailure,
+  isTransientGraphqlError,
+} from '../../utils/networkErrors';
+import { retryOnNetworkFailureAsync } from '../../utils/retryOnNetworkFailure';
 import { pluralize } from '../../utils/strings';
 import {
   type DeviceRunSessionWorkerApi,
+  DeviceRunSessionWorkerApiError,
   getDeviceRunSessionWorkerApi,
 } from '../utils/deviceRunSessionWorkerApi';
 
@@ -152,21 +158,34 @@ async function fetchApplicationArchiveUrlAsync({
       signal,
     });
   }
-  const result = await graphqlClient
-    .query(BUILD_BY_ID_QUERY, { buildId }, graphqlAbortContext(signal))
-    .toPromise();
+  let data;
+  try {
+    data = await retryOnNetworkFailureAsync(
+      async () => {
+        const queryResult = await graphqlClient
+          .query(BUILD_BY_ID_QUERY, { buildId }, graphqlAbortContext(signal))
+          .toPromise();
+        if (queryResult.error) {
+          throw queryResult.error;
+        }
+        return queryResult.data;
+      },
+      {
+        operation: 'download_build.build_lookup',
+        signal,
+        isRetryable: error => error instanceof CombinedError && isTransientGraphqlError(error),
+      }
+    );
+  } catch (error) {
+    signal?.throwIfAborted();
+    const message = `Could not fetch build ${buildId}: ${describeError(error)}`;
+    throw error instanceof CombinedError && !isTransientGraphqlError(error)
+      ? new UserError('EAS_DOWNLOAD_BUILD_FETCH_FAILED', message, { cause: error })
+      : new SystemError(message, { cause: error });
+  }
   signal?.throwIfAborted();
 
-  if (result.error) {
-    const { error } = result;
-    const message = `Could not fetch build ${buildId}: ${error.message}`;
-
-    throw error.networkError || result.error.response?.status >= 500
-      ? new SystemError(message, { cause: error })
-      : new UserError('EAS_DOWNLOAD_BUILD_FETCH_FAILED', message, { cause: error });
-  }
-
-  const applicationArchiveUrl = result.data?.builds.byId?.artifacts?.applicationArchiveUrl;
+  const applicationArchiveUrl = data?.builds.byId?.artifacts?.applicationArchiveUrl;
   if (!applicationArchiveUrl) {
     throw new UserError(
       'EAS_DOWNLOAD_BUILD_NO_APPLICATION_ARCHIVE',
@@ -195,13 +214,18 @@ async function fetchDeviceRunSessionApplicationArchiveUrlAsync({
 }): Promise<string> {
   let application;
   try {
-    application = await deviceRunSessionWorkerApi.getApplicationAsync({ signal });
+    application = await retryOnNetworkFailureAsync(
+      () => deviceRunSessionWorkerApi.getApplicationAsync({ signal }),
+      {
+        operation: 'download_build.session_build_lookup',
+        signal,
+        isRetryable: error => error instanceof DeviceRunSessionWorkerApiError && error.isTransient,
+      }
+    );
   } catch (err) {
     signal?.throwIfAborted();
     throw new SystemError(
-      `Could not fetch build ${buildId} for this device run session: ${
-        err instanceof Error ? err.message : String(err)
-      }`,
+      `Could not fetch build ${buildId} for this device run session: ${describeError(err)}`,
       { cause: err }
     );
   }
@@ -281,18 +305,34 @@ export async function downloadBuildAsync(
   );
 
   signal?.throwIfAborted();
-  const response = await retryOnDNSFailure(fetch)(downloadUrl, { headers, signal });
-
-  if (!response.ok) {
-    const textResult = await asyncResult(response.text());
-    throw new Error(`Unexpected response from server (${response.status}): ${textResult.value}`);
+  let archivePath: string;
+  try {
+    archivePath = await retryOnNetworkFailureAsync(
+      async () => {
+        const response = await fetch(downloadUrl, { headers, signal });
+        if (!response.ok) {
+          const textResult = await asyncResult(response.text());
+          throw new Error(
+            `Unexpected response from server (${response.status}): ${textResult.value}`
+          );
+        }
+        const archiveFilename = resolveArchiveFilename({ response, extensions });
+        const attemptArchivePath = path.join(downloadDestinationDirectory, archiveFilename);
+        // On abort, node-fetch emits an error on the body stream, which fails the pipeline.
+        await streamPipeline(response.body, fs.createWriteStream(attemptArchivePath));
+        return attemptArchivePath;
+      },
+      { operation: 'download_build.archive_download', signal, isRetryable: isTransferFailure }
+    );
+  } catch (error) {
+    signal?.throwIfAborted();
+    if (params.buildId && isTransferFailure(error)) {
+      throw new SystemError(`Could not download the application archive: ${describeError(error)}`, {
+        cause: error,
+      });
+    }
+    throw error;
   }
-
-  const archiveFilename = resolveArchiveFilename({ response, extensions });
-  const archivePath = path.join(downloadDestinationDirectory, archiveFilename);
-
-  // On abort, node-fetch emits an error on the body stream, which fails the pipeline.
-  await streamPipeline(response.body, fs.createWriteStream(archivePath));
   signal?.throwIfAborted();
 
   const { size } = await fs.promises.stat(archivePath);

packages/build-tools/src/steps/utils/__tests__/deviceRunSessionWorkerApi.test.ts

@@ -330,7 +330,8 @@ describe(DeviceRunSessionWorkerApi, () => {
   });
 
   it('reports a request that failed before www responded', async () => {
-    jest.mocked(turtleFetch).mockRejectedValueOnce(new Error('socket hang up'));
+    const reason = new Error('socket hang up');
+    jest.mocked(turtleFetch).mockRejectedValueOnce(reason);
 
     const error = await createApi()
       .getSessionAsync()
@@ -338,6 +339,7 @@ describe(DeviceRunSessionWorkerApi, () => {
 
     expect(error).toBeInstanceOf(DeviceRunSessionWorkerApiError);
     expect(error.status).toBeUndefined();
+    expect(error.cause).toBe(reason);
     expect(error.message).toContain('failed before www responded: socket hang up');
   });
 

packages/build-tools/src/steps/utils/deviceRunSessionWorkerApi.ts

@@ -88,11 +88,16 @@ export class DeviceRunSessionWorkerApiError extends Error {
   constructor(
     message: string,
     /** The HTTP status of the response, or undefined when no response arrived. */
-    readonly status: number | undefined
+    readonly status: number | undefined,
+    options?: ErrorOptions
   ) {
-    super(message);
+    super(message, options);
     this.name = 'DeviceRunSessionWorkerApiError';
   }
+
+  get isTransient(): boolean {
+    return this.status === undefined || this.status >= 500;
+  }
 }
 
 /**
@@ -218,7 +223,8 @@ export class DeviceRunSessionWorkerApi {
         `${routeDescription} failed before www responded: ${
           reason instanceof Error ? reason.message : String(reason)
         }. Check the network connection of the worker.`,
-        undefined
+        undefined,
+        { cause: reason }
       );
     }
     const response = responseResult.value;

packages/build-tools/src/utils/__tests__/networkErrors.test.ts

@@ -1,4 +1,16 @@
-import { isConnectFailure, isConnectionInterruptedError } from '../networkErrors';
+import { CombinedError } from '@urql/core';
+
+import {
+  describeError,
+  findErrorCode,
+  findGraphqlErrorCode,
+  isConnectFailure,
+  isConnectionInterruptedError,
+  isTransferFailure,
+  isTransientGraphqlError,
+} from '../networkErrors';
+
+const { FetchError } = jest.requireActual('node-fetch') as typeof import('node-fetch');
 
 it.each([
   'EAI_AGAIN',
@@ -34,3 +46,77 @@ it.each(['ETIMEDOUT', 'CERT_HAS_EXPIRED', undefined])('does not automatically re
   expect(isConnectFailure(error)).toBe(false);
   expect(isConnectionInterruptedError(error)).toBe(false);
 });
+
+it('finds the errno code on the cause of an undici fetch failure', () => {
+  const error = new CombinedError({
+    networkError: new TypeError('fetch failed', {
+      cause: Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }),
+    }),
+  });
+  expect(findErrorCode(error)).toBe('ECONNRESET');
+  expect(findErrorCode(new Error('Request failed'))).toBeUndefined();
+  expect(findErrorCode('Request failed')).toBeUndefined();
+});
+
+it('finds the www error code on a GraphQL error', () => {
+  const error = new CombinedError({
+    graphQLErrors: [
+      { message: 'Too many requests', extensions: { errorCode: 'TOO_MANY_REQUESTS' } },
+    ],
+  });
+  expect(findGraphqlErrorCode(error)).toBe('TOO_MANY_REQUESTS');
+  expect(
+    findGraphqlErrorCode(new CombinedError({ graphQLErrors: ['Build not found'] }))
+  ).toBeUndefined();
+  expect(findGraphqlErrorCode(new Error('Request failed'))).toBeUndefined();
+});
+
+it.each([
+  ['Request failed', 'ECONNRESET', 'Request failed (ECONNRESET)'],
+  ['read ECONNRESET', 'ECONNRESET', 'read ECONNRESET'],
+  ['Request failed', undefined, 'Request failed'],
+])('describes %p with code %p as %p', (message, code, expected) => {
+  expect(describeError(Object.assign(new Error(message), { code }))).toBe(expected);
+});
+
+it('treats a missing response, a 5xx response or an isTransient GraphQL error as transient', () => {
+  expect(
+    isTransientGraphqlError(new CombinedError({ networkError: new Error('fetch failed') }))
+  ).toBe(true);
+  expect(
+    isTransientGraphqlError(
+      new CombinedError({ graphQLErrors: ['Internal Server Error'], response: { status: 503 } })
+    )
+  ).toBe(true);
+  expect(
+    isTransientGraphqlError(
+      new CombinedError({
+        graphQLErrors: [{ message: 'Database unavailable', extensions: { isTransient: true } }],
+        response: { status: 200 },
+      })
+    )
+  ).toBe(true);
+  expect(
+    isTransientGraphqlError(
+      new CombinedError({ graphQLErrors: ['Build not found'], response: { status: 200 } })
+    )
+  ).toBe(false);
+});
+
+it.each([
+  [
+    'a node-fetch socket failure',
+    new FetchError('request failed', 'system', { code: 'ECONNRESET' } as any),
+    true,
+  ],
+  ['a node-fetch body size failure', new FetchError('too large', 'max-size'), false],
+  ['a body reset', Object.assign(new Error('aborted'), { code: 'ECONNRESET' }), true],
+  [
+    'a body closed early',
+    Object.assign(new Error('closed'), { code: 'ERR_STREAM_PREMATURE_CLOSE' }),
+    true,
+  ],
+  ['an HTTP error response', new Error('Unexpected response from server (500)'), false],
+])('classifies %s as a transfer failure or not', (_label, error, expected) => {
+  expect(isTransferFailure(error)).toBe(expected);
+});

packages/build-tools/src/utils/networkErrors.ts

@@ -1,3 +1,5 @@
+import { CombinedError } from '@urql/core';
+
 const CONNECT_FAILURE_CODES = new Set([
   'EAI_AGAIN',
   'ECONNREFUSED',
@@ -30,3 +32,50 @@ export function isConnectionInterruptedError(error: unknown): boolean {
   }
   return false;
 }
+
+/** Errno code of the error or its causes; undici keeps it on cause, urql on networkError. */
+export function findErrorCode(error: unknown): string | undefined {
+  let current: unknown = error;
+  while (current instanceof Error) {
+    if ('code' in current && typeof current.code === 'string') {
+      return current.code;
+    }
+    current = current instanceof CombinedError ? current.networkError : current.cause;
+  }
+  return undefined;
+}
+
+export function findGraphqlErrorCode(error: unknown): string | undefined {
+  if (!(error instanceof CombinedError)) {
+    return undefined;
+  }
+  const code = error.graphQLErrors.at(0)?.extensions.errorCode;
+  return typeof code === 'string' ? code : undefined;
+}
+
+export function describeError(error: unknown): string {
+  const message = error instanceof Error ? error.message : String(error);
+  const code = findErrorCode(error);
+  return code && !message.includes(code) ? `${message} (${code})` : message;
+}
+
+export function isTransientGraphqlError(error: CombinedError): boolean {
+  return (
+    Boolean(error.networkError) ||
+    error.response?.status >= 500 ||
+    error.graphQLErrors.some(graphqlError => graphqlError.extensions.isTransient === true)
+  );
+}
+
+// node-fetch throws type 'system' for a socket failure before the headers; after them it forwards
+// ECONNRESET or ERR_STREAM_PREMATURE_CLOSE to the body. HTTP error responses never throw.
+export function isTransferFailure(error: unknown): boolean {
+  return (
+    (error instanceof Error &&
+      error.name === 'FetchError' &&
+      'type' in error &&
+      error.type === 'system') ||
+    isConnectionInterruptedError(error) ||
+    findErrorCode(error) === 'ERR_STREAM_PREMATURE_CLOSE'
+  );
+}

packages/build-tools/src/utils/retryOnNetworkFailure.ts

@@ -0,0 +1,53 @@
+import { Datadog } from '../datadog';
+import { findErrorCode, findGraphqlErrorCode } from './networkErrors';
+import { promiseRetryWithCondition } from './promiseRetryWithCondition';
+
+export async function retryOnNetworkFailureAsync<T>(
+  attemptAsync: () => Promise<T>,
+  {
+    operation,
+    signal,
+    isRetryable,
+  }: {
+    operation: string;
+    signal?: AbortSignal;
+    isRetryable: (error: unknown) => boolean;
+  }
+): Promise<T> {
+  const shouldRetry = (error: unknown): boolean => !signal?.aborted && isRetryable(error);
+  let attempts = 1;
+  let lastError: unknown;
+  const report = (outcome: 'recovered' | 'failed', error: unknown): void => {
+    Datadog.log(`${operation} ${outcome} on attempt ${attempts}`, {
+      event: 'network_retry',
+      operation,
+      outcome,
+      attempts: String(attempts),
+      code: findErrorCode(error) ?? findGraphqlErrorCode(error) ?? 'unknown',
+    });
+  };
+
+  try {
+    const result = await promiseRetryWithCondition(
+      async () => {
+        signal?.throwIfAborted();
+        return await attemptAsync();
+      },
+      shouldRetry,
+      { retries: 5, minTimeout: 1_000, factor: 2 },
+      ({ attemptNumber, error }) => {
+        attempts = attemptNumber;
+        lastError = error;
+      }
+    )();
+    if (attempts > 1) {
+      report('recovered', lastError);
+    }
+    return result;
+  } catch (error) {
+    if (attempts > 1 && !signal?.aborted) {
+      report('failed', error);
+    }
+    throw error;
+  }
+}

packages/worker/README.md

@@ -42,3 +42,18 @@ Github action builds a docker image for every PR with any change in `infra/eas-b
 ### VM template
 
 Github action builds a new vm template for every PR with any change in `infra/eas-build-worker-images/ios/{{ templateName }}`.
+
+### Packaged startup
+
+Existing services run `src/services/worker/dist/main.js`. In the worker archive,
+that path points to `bin/bootstrap.cjs`. The bootstrap loads only Node built-ins
+and starts `bin/start-worker.sh`. The shell then uses `exec` to start the normal
+worker with the original Node binary, options, and arguments.
+
+Put setup that must precede worker imports or Node startup options in the shell.
+The bootstrap forwards SIGINT, SIGTERM, IPC messages, and the worker exit status.
+It remains the supervised parent process; test service shutdown on both platforms
+before deployment. A failure in the startup shell prevents the worker from
+loading. Direct execution of `packages/worker/dist/main.js` bypasses this hook.
+
+Run `yarn workspace @expo/worker test:startup` for the process-level checks.

packages/worker/bin/bootstrap.cjs

@@ -0,0 +1,53 @@
+// This entry point must only load Node built-ins before the startup shell runs.
+const { spawn } = require('node:child_process');
+const path = require('node:path');
+
+const worker = spawn(
+  '/bin/bash',
+  [
+    path.join(__dirname, 'start-worker.sh'),
+    process.execPath,
+    ...process.execArgv,
+    path.join(__dirname, '../dist/main.js'),
+    ...process.argv.slice(2),
+  ],
+  { stdio: process.send ? ['inherit', 'inherit', 'inherit', 'ipc'] : 'inherit' }
+);
+
+const signals = ['SIGINT', 'SIGTERM'];
+for (const signal of signals) {
+  process.on(signal, () => worker.kill(signal));
+}
+
+// Retain the readiness channel used by process managers in development.
+if (process.send) {
+  worker.on('message', message => {
+    if (process.connected) {
+      process.send(message, () => {});
+    }
+  });
+  process.on('message', message => {
+    if (worker.connected) {
+      worker.send(message, () => {});
+    }
+  });
+  process.on('disconnect', () => {
+    if (worker.connected) {
+      worker.disconnect();
+    }
+  });
+}
+
+worker.on('error', error => {
+  process.stderr.write(`Worker startup failed: ${error.stack ?? error}\n`, () => process.exit(1));
+});
+worker.on('exit', (code, signal) => {
+  if (signal) {
+    for (const forwarded of signals) {
+      process.removeAllListeners(forwarded);
+    }
+    process.kill(process.pid, signal);
+  } else {
+    process.exit(code ?? 1);
+  }
+});

packages/worker/bin/start-worker.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Startup setup belongs here, before the normal worker's Node process starts.
+exec "$@"

packages/worker/package.json

@@ -11,11 +11,12 @@
     "compile": "tsc -p tsconfig.build.json",
     "prepublishOnly": "npm run build",
     "test": "yarn test:unit",
-    "test:unit": "./resources/jest --config jest.config.unit.ts",
+    "test:unit": "yarn test:startup && ./resources/jest --config jest.config.unit.ts",
     "test:unit:watch": "./resources/jest --config jest.config.unit.ts --watch",
     "test:integration": "./resources/jest --runInBand --config jest.config.integration.ts",
     "test:integration:watch": "./resources/jest --runInBand --config jest.config.integration.ts --watchAll",
-    "typecheck": "tsc"
+    "typecheck": "tsc",
+    "test:startup": "node --test tests/*.test.cjs"
   },
   "dependencies": {
     "@babel/core": "7.24.7",

packages/worker/package.sh

@@ -120,10 +120,10 @@ fi
 
 # Create backward-compatible symlink for orchestrator
 # The orchestrator expects ./src/services/worker/dist/main.js
-# but the new structure has ./packages/worker/dist/main.js
+# Route it through the packaged startup wrapper before loading worker modules.
 # We're only symlinking the specific file to discover any other legacy path usage
 mkdir -p "$target_root_dir/src/services/worker/dist"
-ln -s ../../../../packages/worker/dist/main.js "$target_root_dir/src/services/worker/dist/main.js"
+ln -s ../../../../packages/worker/bin/bootstrap.cjs "$target_root_dir/src/services/worker/dist/main.js"
 
 tar zcf $OUTPUT_FILE -C $target_root_dir .
 

packages/worker/tests/startup.test.cjs

@@ -0,0 +1,100 @@
+const assert = require('node:assert/strict');
+const { spawn } = require('node:child_process');
+const { once } = require('node:events');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { test } = require('node:test');
+
+function archive(t, main, shell) {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'worker-startup-'));
+  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+  const worker = path.join(root, 'packages/worker');
+  fs.mkdirSync(path.join(worker, 'dist'), { recursive: true });
+  fs.cpSync(path.join(__dirname, '../bin'), path.join(worker, 'bin'), { recursive: true });
+  fs.writeFileSync(path.join(worker, 'dist/main.js'), main);
+  if (shell) {
+    fs.writeFileSync(path.join(worker, 'bin/start-worker.sh'), shell);
+  }
+  // Exercise the path that existing images run, including Node's symlink resolution.
+  const legacy = path.join(root, 'src/services/worker/dist/main.js');
+  fs.mkdirSync(path.dirname(legacy), { recursive: true });
+  fs.symlinkSync('../../../../packages/worker/bin/bootstrap.cjs', legacy);
+  return legacy;
+}
+
+function launch(t, entry, args = []) {
+  const child = spawn(process.execPath, [entry, ...args], {
+    stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+  });
+  t.after(() => {
+    if (child.exitCode === null && child.signalCode === null) {
+      child.kill('SIGKILL');
+    }
+  });
+  return child;
+}
+
+test(
+  'startup settings reach the worker before its first module runs',
+  { timeout: 10000 },
+  async t => {
+    const entry = archive(
+      t,
+      'process.send({ ready: true, args: process.argv.slice(2), setting: process.env.STARTUP_TEST }); process.on("message", () => process.exit(7));',
+      '#!/bin/bash\nset -euo pipefail\nexport STARTUP_TEST=installed\nexec "$@"\n'
+    );
+    const child = launch(t, entry, ['argument with spaces']);
+    const exited = once(child, 'exit');
+    assert.deepEqual((await once(child, 'message'))[0], {
+      ready: true,
+      args: ['argument with spaces'],
+      setting: 'installed',
+    });
+    child.send('stop');
+    assert.deepEqual(await exited, [7, null]);
+  }
+);
+
+for (const signal of ['SIGINT', 'SIGTERM']) {
+  test(`forwards ${signal} and preserves termination status`, { timeout: 10000 }, async t => {
+    const entry = archive(t, 'process.send("ready"); setInterval(() => {}, 1000);');
+    const child = launch(t, entry);
+    const exited = once(child, 'exit');
+    await once(child, 'message');
+    child.kill(signal);
+    assert.deepEqual(await exited, [null, signal]);
+  });
+}
+
+test('startup failure prevents loading the worker', { timeout: 10000 }, async t => {
+  const entry = archive(
+    t,
+    'process.send("must not start");',
+    '#!/bin/bash\nset -euo pipefail\nexit 42\nexec "$@"\n'
+  );
+  const child = launch(t, entry);
+  const messages = [];
+  child.on('message', message => messages.push(message));
+  assert.deepEqual(await once(child, 'exit'), [42, null]);
+  assert.deepEqual(messages, []);
+});
+
+test('runs without IPC and retains stdout, stderr, and exit code', { timeout: 10000 }, async t => {
+  const entry = archive(
+    t,
+    'console.log("worker output"); console.error("worker error"); process.exit(9);'
+  );
+  const child = spawn(process.execPath, [entry], { stdio: ['ignore', 'pipe', 'pipe'] });
+  let stdout = '';
+  let stderr = '';
+  child.stdout.on('data', data => {
+    stdout += data;
+  });
+  child.stderr.on('data', data => {
+    stderr += data;
+  });
+  assert.deepEqual(await once(child, 'close'), [9, null]);
+  assert.equal(stdout, 'worker output\n');
+  assert.equal(stderr, 'worker error\n');
+});