[eas-cli] Keep update uploads alive while bytes are sent and report slow uploads

.changeset/download-build-network-retries.md

@@ -1,5 +0,0 @@
----
-'@expo/build-tools': patch
----
-
-Retry the build lookup and the archive download of `eas/download_build` on transient failures, and name the failure cause in the build log.

packages/build-tools/src/steps/functions/__tests__/downloadBuild.test.ts

@@ -11,7 +11,6 @@ import * as tar from 'tar';
 
 import { createGlobalContextMock } from '../../../__tests__/utils/context';
 import { createMockLogger } from '../../../__tests__/utils/logger';
-import { Datadog } from '../../../datadog';
 import * as files from '../../../utils/files';
 import { createDownloadBuildFunction, downloadBuildAsync } from '../downloadBuild';
 
@@ -31,17 +30,11 @@ const APPLICATION_ARCHIVE_URL = `https://expo.dev/artifacts/eas/${randomUUID()}.
 function createMockGraphqlClient({
   applicationArchiveUrl,
   error,
-  transientError,
 }: {
   applicationArchiveUrl?: string | null;
   error?: Error;
-  transientError?: Error;
 }): Client {
-  const toPromise = jest.fn();
-  if (transientError) {
-    toPromise.mockResolvedValueOnce({ error: transientError, data: undefined });
-  }
-  toPromise.mockResolvedValue(
+  const toPromise = jest.fn().mockResolvedValue(
     error
       ? { error, data: undefined }
       : {
@@ -69,7 +62,7 @@ function createSuccessfulResponse({
   url,
   headers,
 }: {
-  body: Buffer | Readable;
+  body: Buffer;
   url: string;
   headers?: Record<string, string>;
 }): Response {
@@ -79,7 +72,7 @@ function createSuccessfulResponse({
 
   return {
     ok: true,
-    body: body instanceof Readable ? body : Readable.from(body),
+    body: Readable.from(body),
     url,
     headers: {
       get: (name: string): string | null => normalizedHeaders[name.toLowerCase()] ?? null,
@@ -87,47 +80,6 @@ function createSuccessfulResponse({
   } as unknown as Response;
 }
 
-function createUndiciNetworkError(code: string): CombinedError {
-  return new CombinedError({
-    networkError: Object.assign(new TypeError('fetch failed'), {
-      cause: Object.assign(new Error(`read ${code}`), { code }),
-    }),
-  });
-}
-
-const { FetchError: ActualFetchError } = jest.requireActual(
-  'node-fetch'
-) as typeof import('node-fetch');
-
-function createSocketFetchError(url: string, code: string): Error {
-  return new ActualFetchError(`request to ${url} failed, reason: connect ${code}`, 'system', {
-    code,
-  } as any);
-}
-
-function useFakeBackoffTimers(): void {
-  // Streams and fs callbacks need the real nextTick and setImmediate; only the backoff timers are faked.
-  jest.useFakeTimers({ doNotFake: ['nextTick', 'setImmediate', 'queueMicrotask'] });
-}
-
-const MAX_IO_TURNS_BEFORE_BACKOFF = 1000;
-
-async function waitForBackoffTimerAsync(): Promise<void> {
-  for (let turns = 0; jest.getTimerCount() === 0; turns++) {
-    if (turns === MAX_IO_TURNS_BEFORE_BACKOFF) {
-      throw new Error('No backoff timer was scheduled.');
-    }
-    await new Promise(resolve => setImmediate(resolve));
-  }
-}
-
-async function fireBackoffTimersAsync(count: number): Promise<void> {
-  for (let i = 0; i < count; i++) {
-    await waitForBackoffTimerAsync();
-    await jest.advanceTimersToNextTimerAsync();
-  }
-}
-
 async function createFlatAppTarGzBufferAsync(): Promise<Buffer> {
   const sourceDirectory = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'flat-app-source-'));
   const archivePath = path.join(os.tmpdir(), `${randomUUID()}.tar.gz`);
@@ -156,11 +108,6 @@ async function createFlatAppTarGzBufferAsync(): Promise<Buffer> {
   }
 }
 
-afterEach(() => {
-  jest.useRealTimers();
-  jest.restoreAllMocks();
-});
-
 describe('downloadBuild', () => {
   it('downloads from applicationArchiveUrl returned by GraphQL', async () => {
     const buildId = randomUUID();
@@ -518,13 +465,13 @@ describe('downloadBuild', () => {
     expect(jest.mocked(fetch)).not.toHaveBeenCalled();
   });
 
-  it('throws SystemError with the cause code after retrying a failing build lookup', async () => {
-    useFakeBackoffTimers();
+  it('throws SystemError when GraphQL request fails with a network error', async () => {
     const buildId = randomUUID();
     const graphqlClient = createMockGraphqlClient({
-      error: createUndiciNetworkError('ECONNRESET'),
+      error: new CombinedError({
+        networkError: new Error('Network request failed'),
+      }),
     });
-    const datadogLog = jest.spyOn(Datadog, 'log');
 
     const promise = downloadBuildAsync({
       logger: createLogger({ name: 'test' }),
@@ -533,81 +480,19 @@ describe('downloadBuild', () => {
       robotAccessToken: null,
       extensions: ['app'],
     });
-    const rejection = expect(promise).rejects.toBeInstanceOf(SystemError);
-    await fireBackoffTimersAsync(5);
 
-    await rejection;
+    await expect(promise).rejects.toBeInstanceOf(SystemError);
     await expect(promise).rejects.toMatchObject({
-      message: `Could not fetch build ${buildId}: [Network] fetch failed (ECONNRESET)`,
-    });
-    expect(graphqlClient.query).toHaveBeenCalledTimes(6);
-    expect(datadogLog).toHaveBeenCalledWith('download_build.build_lookup failed on attempt 6', {
-      event: 'network_retry',
-      operation: 'download_build.build_lookup',
-      outcome: 'failed',
-      attempts: '6',
-      code: 'ECONNRESET',
+      message: `Could not fetch build ${buildId}: [Network] Network request failed`,
     });
+
     expect(jest.mocked(fetch)).not.toHaveBeenCalled();
   });
 
-  it.each([
-    ['a network error', createUndiciNetworkError('ECONNRESET'), 'ECONNRESET'],
-    [
-      'a transient GraphQL error',
-      new CombinedError({
-        graphQLErrors: [
-          {
-            message: 'Too many requests',
-            extensions: { isTransient: true, errorCode: 'TOO_MANY_REQUESTS' },
-          },
-        ],
-        response: { status: 200 },
-      }),
-      'TOO_MANY_REQUESTS',
-    ],
-  ])('downloads after a build lookup that failed once with %s', async (_, transientError, code) => {
-    useFakeBackoffTimers();
-    const buildId = randomUUID();
-    const graphqlClient = createMockGraphqlClient({
-      applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
-      transientError,
-    });
-    jest
-      .mocked(fetch)
-      .mockResolvedValue(
-        createSuccessfulResponse({ body: Buffer.from('hello'), url: 'https://cdn.test/app.apk' })
-      );
-
-    const datadogLog = jest.spyOn(Datadog, 'log');
-
-    const promise = downloadBuildAsync({
-      logger: createLogger({ name: 'test' }),
-      buildId,
-      graphqlClient,
-      robotAccessToken: null,
-      extensions: ['apk'],
-    });
-    await fireBackoffTimersAsync(1);
-
-    const { artifactPath } = await promise;
-    expect(path.basename(artifactPath)).toBe('app.apk');
-    expect(graphqlClient.query).toHaveBeenCalledTimes(2);
-    expect(datadogLog).toHaveBeenCalledWith('download_build.build_lookup recovered on attempt 2', {
-      event: 'network_retry',
-      operation: 'download_build.build_lookup',
-      outcome: 'recovered',
-      attempts: '2',
-      code,
-    });
-  });
-
   it('throws SystemError when GraphQL request returns 5xx', async () => {
-    useFakeBackoffTimers();
     const buildId = randomUUID();
     const graphqlClient = createMockGraphqlClient({
-      error: new CombinedError({
-        graphQLErrors: [{ message: 'Internal Server Error' }],
+      error: Object.assign(new Error('Internal Server Error'), {
         response: { status: 500 },
       }),
     });
@@ -619,14 +504,12 @@ describe('downloadBuild', () => {
       robotAccessToken: null,
       extensions: ['app'],
     });
-    const rejection = expect(promise).rejects.toBeInstanceOf(SystemError);
-    await fireBackoffTimersAsync(5);
 
-    await rejection;
+    await expect(promise).rejects.toBeInstanceOf(SystemError);
     await expect(promise).rejects.toMatchObject({
-      message: `Could not fetch build ${buildId}: [GraphQL] Internal Server Error`,
+      message: `Could not fetch build ${buildId}: Internal Server Error`,
     });
-    expect(graphqlClient.query).toHaveBeenCalledTimes(6);
+
     expect(jest.mocked(fetch)).not.toHaveBeenCalled();
   });
 
@@ -651,193 +534,10 @@ describe('downloadBuild', () => {
       errorCode: 'EAS_DOWNLOAD_BUILD_FETCH_FAILED',
       message: `Could not fetch build ${buildId}: [GraphQL] Build not found`,
     });
-    expect(graphqlClient.query).toHaveBeenCalledTimes(1);
+
     expect(jest.mocked(fetch)).not.toHaveBeenCalled();
   });
 
-  it('downloads after an archive request that failed once with a socket error', async () => {
-    useFakeBackoffTimers();
-    const graphqlClient = createMockGraphqlClient({});
-    jest
-      .mocked(fetch)
-      .mockRejectedValueOnce(createSocketFetchError(APPLICATION_ARCHIVE_URL, 'ECONNRESET'))
-      .mockResolvedValueOnce(
-        createSuccessfulResponse({ body: Buffer.from('hello'), url: 'https://cdn.test/app.apk' })
-      );
-
-    const promise = downloadBuildAsync({
-      logger: createLogger({ name: 'test' }),
-      applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
-      graphqlClient,
-      robotAccessToken: null,
-      extensions: ['apk'],
-    });
-    await fireBackoffTimersAsync(1);
-
-    const { artifactPath } = await promise;
-    expect(path.basename(artifactPath)).toBe('app.apk');
-    expect(jest.mocked(fetch)).toHaveBeenCalledTimes(2);
-  });
-
-  it('throws SystemError with the cause code after retrying a failing build archive request', async () => {
-    useFakeBackoffTimers();
-    const graphqlClient = createMockGraphqlClient({
-      applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
-    });
-    jest
-      .mocked(fetch)
-      .mockRejectedValue(createSocketFetchError(APPLICATION_ARCHIVE_URL, 'ETIMEDOUT'));
-    const datadogLog = jest.spyOn(Datadog, 'log');
-
-    const promise = downloadBuildAsync({
-      logger: createLogger({ name: 'test' }),
-      buildId: randomUUID(),
-      graphqlClient,
-      robotAccessToken: null,
-      extensions: ['apk'],
-    });
-    const rejection = expect(promise).rejects.toBeInstanceOf(SystemError);
-    await fireBackoffTimersAsync(5);
-
-    await rejection;
-    await expect(promise).rejects.toMatchObject({
-      message: `Could not download the application archive: request to ${APPLICATION_ARCHIVE_URL} failed, reason: connect ETIMEDOUT`,
-    });
-    expect(jest.mocked(fetch)).toHaveBeenCalledTimes(6);
-    expect(datadogLog).toHaveBeenCalledWith('download_build.archive_download failed on attempt 6', {
-      event: 'network_retry',
-      operation: 'download_build.archive_download',
-      outcome: 'failed',
-      attempts: '6',
-      code: 'ETIMEDOUT',
-    });
-  });
-
-  function createBrokenBody(code: string): Readable {
-    return Readable.from(
-      (async function* () {
-        yield Buffer.from('he');
-        throw Object.assign(new Error(`body failed with ${code}`), { code });
-      })()
-    );
-  }
-
-  it.each(['ERR_STREAM_PREMATURE_CLOSE', 'ECONNRESET'])(
-    'downloads again when the archive body breaks off with %s after the headers arrived',
-    async code => {
-      useFakeBackoffTimers();
-      const graphqlClient = createMockGraphqlClient({});
-      const brokenBody = createBrokenBody(code);
-      jest
-        .mocked(fetch)
-        .mockResolvedValueOnce(
-          createSuccessfulResponse({ body: brokenBody, url: 'https://cdn.test/app.apk' })
-        )
-        .mockResolvedValueOnce(
-          createSuccessfulResponse({ body: Buffer.from('hello'), url: 'https://cdn.test/app.apk' })
-        );
-      const datadogLog = jest.spyOn(Datadog, 'log');
-
-      const promise = downloadBuildAsync({
-        logger: createLogger({ name: 'test' }),
-        applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
-        graphqlClient,
-        robotAccessToken: null,
-        extensions: ['apk'],
-      });
-      await fireBackoffTimersAsync(1);
-
-      const { artifactPath } = await promise;
-      expect(await fs.promises.readFile(artifactPath, 'utf-8')).toBe('hello');
-      expect(jest.mocked(fetch)).toHaveBeenCalledTimes(2);
-      expect(datadogLog).toHaveBeenCalledWith(
-        'download_build.archive_download recovered on attempt 2',
-        expect.objectContaining({ outcome: 'recovered', code })
-      );
-    }
-  );
-
-  it('throws SystemError when the build archive body keeps breaking off', async () => {
-    useFakeBackoffTimers();
-    const graphqlClient = createMockGraphqlClient({
-      applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
-    });
-    jest.mocked(fetch).mockImplementation(async () =>
-      createSuccessfulResponse({
-        body: createBrokenBody('ECONNRESET'),
-        url: 'https://cdn.test/app.apk',
-      })
-    );
-    const datadogLog = jest.spyOn(Datadog, 'log');
-
-    const promise = downloadBuildAsync({
-      logger: createLogger({ name: 'test' }),
-      buildId: randomUUID(),
-      graphqlClient,
-      robotAccessToken: null,
-      extensions: ['apk'],
-    });
-    const rejection = expect(promise).rejects.toBeInstanceOf(SystemError);
-    await fireBackoffTimersAsync(5);
-
-    await rejection;
-    await expect(promise).rejects.toMatchObject({
-      message: 'Could not download the application archive: body failed with ECONNRESET',
-    });
-    expect(jest.mocked(fetch)).toHaveBeenCalledTimes(6);
-    expect(datadogLog).toHaveBeenCalledWith(
-      'download_build.archive_download failed on attempt 6',
-      expect.objectContaining({ outcome: 'failed', code: 'ECONNRESET' })
-    );
-  });
-
-  it('does not start another archive attempt after an abort during the backoff', async () => {
-    useFakeBackoffTimers();
-    const graphqlClient = createMockGraphqlClient({});
-    jest
-      .mocked(fetch)
-      .mockRejectedValueOnce(createSocketFetchError(APPLICATION_ARCHIVE_URL, 'ECONNRESET'));
-    const controller = new AbortController();
-    const reason = new Error('step timed out');
-
-    const promise = downloadBuildAsync({
-      logger: createLogger({ name: 'test' }),
-      applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
-      graphqlClient,
-      robotAccessToken: null,
-      extensions: ['apk'],
-      signal: controller.signal,
-    });
-    const rejection = expect(promise).rejects.toBe(reason);
-    await waitForBackoffTimerAsync();
-    controller.abort(reason);
-    await jest.advanceTimersToNextTimerAsync();
-
-    await rejection;
-    expect(jest.mocked(fetch)).toHaveBeenCalledTimes(1);
-  });
-
-  it('rethrows the raw error of a failing user-supplied archive URL after the retries', async () => {
-    useFakeBackoffTimers();
-    const graphqlClient = createMockGraphqlClient({});
-    jest
-      .mocked(fetch)
-      .mockRejectedValue(createSocketFetchError(APPLICATION_ARCHIVE_URL, 'ENOTFOUND'));
-
-    const promise = downloadBuildAsync({
-      logger: createLogger({ name: 'test' }),
-      applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
-      graphqlClient,
-      robotAccessToken: null,
-      extensions: ['apk'],
-    });
-    const rejection = expect(promise).rejects.toBeInstanceOf(ActualFetchError);
-    await fireBackoffTimersAsync(5);
-
-    await rejection;
-    expect(jest.mocked(fetch)).toHaveBeenCalledTimes(6);
-  });
-
   it('should throw an error if no matching files are found', async () => {
     const graphqlClient = createMockGraphqlClient({
       applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
@@ -905,21 +605,6 @@ describe('createDownloadBuildFunction', () => {
       };
     }
 
-    function createApplicationResponse(buildId: string): Response {
-      return new ActualResponse(
-        JSON.stringify({
-          data: {
-            build: {
-              id: buildId,
-              gitCommitHash: null,
-              applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
-            },
-          },
-        }),
-        { status: 200 }
-      );
-    }
-
     function createStep(ctx: any, callInputs: Record<string, unknown>) {
       return createDownloadBuildFunction(ctx).createBuildStepFromFunctionCall(
         createGlobalContextMock({
@@ -935,7 +620,18 @@ describe('createDownloadBuildFunction', () => {
       const graphqlClient = createMockGraphqlClient({});
       jest.mocked(fetch).mockImplementation(async url => {
         if (url === workerApiUrl) {
-          return createApplicationResponse(buildId);
+          return new ActualResponse(
+            JSON.stringify({
+              data: {
+                build: {
+                  id: buildId,
+                  gitCommitHash: null,
+                  applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
+                },
+              },
+            }),
+            { status: 200 }
+          );
         }
         return {
           ok: false,
@@ -965,33 +661,6 @@ describe('createDownloadBuildFunction', () => {
       expect(graphqlClient.query).not.toHaveBeenCalled();
     });
 
-    it('retries the worker API lookup after a 5xx response', async () => {
-      useFakeBackoffTimers();
-      const buildId = randomUUID();
-      const graphqlClient = createMockGraphqlClient({});
-      jest
-        .mocked(fetch)
-        .mockResolvedValueOnce(new ActualResponse('Service Unavailable', { status: 503 }))
-        .mockResolvedValueOnce(createApplicationResponse(buildId))
-        .mockResolvedValueOnce(
-          createSuccessfulResponse({ body: Buffer.from('hello'), url: 'https://cdn.test/app.apk' })
-        );
-
-      const execution = createStep(createSessionCtx(graphqlClient), {
-        build_id: buildId,
-        extensions: ['apk'],
-      }).executeAsync();
-      await fireBackoffTimersAsync(1);
-
-      await execution;
-      expect(jest.mocked(fetch)).toHaveBeenCalledTimes(3);
-      expect(jest.mocked(fetch).mock.calls.map(([url]) => url)).toEqual([
-        workerApiUrl,
-        workerApiUrl,
-        APPLICATION_ARCHIVE_URL,
-      ]);
-    });
-
     it('does not call the worker API for an application_archive_url input', async () => {
       const graphqlClient = createMockGraphqlClient({});
       jest.mocked(fetch).mockResolvedValue({

packages/build-tools/src/steps/functions/downloadBuild.ts

@@ -7,7 +7,7 @@ import {
   BuildStepInputValueTypeName,
   BuildStepOutput,
 } from '@expo/steps';
-import { Client, CombinedError } from '@urql/core';
+import { Client } from '@urql/core';
 import contentDisposition from 'content-disposition';
 import { glob } from 'fast-glob';
 import { graphql } from 'gql.tada';
@@ -26,16 +26,10 @@ import { CustomBuildContext } from '../../customBuildContext';
 import { formatBytes } from '../../utils/artifacts';
 import { decompressTarAsync, isFileTarGzAsync } from '../../utils/files';
 import { graphqlAbortContext } from '../../utils/graphqlAbort';
-import {
-  describeError,
-  isTransferFailure,
-  isTransientGraphqlError,
-} from '../../utils/networkErrors';
-import { retryOnNetworkFailureAsync } from '../../utils/retryOnNetworkFailure';
+import { retryOnDNSFailure } from '../../utils/retryOnDNSFailure';
 import { pluralize } from '../../utils/strings';
 import {
   type DeviceRunSessionWorkerApi,
-  DeviceRunSessionWorkerApiError,
   getDeviceRunSessionWorkerApi,
 } from '../utils/deviceRunSessionWorkerApi';
 
@@ -158,34 +152,21 @@ async function fetchApplicationArchiveUrlAsync({
       signal,
     });
   }
-  let data;
-  try {
-    data = await retryOnNetworkFailureAsync(
-      async () => {
-        const queryResult = await graphqlClient
-          .query(BUILD_BY_ID_QUERY, { buildId }, graphqlAbortContext(signal))
-          .toPromise();
-        if (queryResult.error) {
-          throw queryResult.error;
-        }
-        return queryResult.data;
-      },
-      {
-        operation: 'download_build.build_lookup',
-        signal,
-        isRetryable: error => error instanceof CombinedError && isTransientGraphqlError(error),
-      }
-    );
-  } catch (error) {
-    signal?.throwIfAborted();
-    const message = `Could not fetch build ${buildId}: ${describeError(error)}`;
-    throw error instanceof CombinedError && !isTransientGraphqlError(error)
-      ? new UserError('EAS_DOWNLOAD_BUILD_FETCH_FAILED', message, { cause: error })
-      : new SystemError(message, { cause: error });
-  }
+  const result = await graphqlClient
+    .query(BUILD_BY_ID_QUERY, { buildId }, graphqlAbortContext(signal))
+    .toPromise();
   signal?.throwIfAborted();
 
-  const applicationArchiveUrl = data?.builds.byId?.artifacts?.applicationArchiveUrl;
+  if (result.error) {
+    const { error } = result;
+    const message = `Could not fetch build ${buildId}: ${error.message}`;
+
+    throw error.networkError || result.error.response?.status >= 500
+      ? new SystemError(message, { cause: error })
+      : new UserError('EAS_DOWNLOAD_BUILD_FETCH_FAILED', message, { cause: error });
+  }
+
+  const applicationArchiveUrl = result.data?.builds.byId?.artifacts?.applicationArchiveUrl;
   if (!applicationArchiveUrl) {
     throw new UserError(
       'EAS_DOWNLOAD_BUILD_NO_APPLICATION_ARCHIVE',
@@ -214,18 +195,13 @@ async function fetchDeviceRunSessionApplicationArchiveUrlAsync({
 }): Promise<string> {
   let application;
   try {
-    application = await retryOnNetworkFailureAsync(
-      () => deviceRunSessionWorkerApi.getApplicationAsync({ signal }),
-      {
-        operation: 'download_build.session_build_lookup',
-        signal,
-        isRetryable: error => error instanceof DeviceRunSessionWorkerApiError && error.isTransient,
-      }
-    );
+    application = await deviceRunSessionWorkerApi.getApplicationAsync({ signal });
   } catch (err) {
     signal?.throwIfAborted();
     throw new SystemError(
-      `Could not fetch build ${buildId} for this device run session: ${describeError(err)}`,
+      `Could not fetch build ${buildId} for this device run session: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
       { cause: err }
     );
   }
@@ -305,34 +281,18 @@ export async function downloadBuildAsync(
   );
 
   signal?.throwIfAborted();
-  let archivePath: string;
-  try {
-    archivePath = await retryOnNetworkFailureAsync(
-      async () => {
-        const response = await fetch(downloadUrl, { headers, signal });
-        if (!response.ok) {
-          const textResult = await asyncResult(response.text());
-          throw new Error(
-            `Unexpected response from server (${response.status}): ${textResult.value}`
-          );
-        }
-        const archiveFilename = resolveArchiveFilename({ response, extensions });
-        const attemptArchivePath = path.join(downloadDestinationDirectory, archiveFilename);
-        // On abort, node-fetch emits an error on the body stream, which fails the pipeline.
-        await streamPipeline(response.body, fs.createWriteStream(attemptArchivePath));
-        return attemptArchivePath;
-      },
-      { operation: 'download_build.archive_download', signal, isRetryable: isTransferFailure }
-    );
-  } catch (error) {
-    signal?.throwIfAborted();
-    if (params.buildId && isTransferFailure(error)) {
-      throw new SystemError(`Could not download the application archive: ${describeError(error)}`, {
-        cause: error,
-      });
-    }
-    throw error;
+  const response = await retryOnDNSFailure(fetch)(downloadUrl, { headers, signal });
+
+  if (!response.ok) {
+    const textResult = await asyncResult(response.text());
+    throw new Error(`Unexpected response from server (${response.status}): ${textResult.value}`);
   }
+
+  const archiveFilename = resolveArchiveFilename({ response, extensions });
+  const archivePath = path.join(downloadDestinationDirectory, archiveFilename);
+
+  // On abort, node-fetch emits an error on the body stream, which fails the pipeline.
+  await streamPipeline(response.body, fs.createWriteStream(archivePath));
   signal?.throwIfAborted();
 
   const { size } = await fs.promises.stat(archivePath);

packages/build-tools/src/steps/utils/__tests__/deviceRunSessionWorkerApi.test.ts

@@ -330,8 +330,7 @@ describe(DeviceRunSessionWorkerApi, () => {
   });
 
   it('reports a request that failed before www responded', async () => {
-    const reason = new Error('socket hang up');
-    jest.mocked(turtleFetch).mockRejectedValueOnce(reason);
+    jest.mocked(turtleFetch).mockRejectedValueOnce(new Error('socket hang up'));
 
     const error = await createApi()
       .getSessionAsync()
@@ -339,7 +338,6 @@ describe(DeviceRunSessionWorkerApi, () => {
 
     expect(error).toBeInstanceOf(DeviceRunSessionWorkerApiError);
     expect(error.status).toBeUndefined();
-    expect(error.cause).toBe(reason);
     expect(error.message).toContain('failed before www responded: socket hang up');
   });
 

packages/build-tools/src/steps/utils/deviceRunSessionWorkerApi.ts

@@ -88,16 +88,11 @@ export class DeviceRunSessionWorkerApiError extends Error {
   constructor(
     message: string,
     /** The HTTP status of the response, or undefined when no response arrived. */
-    readonly status: number | undefined,
-    options?: ErrorOptions
+    readonly status: number | undefined
   ) {
-    super(message, options);
+    super(message);
     this.name = 'DeviceRunSessionWorkerApiError';
   }
-
-  get isTransient(): boolean {
-    return this.status === undefined || this.status >= 500;
-  }
 }
 
 /**
@@ -223,8 +218,7 @@ export class DeviceRunSessionWorkerApi {
         `${routeDescription} failed before www responded: ${
           reason instanceof Error ? reason.message : String(reason)
         }. Check the network connection of the worker.`,
-        undefined,
-        { cause: reason }
+        undefined
       );
     }
     const response = responseResult.value;

packages/build-tools/src/utils/__tests__/networkErrors.test.ts

@@ -1,16 +1,4 @@
-import { CombinedError } from '@urql/core';
-
-import {
-  describeError,
-  findErrorCode,
-  findGraphqlErrorCode,
-  isConnectFailure,
-  isConnectionInterruptedError,
-  isTransferFailure,
-  isTransientGraphqlError,
-} from '../networkErrors';
-
-const { FetchError } = jest.requireActual('node-fetch') as typeof import('node-fetch');
+import { isConnectFailure, isConnectionInterruptedError } from '../networkErrors';
 
 it.each([
   'EAI_AGAIN',
@@ -46,77 +34,3 @@ it.each(['ETIMEDOUT', 'CERT_HAS_EXPIRED', undefined])('does not automatically re
   expect(isConnectFailure(error)).toBe(false);
   expect(isConnectionInterruptedError(error)).toBe(false);
 });
-
-it('finds the errno code on the cause of an undici fetch failure', () => {
-  const error = new CombinedError({
-    networkError: new TypeError('fetch failed', {
-      cause: Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }),
-    }),
-  });
-  expect(findErrorCode(error)).toBe('ECONNRESET');
-  expect(findErrorCode(new Error('Request failed'))).toBeUndefined();
-  expect(findErrorCode('Request failed')).toBeUndefined();
-});
-
-it('finds the www error code on a GraphQL error', () => {
-  const error = new CombinedError({
-    graphQLErrors: [
-      { message: 'Too many requests', extensions: { errorCode: 'TOO_MANY_REQUESTS' } },
-    ],
-  });
-  expect(findGraphqlErrorCode(error)).toBe('TOO_MANY_REQUESTS');
-  expect(
-    findGraphqlErrorCode(new CombinedError({ graphQLErrors: ['Build not found'] }))
-  ).toBeUndefined();
-  expect(findGraphqlErrorCode(new Error('Request failed'))).toBeUndefined();
-});
-
-it.each([
-  ['Request failed', 'ECONNRESET', 'Request failed (ECONNRESET)'],
-  ['read ECONNRESET', 'ECONNRESET', 'read ECONNRESET'],
-  ['Request failed', undefined, 'Request failed'],
-])('describes %p with code %p as %p', (message, code, expected) => {
-  expect(describeError(Object.assign(new Error(message), { code }))).toBe(expected);
-});
-
-it('treats a missing response, a 5xx response or an isTransient GraphQL error as transient', () => {
-  expect(
-    isTransientGraphqlError(new CombinedError({ networkError: new Error('fetch failed') }))
-  ).toBe(true);
-  expect(
-    isTransientGraphqlError(
-      new CombinedError({ graphQLErrors: ['Internal Server Error'], response: { status: 503 } })
-    )
-  ).toBe(true);
-  expect(
-    isTransientGraphqlError(
-      new CombinedError({
-        graphQLErrors: [{ message: 'Database unavailable', extensions: { isTransient: true } }],
-        response: { status: 200 },
-      })
-    )
-  ).toBe(true);
-  expect(
-    isTransientGraphqlError(
-      new CombinedError({ graphQLErrors: ['Build not found'], response: { status: 200 } })
-    )
-  ).toBe(false);
-});
-
-it.each([
-  [
-    'a node-fetch socket failure',
-    new FetchError('request failed', 'system', { code: 'ECONNRESET' } as any),
-    true,
-  ],
-  ['a node-fetch body size failure', new FetchError('too large', 'max-size'), false],
-  ['a body reset', Object.assign(new Error('aborted'), { code: 'ECONNRESET' }), true],
-  [
-    'a body closed early',
-    Object.assign(new Error('closed'), { code: 'ERR_STREAM_PREMATURE_CLOSE' }),
-    true,
-  ],
-  ['an HTTP error response', new Error('Unexpected response from server (500)'), false],
-])('classifies %s as a transfer failure or not', (_label, error, expected) => {
-  expect(isTransferFailure(error)).toBe(expected);
-});

packages/build-tools/src/utils/networkErrors.ts

@@ -1,5 +1,3 @@
-import { CombinedError } from '@urql/core';
-
 const CONNECT_FAILURE_CODES = new Set([
   'EAI_AGAIN',
   'ECONNREFUSED',
@@ -32,50 +30,3 @@ export function isConnectionInterruptedError(error: unknown): boolean {
   }
   return false;
 }
-
-/** Errno code of the error or its causes; undici keeps it on cause, urql on networkError. */
-export function findErrorCode(error: unknown): string | undefined {
-  let current: unknown = error;
-  while (current instanceof Error) {
-    if ('code' in current && typeof current.code === 'string') {
-      return current.code;
-    }
-    current = current instanceof CombinedError ? current.networkError : current.cause;
-  }
-  return undefined;
-}
-
-export function findGraphqlErrorCode(error: unknown): string | undefined {
-  if (!(error instanceof CombinedError)) {
-    return undefined;
-  }
-  const code = error.graphQLErrors.at(0)?.extensions.errorCode;
-  return typeof code === 'string' ? code : undefined;
-}
-
-export function describeError(error: unknown): string {
-  const message = error instanceof Error ? error.message : String(error);
-  const code = findErrorCode(error);
-  return code && !message.includes(code) ? `${message} (${code})` : message;
-}
-
-export function isTransientGraphqlError(error: CombinedError): boolean {
-  return (
-    Boolean(error.networkError) ||
-    error.response?.status >= 500 ||
-    error.graphQLErrors.some(graphqlError => graphqlError.extensions.isTransient === true)
-  );
-}
-
-// node-fetch throws type 'system' for a socket failure before the headers; after them it forwards
-// ECONNRESET or ERR_STREAM_PREMATURE_CLOSE to the body. HTTP error responses never throw.
-export function isTransferFailure(error: unknown): boolean {
-  return (
-    (error instanceof Error &&
-      error.name === 'FetchError' &&
-      'type' in error &&
-      error.type === 'system') ||
-    isConnectionInterruptedError(error) ||
-    findErrorCode(error) === 'ERR_STREAM_PREMATURE_CLOSE'
-  );
-}

packages/build-tools/src/utils/retryOnNetworkFailure.ts

@@ -1,53 +0,0 @@
-import { Datadog } from '../datadog';
-import { findErrorCode, findGraphqlErrorCode } from './networkErrors';
-import { promiseRetryWithCondition } from './promiseRetryWithCondition';
-
-export async function retryOnNetworkFailureAsync<T>(
-  attemptAsync: () => Promise<T>,
-  {
-    operation,
-    signal,
-    isRetryable,
-  }: {
-    operation: string;
-    signal?: AbortSignal;
-    isRetryable: (error: unknown) => boolean;
-  }
-): Promise<T> {
-  const shouldRetry = (error: unknown): boolean => !signal?.aborted && isRetryable(error);
-  let attempts = 1;
-  let lastError: unknown;
-  const report = (outcome: 'recovered' | 'failed', error: unknown): void => {
-    Datadog.log(`${operation} ${outcome} on attempt ${attempts}`, {
-      event: 'network_retry',
-      operation,
-      outcome,
-      attempts: String(attempts),
-      code: findErrorCode(error) ?? findGraphqlErrorCode(error) ?? 'unknown',
-    });
-  };
-
-  try {
-    const result = await promiseRetryWithCondition(
-      async () => {
-        signal?.throwIfAborted();
-        return await attemptAsync();
-      },
-      shouldRetry,
-      { retries: 5, minTimeout: 1_000, factor: 2 },
-      ({ attemptNumber, error }) => {
-        attempts = attemptNumber;
-        lastError = error;
-      }
-    )();
-    if (attempts > 1) {
-      report('recovered', lastError);
-    }
-    return result;
-  } catch (error) {
-    if (attempts > 1 && !signal?.aborted) {
-      report('failed', error);
-    }
-    throw error;
-  }
-}

packages/eas-cli/src/__tests__/uploads-test.ts

@@ -3,8 +3,56 @@ import os from 'node:os';
 import path from 'node:path';
 
 import fs from 'fs-extra';
+import nock from 'nock';
 
-import { computeFileMd5Base64Async } from '../uploads';
+import { computeFileMd5Base64Async, uploadWithPresignedPostWithRetryAsync } from '../uploads';
+
+describe(uploadWithPresignedPostWithRetryAsync, () => {
+  const presignedPost = {
+    url: 'https://storage.example.com/upload',
+    fields: { key: 'object-key' },
+  };
+  const contents = Array.from({ length: 50000 }, (_, i) => `line ${i}\n`).join('');
+  let tmpDir: string;
+  let file: string;
+
+  beforeAll(async () => {
+    tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'eas-cli-presigned-post-test-'));
+    file = path.join(tmpDir, 'bundle.js');
+    await fs.writeFile(file, contents);
+  });
+
+  afterAll(async () => {
+    await fs.remove(tmpDir);
+  });
+
+  afterEach(() => {
+    nock.cleanAll();
+  });
+
+  it('sends the whole file and reports activity while it uploads', async () => {
+    let body = '';
+    nock('https://storage.example.com')
+      .post('/upload', requestBody => {
+        body = requestBody;
+        return true;
+      })
+      .reply(204);
+    const onAssetUploadBegin = jest.fn();
+
+    await uploadWithPresignedPostWithRetryAsync(file, presignedPost, onAssetUploadBegin);
+
+    expect(body).toContain('filename="bundle.js"');
+    expect(body).toContain(contents);
+    expect(onAssetUploadBegin.mock.calls.length).toBeGreaterThan(2);
+    expect(onAssetUploadBegin).toHaveBeenCalledWith(
+      expect.objectContaining({ file, fileSize: contents.length, uploadedBytes: 0 })
+    );
+    expect(onAssetUploadBegin).toHaveBeenLastCalledWith(
+      expect.objectContaining({ file, fileSize: contents.length, uploadedBytes: contents.length })
+    );
+  });
+});
 
 describe(computeFileMd5Base64Async, () => {
   let tmpDir: string;

packages/eas-cli/src/commands/update/__tests__/index.test.ts

@@ -61,6 +61,7 @@ jest.mock('../../../graphql/queries/UpdateQuery');
 jest.mock('../../../prompts');
 jest.mock('../../../graphql/queries/EnvironmentVariablesQuery');
 jest.mock('../../../ora', () => ({
+  isSpinnerEnabled: () => false,
   ora: () => ({
     start: () => ({ succeed: () => {}, fail: () => {}, stop: () => {} }),
   }),

packages/eas-cli/src/commands/update/index.ts

@@ -3,6 +3,7 @@ import { EasJson, EasJsonAccessor, EasJsonUtils } from '@expo/eas-json';
 import { Errors, Flags } from '@oclif/core';
 import chalk from 'chalk';
 import nullthrows from 'nullthrows';
+import path from 'path';
 
 import { ensureBranchExistsAsync } from '../../branch/queries';
 import { ensureRepoIsCleanAsync } from '../../build/utils/repository';
@@ -28,7 +29,7 @@ import {
 } from '../../graphql/generated';
 import { PublishMutation } from '../../graphql/mutations/PublishMutation';
 import Log, { learnMore, link } from '../../log';
-import { ora } from '../../ora';
+import { isSpinnerEnabled, ora } from '../../ora';
 import { RequestedPlatform } from '../../platform';
 import { maybeUploadAssetMapAsync } from '../../project/maybeUploadAssetMapAsync';
 import {
@@ -56,6 +57,7 @@ import {
   resolveInputDirectoryAsync,
   uploadAssetsAsync,
 } from '../../project/publish';
+import { createUploadProgressReporter } from '../../project/uploadProgressReporter';
 import { resolveWorkflowPerPlatformAsync } from '../../project/workflow';
 import { resolveUpdateGroupsSupersedingActiveRolloutsAsync } from '../../update/active-rollout';
 import { ensureEASUpdateIsConfiguredAsync } from '../../update/configure';
@@ -389,13 +391,41 @@ export default class UpdatePublish extends EasCommand {
       // - Start with NO_ACTIVITY_TIMEOUT. 180 seconds is chosen because the cloud function that processes
       //   uploaded assets has a timeout of 60 seconds and uploading can take some time on a slow connection.
       // - Each time one or more assets reports as ready, reset the timeout.
-      // - Each time an asset upload begins, reset the timeout. This includes retries.
+      // - Each time an asset upload begins or sends more bytes, reset the timeout. This includes retries.
       // - Start upload. Internally, uploadAssetsAsync uploads them all first and then checks for successful
       //   processing every (5 + n) seconds with a linear backoff of n + 1 second.
       // - At the same time as upload is started, start timeout checker which checks every 1 second to see
       //   if timeout has been reached. When timeout expires, send a cancellation signal to currently running
       //   upload function call to instruct it to stop uploading or checking for successful processing.
       const NO_ACTIVITY_TIMEOUT = 180 * 1000; // 180 seconds
+      let uploadCountText = 'Uploading...';
+      let activeUploadLines: string[] = [];
+      let unfinishedAssetPaths: string[] = [];
+      const fullySentAssetPaths = new Set<string>();
+      const updateAssetSpinnerText = (): void => {
+        const processingAssetNames = unfinishedAssetPaths
+          .filter(assetPath => fullySentAssetPaths.has(assetPath))
+          .map(assetPath => path.basename(assetPath));
+        assetSpinner.text = [
+          uploadCountText,
+          ...activeUploadLines,
+          ...(processingAssetNames.length > 0
+            ? [
+                `Waiting for the server to process ${processingAssetNames.slice(0, 3).join(', ')}` +
+                  (processingAssetNames.length > 3
+                    ? ` and ${processingAssetNames.length - 3} more`
+                    : ''),
+              ]
+            : []),
+        ].join('\n');
+      };
+      const reportUploadProgress = createUploadProgressReporter({
+        isInteractive: isSpinnerEnabled(),
+        onActiveUploadsChanged: lines => {
+          activeUploadLines = lines;
+          updateAssetSpinnerText();
+        },
+      });
       let lastUploadedStorageKeys = new Set<string>();
       let lastAssetUploadResults: {
         asset: RawAsset & { storageKey: string };
@@ -422,11 +452,23 @@ export default class UpdatePublish extends EasCommand {
 
             const totalAssets = assetUploadResults.length;
             const missingAssetCount = assetUploadResults.filter(a => !a.finished).length;
-            assetSpinner.text = `Uploading (${totalAssets - missingAssetCount}/${totalAssets})`;
+            uploadCountText = `Uploading (${totalAssets - missingAssetCount}/${totalAssets})`;
+            unfinishedAssetPaths = assetUploadResults
+              .filter(r => !r.finished)
+              .map(r => r.asset.path);
+            updateAssetSpinnerText();
           },
-          () => {
+          progress => {
             // when an upload is retried, reset the timeout as we know this will now need more time
             timeAtWhichToTimeout = Date.now() + NO_ACTIVITY_TIMEOUT; // reset timeout to NO_ACTIVITY_TIMEOUT
+            if (progress) {
+              if (progress.uploadedBytes >= progress.fileSize) {
+                fullySentAssetPaths.add(progress.file);
+              } else {
+                fullySentAssetPaths.delete(progress.file);
+              }
+              reportUploadProgress(progress);
+            }
           }
         ),
         (async () => {

packages/eas-cli/src/project/__tests__/uploadProgressReporter-test.ts

@@ -0,0 +1,126 @@
+import Log from '../../log';
+import { UploadProgress } from '../../uploads';
+import { createUploadProgressReporter } from '../uploadProgressReporter';
+
+const MB = 1024 * 1024;
+
+function makeProgress(file: string, uploadedBytes: number, elapsedMs: number): UploadProgress {
+  return { file, fileSize: 7 * MB, uploadedBytes, elapsedMs };
+}
+
+describe(createUploadProgressReporter, () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  describe('interactive', () => {
+    it('shows a line for each active upload once it has run for 10 seconds', () => {
+      const onActiveUploadsChanged = jest.fn();
+      const report = createUploadProgressReporter({ isInteractive: true, onActiveUploadsChanged });
+
+      report(makeProgress('/dist/ios.hbc', MB, 5_000));
+      expect(onActiveUploadsChanged).toHaveBeenLastCalledWith([]);
+
+      report(makeProgress('/dist/ios.hbc', MB, 20_000));
+      report(makeProgress('/dist/android.hbc', 6 * MB, 10_000));
+
+      expect(onActiveUploadsChanged).toHaveBeenLastCalledWith([
+        'ios.hbc: 1.0 MB of 7.0 MB, 51.2 KB/s, about 2 minutes left',
+        'android.hbc: 6.0 MB of 7.0 MB, 614 KB/s',
+      ]);
+    });
+
+    it('removes an upload once all of its bytes are sent', () => {
+      const onActiveUploadsChanged = jest.fn();
+      const report = createUploadProgressReporter({ isInteractive: true, onActiveUploadsChanged });
+
+      report(makeProgress('/dist/ios.hbc', MB, 20_000));
+      report(makeProgress('/dist/ios.hbc', 7 * MB, 30_000));
+
+      expect(onActiveUploadsChanged).toHaveBeenLastCalledWith([]);
+    });
+
+    it('shows at most 5 uploads', () => {
+      const onActiveUploadsChanged = jest.fn();
+      const report = createUploadProgressReporter({ isInteractive: true, onActiveUploadsChanged });
+
+      for (let i = 0; i < 7; i++) {
+        report(makeProgress(`/dist/asset-${i}.png`, MB, 20_000));
+      }
+
+      const lines = onActiveUploadsChanged.mock.calls.at(-1)?.[0];
+      expect(lines).toHaveLength(6);
+      expect(lines[5]).toBe('and 2 more uploads');
+    });
+
+    it('does not log', () => {
+      const warnSpy = jest.spyOn(Log, 'warn').mockImplementation(() => {});
+      const logSpy = jest.spyOn(Log, 'log').mockImplementation(() => {});
+      const report = createUploadProgressReporter({
+        isInteractive: true,
+        onActiveUploadsChanged: () => {},
+      });
+
+      report(makeProgress('/dist/ios.hbc', MB, 20_000));
+
+      expect(warnSpy).not.toHaveBeenCalled();
+      expect(logSpy).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('non-interactive', () => {
+    it('warns when an upload is slow and then logs its progress every minute', () => {
+      let now = 0;
+      jest.spyOn(Date, 'now').mockImplementation(() => now);
+      const warnSpy = jest.spyOn(Log, 'warn').mockImplementation(() => {});
+      const logSpy = jest.spyOn(Log, 'log').mockImplementation(() => {});
+      const report = createUploadProgressReporter({
+        isInteractive: false,
+        onActiveUploadsChanged: () => {},
+      });
+
+      report(makeProgress('/dist/ios.hbc', MB, 20_000));
+      now = 30_000;
+      report(makeProgress('/dist/ios.hbc', 2 * MB, 50_000));
+      now = 60_000;
+      report(makeProgress('/dist/ios.hbc', 3 * MB, 80_000));
+
+      expect(warnSpy).toHaveBeenCalledTimes(1);
+      expect(warnSpy).toHaveBeenCalledWith(
+        'Uploading ios.hbc (7.0 MB) is slow (51.2 KB/s). It may take about 2 minutes.'
+      );
+      expect(logSpy).toHaveBeenCalledTimes(1);
+      expect(logSpy).toHaveBeenCalledWith(
+        'Still uploading ios.hbc: 3.0 MB of 7.0 MB, 38.4 KB/s, about 2 minutes left.'
+      );
+    });
+
+    it('warns again when a retried upload is still slow', () => {
+      const warnSpy = jest.spyOn(Log, 'warn').mockImplementation(() => {});
+      const report = createUploadProgressReporter({
+        isInteractive: false,
+        onActiveUploadsChanged: () => {},
+      });
+
+      report(makeProgress('/dist/ios.hbc', MB, 20_000));
+      report(makeProgress('/dist/ios.hbc', 0, 0));
+      report(makeProgress('/dist/ios.hbc', MB, 20_000));
+
+      expect(warnSpy).toHaveBeenCalledTimes(2);
+    });
+
+    it('does not log when an upload is fast', () => {
+      const warnSpy = jest.spyOn(Log, 'warn').mockImplementation(() => {});
+      const logSpy = jest.spyOn(Log, 'log').mockImplementation(() => {});
+      const report = createUploadProgressReporter({
+        isInteractive: false,
+        onActiveUploadsChanged: () => {},
+      });
+
+      report(makeProgress('/dist/ios.hbc', 6 * MB, 10_000));
+
+      expect(warnSpy).not.toHaveBeenCalled();
+      expect(logSpy).not.toHaveBeenCalled();
+    });
+  });
+});

packages/eas-cli/src/project/publish.ts

@@ -44,7 +44,7 @@ import {
   truncateString as truncateUpdateMessage,
   updatePublishPlatformToAppPlatform,
 } from '../update/utils';
-import { PresignedPost, uploadWithPresignedPostWithRetryAsync } from '../uploads';
+import { PresignedPost, UploadProgress, uploadWithPresignedPostWithRetryAsync } from '../uploads';
 import {
   expoCommandAsync,
   shouldUseVersionedExpoCLI,
@@ -496,7 +496,7 @@ export async function uploadAssetsAsync(
   onAssetUploadResultsChanged: (
     assetUploadResults: { asset: RawAsset & { storageKey: string }; finished: boolean }[]
   ) => void,
-  onAssetUploadBegin: () => void
+  onAssetUploadBegin: (progress?: UploadProgress) => void
 ): Promise<AssetUploadResult> {
   let assets: RawAsset[] = [];
   let platform: keyof FilteredCollectedAssets;
@@ -552,7 +552,7 @@ export async function uploadAssetsAsync(
   // Sign URLs in chunks (endpoint caps at 100), then upload all from one pool so it stays saturated.
   const uploadBatchAsync = async (
     assetsToUpload: AssetWithStorageKey[],
-    onBegin: () => void
+    onBegin: (progress?: UploadProgress) => void
   ): Promise<void> => {
     const signedAssets: { asset: AssetWithStorageKey; specification: string }[] = [];
     for (const assetChunk of chunk(assetsToUpload, 100)) {

packages/eas-cli/src/project/uploadProgressReporter.ts

@@ -0,0 +1,91 @@
+import path from 'path';
+
+import Log from '../log';
+import { UploadProgress } from '../uploads';
+import { formatBytes } from '../utils/files';
+
+const MIN_UPLOAD_MS_BEFORE_REPORTING = 10_000;
+const SLOW_UPLOAD_SECONDS_LEFT = 60;
+const SLOW_UPLOAD_LOG_INTERVAL_MS = 60_000;
+const MAX_ACTIVE_UPLOAD_LINES = 5;
+
+function getBytesPerSecond(progress: UploadProgress): number {
+  return progress.uploadedBytes / (progress.elapsedMs / 1000);
+}
+
+function getSecondsLeft(progress: UploadProgress): number {
+  return (progress.fileSize - progress.uploadedBytes) / getBytesPerSecond(progress);
+}
+
+function formatMinutesLeft(progress: UploadProgress): string {
+  const minutes = Math.ceil(getSecondsLeft(progress) / 60);
+  return minutes === 1 ? '1 minute' : `${minutes} minutes`;
+}
+
+function formatUploadProgress(progress: UploadProgress): string {
+  const message =
+    `${path.basename(progress.file)}: ${formatBytes(progress.uploadedBytes)} of ${formatBytes(progress.fileSize)}, ` +
+    `${formatBytes(getBytesPerSecond(progress))}/s`;
+  return getSecondsLeft(progress) > SLOW_UPLOAD_SECONDS_LEFT
+    ? `${message}, about ${formatMinutesLeft(progress)} left`
+    : message;
+}
+
+function isSlowUpload(progress: UploadProgress): boolean {
+  return (
+    progress.elapsedMs >= MIN_UPLOAD_MS_BEFORE_REPORTING &&
+    getSecondsLeft(progress) > SLOW_UPLOAD_SECONDS_LEFT
+  );
+}
+
+export function createUploadProgressReporter({
+  isInteractive,
+  onActiveUploadsChanged,
+}: {
+  isInteractive: boolean;
+  onActiveUploadsChanged: (lines: string[]) => void;
+}): (progress: UploadProgress) => void {
+  const activeUploads = new Map<string, UploadProgress>();
+  const lastSlowUploadLogTimes = new Map<string, number>();
+
+  return progress => {
+    if (progress.uploadedBytes === 0) {
+      lastSlowUploadLogTimes.delete(progress.file);
+    }
+    if (progress.uploadedBytes >= progress.fileSize) {
+      activeUploads.delete(progress.file);
+    } else {
+      activeUploads.set(progress.file, progress);
+    }
+
+    if (isInteractive) {
+      const lines = [...activeUploads.values()]
+        .filter(upload => upload.elapsedMs >= MIN_UPLOAD_MS_BEFORE_REPORTING)
+        .map(formatUploadProgress);
+      onActiveUploadsChanged(
+        lines.length > MAX_ACTIVE_UPLOAD_LINES
+          ? [
+              ...lines.slice(0, MAX_ACTIVE_UPLOAD_LINES),
+              `and ${lines.length - MAX_ACTIVE_UPLOAD_LINES} more uploads`,
+            ]
+          : lines
+      );
+      return;
+    }
+
+    if (!isSlowUpload(progress)) {
+      return;
+    }
+    const lastLogTime = lastSlowUploadLogTimes.get(progress.file);
+    if (lastLogTime === undefined) {
+      Log.warn(
+        `Uploading ${path.basename(progress.file)} (${formatBytes(progress.fileSize)}) is slow ` +
+          `(${formatBytes(getBytesPerSecond(progress))}/s). It may take about ${formatMinutesLeft(progress)}.`
+      );
+      lastSlowUploadLogTimes.set(progress.file, Date.now());
+    } else if (Date.now() - lastLogTime >= SLOW_UPLOAD_LOG_INTERVAL_MS) {
+      Log.log(`Still uploading ${formatUploadProgress(progress)}.`);
+      lastSlowUploadLogTimes.set(progress.file, Date.now());
+    }
+  };
+}

packages/eas-cli/src/uploads.ts

@@ -20,6 +20,13 @@ export interface PresignedPost {
   fields: Record<string, string>;
 }
 
+export interface UploadProgress {
+  file: string;
+  fileSize: number;
+  uploadedBytes: number;
+  elapsedMs: number;
+}
+
 export async function uploadFileAtPathToGCSAsync(
   graphqlClient: ExpoGraphqlClient,
   type: UploadSessionType,
@@ -88,7 +95,7 @@ export async function uploadAppScopedFileAtPathToGCSAsync(
 export async function uploadWithPresignedPostWithRetryAsync(
   file: string,
   presignedPost: PresignedPost,
-  onAssetUploadBegin: () => void
+  onAssetUploadBegin: (progress?: UploadProgress) => void
 ): Promise<Response> {
   return await promiseRetry(
     async retry => {
@@ -96,7 +103,7 @@ export async function uploadWithPresignedPostWithRetryAsync(
       let response: Response;
       try {
         onAssetUploadBegin();
-        response = await uploadWithPresignedPostAsync(file, presignedPost);
+        response = await uploadWithPresignedPostAsync(file, presignedPost, onAssetUploadBegin);
       } catch (e: any) {
         return retry(e);
       }
@@ -127,7 +134,8 @@ export async function uploadWithPresignedPostWithRetryAsync(
 
 async function uploadWithPresignedPostAsync(
   file: string,
-  presignedPost: PresignedPost
+  presignedPost: PresignedPost,
+  onProgress: (progress: UploadProgress) => void
 ): Promise<Response> {
   const fileStat = await fs.stat(file);
   const fileSize = fileStat.size;
@@ -135,7 +143,15 @@ async function uploadWithPresignedPostAsync(
   for (const [fieldKey, fieldValue] of Object.entries(presignedPost.fields)) {
     form.append(fieldKey, fieldValue);
   }
-  form.append('file', fs.createReadStream(file), { knownLength: fileSize });
+  const readStream = fs.createReadStream(file);
+  form.append('file', readStream, { knownLength: fileSize });
+  const startTime = Date.now();
+  let uploadedBytes = 0;
+  onProgress({ file, fileSize, uploadedBytes, elapsedMs: 0 });
+  readStream.addListener('data', (chunk: Buffer | string) => {
+    uploadedBytes += Buffer.byteLength(chunk);
+    onProgress({ file, fileSize, uploadedBytes, elapsedMs: Date.now() - startTime });
+  });
   const formHeaders = form.getHeaders();
   return await fetch(presignedPost.url, {
     method: 'POST',

packages/worker/README.md

@@ -42,18 +42,3 @@ Github action builds a docker image for every PR with any change in `infra/eas-b
 ### VM template
 
 Github action builds a new vm template for every PR with any change in `infra/eas-build-worker-images/ios/{{ templateName }}`.
-
-### Packaged startup
-
-Existing services run `src/services/worker/dist/main.js`. In the worker archive,
-that path points to `bin/bootstrap.cjs`. The bootstrap loads only Node built-ins
-and starts `bin/start-worker.sh`. The shell then uses `exec` to start the normal
-worker with the original Node binary, options, and arguments.
-
-Put setup that must precede worker imports or Node startup options in the shell.
-The bootstrap forwards SIGINT, SIGTERM, IPC messages, and the worker exit status.
-It remains the supervised parent process; test service shutdown on both platforms
-before deployment. A failure in the startup shell prevents the worker from
-loading. Direct execution of `packages/worker/dist/main.js` bypasses this hook.
-
-Run `yarn workspace @expo/worker test:startup` for the process-level checks.

packages/worker/bin/bootstrap.cjs

@@ -1,53 +0,0 @@
-// This entry point must only load Node built-ins before the startup shell runs.
-const { spawn } = require('node:child_process');
-const path = require('node:path');
-
-const worker = spawn(
-  '/bin/bash',
-  [
-    path.join(__dirname, 'start-worker.sh'),
-    process.execPath,
-    ...process.execArgv,
-    path.join(__dirname, '../dist/main.js'),
-    ...process.argv.slice(2),
-  ],
-  { stdio: process.send ? ['inherit', 'inherit', 'inherit', 'ipc'] : 'inherit' }
-);
-
-const signals = ['SIGINT', 'SIGTERM'];
-for (const signal of signals) {
-  process.on(signal, () => worker.kill(signal));
-}
-
-// Retain the readiness channel used by process managers in development.
-if (process.send) {
-  worker.on('message', message => {
-    if (process.connected) {
-      process.send(message, () => {});
-    }
-  });
-  process.on('message', message => {
-    if (worker.connected) {
-      worker.send(message, () => {});
-    }
-  });
-  process.on('disconnect', () => {
-    if (worker.connected) {
-      worker.disconnect();
-    }
-  });
-}
-
-worker.on('error', error => {
-  process.stderr.write(`Worker startup failed: ${error.stack ?? error}\n`, () => process.exit(1));
-});
-worker.on('exit', (code, signal) => {
-  if (signal) {
-    for (const forwarded of signals) {
-      process.removeAllListeners(forwarded);
-    }
-    process.kill(process.pid, signal);
-  } else {
-    process.exit(code ?? 1);
-  }
-});

packages/worker/bin/start-worker.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Startup setup belongs here, before the normal worker's Node process starts.
-exec "$@"

packages/worker/package.json

@@ -11,12 +11,11 @@
     "compile": "tsc -p tsconfig.build.json",
     "prepublishOnly": "npm run build",
     "test": "yarn test:unit",
-    "test:unit": "yarn test:startup && ./resources/jest --config jest.config.unit.ts",
+    "test:unit": "./resources/jest --config jest.config.unit.ts",
     "test:unit:watch": "./resources/jest --config jest.config.unit.ts --watch",
     "test:integration": "./resources/jest --runInBand --config jest.config.integration.ts",
     "test:integration:watch": "./resources/jest --runInBand --config jest.config.integration.ts --watchAll",
-    "typecheck": "tsc",
-    "test:startup": "node --test tests/*.test.cjs"
+    "typecheck": "tsc"
   },
   "dependencies": {
     "@babel/core": "7.24.7",

packages/worker/package.sh

@@ -120,10 +120,10 @@ fi
 
 # Create backward-compatible symlink for orchestrator
 # The orchestrator expects ./src/services/worker/dist/main.js
-# Route it through the packaged startup wrapper before loading worker modules.
+# but the new structure has ./packages/worker/dist/main.js
 # We're only symlinking the specific file to discover any other legacy path usage
 mkdir -p "$target_root_dir/src/services/worker/dist"
-ln -s ../../../../packages/worker/bin/bootstrap.cjs "$target_root_dir/src/services/worker/dist/main.js"
+ln -s ../../../../packages/worker/dist/main.js "$target_root_dir/src/services/worker/dist/main.js"
 
 tar zcf $OUTPUT_FILE -C $target_root_dir .
 

packages/worker/tests/startup.test.cjs

@@ -1,100 +0,0 @@
-const assert = require('node:assert/strict');
-const { spawn } = require('node:child_process');
-const { once } = require('node:events');
-const fs = require('node:fs');
-const os = require('node:os');
-const path = require('node:path');
-const { test } = require('node:test');
-
-function archive(t, main, shell) {
-  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'worker-startup-'));
-  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
-  const worker = path.join(root, 'packages/worker');
-  fs.mkdirSync(path.join(worker, 'dist'), { recursive: true });
-  fs.cpSync(path.join(__dirname, '../bin'), path.join(worker, 'bin'), { recursive: true });
-  fs.writeFileSync(path.join(worker, 'dist/main.js'), main);
-  if (shell) {
-    fs.writeFileSync(path.join(worker, 'bin/start-worker.sh'), shell);
-  }
-  // Exercise the path that existing images run, including Node's symlink resolution.
-  const legacy = path.join(root, 'src/services/worker/dist/main.js');
-  fs.mkdirSync(path.dirname(legacy), { recursive: true });
-  fs.symlinkSync('../../../../packages/worker/bin/bootstrap.cjs', legacy);
-  return legacy;
-}
-
-function launch(t, entry, args = []) {
-  const child = spawn(process.execPath, [entry, ...args], {
-    stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
-  });
-  t.after(() => {
-    if (child.exitCode === null && child.signalCode === null) {
-      child.kill('SIGKILL');
-    }
-  });
-  return child;
-}
-
-test(
-  'startup settings reach the worker before its first module runs',
-  { timeout: 10000 },
-  async t => {
-    const entry = archive(
-      t,
-      'process.send({ ready: true, args: process.argv.slice(2), setting: process.env.STARTUP_TEST }); process.on("message", () => process.exit(7));',
-      '#!/bin/bash\nset -euo pipefail\nexport STARTUP_TEST=installed\nexec "$@"\n'
-    );
-    const child = launch(t, entry, ['argument with spaces']);
-    const exited = once(child, 'exit');
-    assert.deepEqual((await once(child, 'message'))[0], {
-      ready: true,
-      args: ['argument with spaces'],
-      setting: 'installed',
-    });
-    child.send('stop');
-    assert.deepEqual(await exited, [7, null]);
-  }
-);
-
-for (const signal of ['SIGINT', 'SIGTERM']) {
-  test(`forwards ${signal} and preserves termination status`, { timeout: 10000 }, async t => {
-    const entry = archive(t, 'process.send("ready"); setInterval(() => {}, 1000);');
-    const child = launch(t, entry);
-    const exited = once(child, 'exit');
-    await once(child, 'message');
-    child.kill(signal);
-    assert.deepEqual(await exited, [null, signal]);
-  });
-}
-
-test('startup failure prevents loading the worker', { timeout: 10000 }, async t => {
-  const entry = archive(
-    t,
-    'process.send("must not start");',
-    '#!/bin/bash\nset -euo pipefail\nexit 42\nexec "$@"\n'
-  );
-  const child = launch(t, entry);
-  const messages = [];
-  child.on('message', message => messages.push(message));
-  assert.deepEqual(await once(child, 'exit'), [42, null]);
-  assert.deepEqual(messages, []);
-});
-
-test('runs without IPC and retains stdout, stderr, and exit code', { timeout: 10000 }, async t => {
-  const entry = archive(
-    t,
-    'console.log("worker output"); console.error("worker error"); process.exit(9);'
-  );
-  const child = spawn(process.execPath, [entry], { stdio: ['ignore', 'pipe', 'pipe'] });
-  let stdout = '';
-  let stderr = '';
-  child.stdout.on('data', data => {
-    stdout += data;
-  });
-  child.stderr.on('data', data => {
-    stderr += data;
-  });
-  assert.deepEqual(await once(child, 'close'), [9, null]);
-  assert.equal(stdout, 'worker output\n');
-  assert.equal(stderr, 'worker error\n');
-});