[build-tools] Add npm cache fallback to install_node_modules step

packages/build-tools/src/common/__tests__/installDependencies.test.ts

@@ -40,7 +40,7 @@ describe(installDependenciesWithNpmCacheFallbackAsync, () => {
       env: {
         EAS_VERBOSE: '1',
         EAS_USE_NPM_CACHE: '1',
-        NPM_CONFIG_REGISTRY: npmCacheUrl,
+        EAS_BUILD_NPM_CACHE_URL: npmCacheUrl,
       },
       logger,
       cwd: '/tmp/build',
@@ -56,6 +56,7 @@ describe(installDependenciesWithNpmCacheFallbackAsync, () => {
       env: {
         EAS_VERBOSE: '1',
         EAS_USE_NPM_CACHE: '1',
+        EAS_BUILD_NPM_CACHE_URL: npmCacheUrl,
         NPM_CONFIG_REGISTRY: npmCacheUrl,
       },
     });
@@ -66,6 +67,7 @@ describe(installDependenciesWithNpmCacheFallbackAsync, () => {
       env: {
         EAS_VERBOSE: '1',
         EAS_USE_NPM_CACHE: '1',
+        EAS_BUILD_NPM_CACHE_URL: npmCacheUrl,
       },
     });
     expect(logger.warn).toHaveBeenCalledWith(
@@ -110,7 +112,7 @@ describe(installDependenciesWithNpmCacheFallbackAsync, () => {
       packageManager: PackageManager.NPM,
       env: {
         EAS_USE_NPM_CACHE: '1',
-        NPM_CONFIG_REGISTRY: npmCacheUrl,
+        EAS_BUILD_NPM_CACHE_URL: npmCacheUrl,
       },
       logger,
       cwd: '/tmp/build',
@@ -158,7 +160,7 @@ describe(installDependenciesWithNpmCacheFallbackAsync, () => {
         packageManager: PackageManager.NPM,
         env: {
           EAS_USE_NPM_CACHE: '1',
-          NPM_CONFIG_REGISTRY: 'http://npm.staging.caches.eas-build.internal',
+          EAS_BUILD_NPM_CACHE_URL: 'http://npm.staging.caches.eas-build.internal',
         },
         logger,
         cwd: '/tmp/build',
@@ -170,7 +172,7 @@ describe(installDependenciesWithNpmCacheFallbackAsync, () => {
     expect(Sentry.capture).not.toHaveBeenCalled();
   });
 
-  it('does not retry when EAS_USE_NPM_CACHE is not enabled', async () => {
+  it('does not retry when the npm cache is not enabled', async () => {
     const logger = createMockLogger();
     const npmCacheUrl = 'http://npm.staging.caches.eas-build.internal';
     const error = Object.assign(
@@ -188,9 +190,7 @@ describe(installDependenciesWithNpmCacheFallbackAsync, () => {
     await expect(
       installDependenciesWithNpmCacheFallbackAsync({
         packageManager: PackageManager.NPM,
-        env: {
-          NPM_CONFIG_REGISTRY: npmCacheUrl,
-        },
+        env: {},
         logger,
         cwd: '/tmp/build',
         useFrozenLockfile: false,
@@ -200,6 +200,40 @@ describe(installDependenciesWithNpmCacheFallbackAsync, () => {
     expect(spawn).toHaveBeenCalledTimes(1);
     expect(Sentry.capture).not.toHaveBeenCalled();
   });
+
+  it('installs through the npm cache registry without retrying when the cache install succeeds', async () => {
+    const logger = createMockLogger();
+    const npmCacheUrl = 'http://npm.staging.caches.eas-build.internal';
+
+    jest
+      .mocked(spawn)
+      .mockReturnValueOnce(createSpawnPromise(Promise.resolve(createSpawnResult())));
+
+    await installDependenciesWithNpmCacheFallbackAsync({
+      packageManager: PackageManager.NPM,
+      env: {
+        EAS_USE_NPM_CACHE: '1',
+        EAS_BUILD_NPM_CACHE_URL: npmCacheUrl,
+      },
+      logger,
+      cwd: '/tmp/build',
+      useFrozenLockfile: false,
+    });
+
+    expect(spawn).toHaveBeenCalledTimes(1);
+    expect(spawn).toHaveBeenCalledWith('npm', ['install', '--include=dev'], {
+      cwd: '/tmp/build',
+      logger,
+      infoCallbackFn: undefined,
+      lineTransformer: expect.any(Function),
+      env: {
+        EAS_USE_NPM_CACHE: '1',
+        EAS_BUILD_NPM_CACHE_URL: npmCacheUrl,
+        NPM_CONFIG_REGISTRY: npmCacheUrl,
+      },
+    });
+    expect(Sentry.capture).not.toHaveBeenCalled();
+  });
 });
 
 function createSpawnPromise(result: Promise<SpawnResult>): SpawnPromise<SpawnResult> {

packages/build-tools/src/common/installDependencies.ts

@@ -93,7 +93,24 @@ export async function installDependenciesWithNpmCacheFallbackAsync({
   infoCallbackFn?: SpawnOptions['infoCallbackFn'];
   useFrozenLockfile: boolean;
 }): Promise<void> {
-  const npmCacheUrl = env.NPM_CONFIG_REGISTRY;
+  const npmCacheUrl = env.EAS_BUILD_NPM_CACHE_URL;
+
+  if (!npmCacheUrl) {
+    await (
+      await installDependenciesAsync({
+        packageManager,
+        env,
+        logger,
+        infoCallbackFn,
+        cwd,
+        useFrozenLockfile,
+      })
+    ).spawnPromise;
+    return;
+  }
+
+  logger.info(`Installing dependencies using the npm cache registry (${npmCacheUrl}).`);
+
   let firstErrorLine: string | undefined;
   let errorLineCount = 0;
 
@@ -101,11 +118,11 @@ export async function installDependenciesWithNpmCacheFallbackAsync({
     await (
       await installDependenciesAsync({
         packageManager,
-        env,
+        env: { ...env, NPM_CONFIG_REGISTRY: npmCacheUrl },
         logger,
         infoCallbackFn,
         lineTransformer: (line: string) => {
-          if (isNpmCacheRegistryErrorLine(line, { env, npmCacheUrl })) {
+          if (isNpmCacheRegistryErrorLine(line, npmCacheUrl)) {
             firstErrorLine ??= line;
             errorLineCount += 1;
           }
@@ -124,7 +141,7 @@ export async function installDependenciesWithNpmCacheFallbackAsync({
       });
     }
   } catch (err: unknown) {
-    if (!isNpmCacheInstallFailure(err, { env, npmCacheUrl })) {
+    if (!isNpmCacheInstallFailure(err, npmCacheUrl)) {
       throw err;
     }
 
@@ -144,11 +161,10 @@ export async function installDependenciesWithNpmCacheFallbackAsync({
       },
     });
 
-    const { NPM_CONFIG_REGISTRY: _NPM_CONFIG_REGISTRY, ...fallbackEnv } = env;
     await (
       await installDependenciesAsync({
         packageManager,
-        env: fallbackEnv,
+        env,
         logger,
         infoCallbackFn,
         cwd,
@@ -172,35 +188,17 @@ export function resolvePackagerDir(ctx: BuildContext<Job>): string {
   return packagerRunDir;
 }
 
-function isNpmCacheInstallFailure(
-  err: unknown,
-  { env, npmCacheUrl }: { env: Record<string, string | undefined>; npmCacheUrl: string | undefined }
-): boolean {
-  if (!isNpmCacheRegistryEnabled(env, npmCacheUrl)) {
-    return false;
-  }
-  const errorOutput = getErrorOutput(err);
-  return errorOutput.includes(npmCacheUrl);
+function isNpmCacheInstallFailure(err: unknown, npmCacheUrl: string): boolean {
+  return getErrorOutput(err).includes(npmCacheUrl);
 }
 
-function isNpmCacheRegistryErrorLine(
-  line: string,
-  { env, npmCacheUrl }: { env: Record<string, string | undefined>; npmCacheUrl: string | undefined }
-): boolean {
+function isNpmCacheRegistryErrorLine(line: string, npmCacheUrl: string): boolean {
   return (
-    isNpmCacheRegistryEnabled(env, npmCacheUrl) &&
     line.includes(npmCacheUrl) &&
     /(?:error|failed|ENOTFOUND|ECONN|ETIMEDOUT|EAI_AGAIN|FetchError)/i.test(line)
   );
 }
 
-function isNpmCacheRegistryEnabled(
-  env: Record<string, string | undefined>,
-  npmCacheUrl: string | undefined
-): npmCacheUrl is string {
-  return env.EAS_USE_NPM_CACHE === '1' && !!npmCacheUrl;
-}
-
 function getErrorOutput(err: unknown): string {
   if (!(err instanceof Error)) {
     return '';

packages/build-tools/src/steps/functions/__tests__/installNodeModules.test.ts

@@ -0,0 +1,132 @@
+import { BuildStepContext, BuildStepEnv } from '@expo/steps';
+import spawn, { SpawnPromise, SpawnResult } from '@expo/turtle-spawn';
+
+import { createMockLogger } from '../../../__tests__/utils/logger';
+import {
+  PackageManager,
+  findPackagerRootDir,
+  resolvePackageManager,
+  shouldUseFrozenLockfile,
+} from '../../../utils/packageManager';
+import { readPackageJson } from '../../../utils/project';
+import { installNodeModules } from '../installNodeModules';
+
+jest.mock('@expo/turtle-spawn', () => jest.fn());
+jest.mock('../../../sentry', () => ({
+  Sentry: {
+    capture: jest.fn(),
+  },
+}));
+jest.mock('../../../utils/packageManager', () => ({
+  ...jest.requireActual('../../../utils/packageManager'),
+  resolvePackageManager: jest.fn(),
+  findPackagerRootDir: jest.fn(),
+  shouldUseFrozenLockfile: jest.fn(),
+}));
+jest.mock('../../../utils/project', () => ({
+  ...jest.requireActual('../../../utils/project'),
+  readPackageJson: jest.fn(),
+}));
+
+describe(installNodeModules, () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    jest.mocked(resolvePackageManager).mockReturnValue(PackageManager.NPM);
+    jest.mocked(findPackagerRootDir).mockReturnValue('/tmp/build');
+    jest.mocked(shouldUseFrozenLockfile).mockReturnValue(false);
+    jest.mocked(readPackageJson).mockReturnValue({});
+  });
+
+  function createStepCtx(logger: ReturnType<typeof createMockLogger>): BuildStepContext {
+    return {
+      logger,
+      workingDirectory: '/tmp/build',
+      global: {
+        projectTargetDirectory: '/tmp/build',
+        staticContext: { metadata: {} },
+      },
+    } as unknown as BuildStepContext;
+  }
+
+  it('retries the install without the npm cache registry when the cache install fails', async () => {
+    const logger = createMockLogger();
+    const npmCacheUrl = 'http://npm.caches.eas-build.internal';
+    const env: BuildStepEnv = {
+      EAS_USE_NPM_CACHE: '1',
+      EAS_BUILD_NPM_CACHE_URL: npmCacheUrl,
+    };
+    const error = Object.assign(new Error('npm install exited with non-zero code: 1'), {
+      status: 1,
+      signal: null,
+      stdout: '',
+      stderr: `npm error request to ${npmCacheUrl}/left-pad failed, reason: getaddrinfo ENOTFOUND`,
+    });
+
+    jest
+      .mocked(spawn)
+      .mockReturnValueOnce(createSpawnPromise(Promise.reject(error)))
+      .mockReturnValueOnce(createSpawnPromise(Promise.resolve(createSpawnResult())));
+
+    await installNodeModules(createStepCtx(logger), env);
+
+    expect(spawn).toHaveBeenCalledTimes(2);
+    expect(spawn).toHaveBeenNthCalledWith(
+      1,
+      'npm',
+      ['install', '--include=dev'],
+      expect.objectContaining({
+        env: {
+          EAS_USE_NPM_CACHE: '1',
+          EAS_BUILD_NPM_CACHE_URL: npmCacheUrl,
+          NPM_CONFIG_REGISTRY: npmCacheUrl,
+        },
+      })
+    );
+    expect(spawn).toHaveBeenNthCalledWith(
+      2,
+      'npm',
+      ['install', '--include=dev'],
+      expect.objectContaining({
+        env: { EAS_USE_NPM_CACHE: '1', EAS_BUILD_NPM_CACHE_URL: npmCacheUrl },
+      })
+    );
+    expect(logger.warn).toHaveBeenCalledWith(
+      `Failed to install dependencies using the npm cache registry (${npmCacheUrl}). Retrying without the npm cache registry.`
+    );
+  });
+
+  it('does not retry when the npm cache is not enabled', async () => {
+    const logger = createMockLogger();
+    const error = Object.assign(new Error('npm install exited with non-zero code: 1'), {
+      status: 1,
+      signal: null,
+      stdout: '',
+      stderr: 'npm error some unrelated failure',
+    });
+
+    jest.mocked(spawn).mockReturnValueOnce(createSpawnPromise(Promise.reject(error)));
+
+    await expect(installNodeModules(createStepCtx(logger), {})).rejects.toThrow(error);
+
+    expect(spawn).toHaveBeenCalledTimes(1);
+  });
+});
+
+function createSpawnPromise(result: Promise<SpawnResult>): SpawnPromise<SpawnResult> {
+  return Object.assign(result, {
+    child: {
+      pid: 123,
+    },
+  }) as SpawnPromise<SpawnResult>;
+}
+
+function createSpawnResult(): SpawnResult {
+  return {
+    pid: 123,
+    output: ['', ''],
+    stdout: '',
+    stderr: '',
+    status: 0,
+    signal: null,
+  };
+}

packages/build-tools/src/steps/functions/installNodeModules.ts

@@ -1,7 +1,7 @@
 import { BuildFunction, BuildStepContext, BuildStepEnv } from '@expo/steps';
 import path from 'path';
 
-import { installDependenciesAsync } from '../../common/installDependencies';
+import { installDependenciesWithNpmCacheFallbackAsync } from '../../common/installDependencies';
 import {
   findPackagerRootDir,
   getPackageVersionFromPackageJson,
@@ -63,7 +63,7 @@ export async function installNodeModules(
       packageName: 'react-native',
     });
 
-  const { spawnPromise } = await installDependenciesAsync({
+  await installDependenciesWithNpmCacheFallbackAsync({
     packageManager,
     env,
     logger: stepCtx.logger,
@@ -74,5 +74,4 @@ export async function installNodeModules(
       reactNativeVersion,
     }),
   });
-  await spawnPromise;
 }

packages/worker/src/__unit__/env.test.ts

@@ -268,8 +268,8 @@ describe(getBuildEnv.name, () => {
       buildId: 'build-id',
     });
 
-    expect(env.NPM_CACHE_URL).toBe('https://npm.example');
-    expect(env.NPM_CONFIG_REGISTRY).toBe('https://npm.example');
+    expect(env.NPM_CACHE_URL).toBeUndefined();
+    expect(env.NPM_CONFIG_REGISTRY).toBeUndefined();
     expect(env.EAS_BUILD_NPM_CACHE_URL).toBe('https://npm.example');
     expect(env.NVM_NODEJS_ORG_MIRROR).toBe('https://node.example');
     expect(env.EAS_BUILD_MAVEN_CACHE_URL).toBe('https://maven.example');

packages/worker/src/env.ts

@@ -36,14 +36,11 @@ export function getBuildEnv({
   setEnv(env, 'EAS_BUILD_RUNNER', 'eas-build');
   setEnv(env, 'EAS_BUILD_PLATFORM', job.platform);
   setEnv(env, 'EAS_CLI_SENTRY_DSN', config.sentry.dsn);
-  // NPM_CACHE_URL is deprecated
   const npmCacheUrl = RuntimeSettings.getNpmCacheUrl();
   const nodeJsCacheUrl = RuntimeSettings.getNodeJsCacheUrl();
   const mavenCacheUrl = RuntimeSettings.getMavenCacheUrl();
   const cocoapodsCacheUrl = RuntimeSettings.getCocoapodsCacheUrl();
 
-  setEnv(env, 'NPM_CACHE_URL', npmCacheUrl);
-  setEnv(env, 'NPM_CONFIG_REGISTRY', npmCacheUrl);
   setEnv(env, 'NVM_NODEJS_ORG_MIRROR', nodeJsCacheUrl);
   setEnv(env, 'EAS_BUILD_NPM_CACHE_URL', npmCacheUrl);
   setEnv(env, 'EAS_BUILD_PROFILE', metadata.buildProfile);