Compare commits

...
10 changed files with 447 additions and 10 deletions

No files matched your search

+2
View File
@@ -8,6 +8,8 @@ This is the log of notable changes to EAS CLI and related packages.
### 🎉 New features
- [build-tools] Add opt-in Metro transform cache sharing across build and workflow jobs. ([#4386](https://github.com/expo/eas-cli/pull/4386) by [@sjchmiela](https://github.com/sjchmiela))
### 🐛 Bug fixes
### 🧹 Chores
@@ -16,6 +16,7 @@ import { eagerBundleAsync, shouldUseEagerBundle } from '../common/eagerBundle';
import { prebuildAsync } from '../common/prebuild';
import { JobHooksRef, setupAsync } from '../common/setup';
import { Artifacts, BuildContext, SkipNativeBuildError } from '../context';
import { restoreMetroCacheAsync, saveMetroCacheAsync } from '../steps/functions/metroBuildCache';
import {
cacheStatsAsync,
restoreCcacheAsync,
@@ -95,6 +96,16 @@ async function buildInnerAsync(
return;
}
await ctx.cacheManager?.restoreCache(ctx);
Object.assign(
ctx.env,
await restoreMetroCacheAsync({
logger: ctx.logger,
platform: ctx.job.platform,
cacheDirectory: path.join(ctx.workingdir, 'metro-cache'),
env: ctx.env,
secrets: ctx.job.secrets,
})
);
await restoreCcacheAsync({
logger: ctx.logger,
workingDirectory,
@@ -235,6 +246,12 @@ async function buildInnerAsync(
return;
}
await ctx.cacheManager?.saveCache(ctx);
await saveMetroCacheAsync({
logger: ctx.logger,
platform: ctx.job.platform,
env: ctx.env,
secrets: ctx.job.secrets,
});
await saveCcacheAsync({
logger: ctx.logger,
workingDirectory,
+18
View File
@@ -3,6 +3,7 @@ import { BuildMode, BuildPhase, Ios, ManagedArtifactType, Workflow } from '@expo
import plist from '@expo/plist';
import fs from 'fs-extra';
import nullthrows from 'nullthrows';
import path from 'path';
import { runBuilderWithHooksAsync } from './common';
import { runCustomBuildAsync } from './custom';
@@ -18,6 +19,7 @@ import { downloadApplicationArchiveAsync } from '../ios/resign';
import { resolveArtifactPath, resolveBuildConfiguration, resolveScheme } from '../ios/resolve';
import { Sentry } from '../sentry';
import { parseAndReportXcactivitylog } from '../steps/utils/ios/xcactivitylog';
import { restoreMetroCacheAsync, saveMetroCacheAsync } from '../steps/functions/metroBuildCache';
import {
cacheStatsAsync,
restoreCcacheAsync,
@@ -104,6 +106,16 @@ async function buildInnerAsync(
return;
}
await ctx.cacheManager?.restoreCache(ctx);
Object.assign(
ctx.env,
await restoreMetroCacheAsync({
logger: ctx.logger,
platform: ctx.job.platform,
cacheDirectory: path.join(ctx.workingdir, 'metro-cache'),
env: ctx.env,
secrets: ctx.job.secrets,
})
);
await restoreCcacheAsync({
logger: ctx.logger,
workingDirectory,
@@ -252,6 +264,12 @@ async function buildInnerAsync(
return;
}
await ctx.cacheManager?.saveCache(ctx);
await saveMetroCacheAsync({
logger: ctx.logger,
platform: ctx.job.platform,
env: ctx.env,
secrets: ctx.job.secrets,
});
await saveCcacheAsync({
logger: ctx.logger,
workingDirectory,
@@ -0,0 +1,180 @@
import { Platform } from '@expo/eas-build-job';
import fs from 'fs/promises';
import os from 'os';
import path from 'path';
import * as tar from 'tar';
import { createGlobalContextMock } from '../../../__tests__/utils/context';
import { createMockLogger } from '../../../__tests__/utils/logger';
import { restoreMetroCacheAsync, saveMetroCacheAsync } from '../metroBuildCache';
import { createRestoreBuildCacheFunction } from '../restoreBuildCache';
import { downloadCacheAsync } from '../restoreCache';
import { createSaveBuildCacheFunction } from '../saveBuildCache';
import { uploadCacheAsync } from '../saveCache';
jest.mock('../restoreCache', () => ({ downloadCacheAsync: jest.fn() }));
jest.mock('../saveCache', () => ({ uploadCacheAsync: jest.fn() }));
const logger = createMockLogger();
const options = {
logger,
platform: Platform.IOS,
env: {
EAS_METRO_CACHE: '1',
EAS_BUILD_ID: 'build-id',
__API_SERVER_URL: 'https://api.expo.test',
},
secrets: { robotAccessToken: 'token' },
};
async function writeEntry(root: string, name: string, value: string): Promise<void> {
const filename = path.join(root, name);
await fs.mkdir(path.dirname(filename), { recursive: true });
await fs.writeFile(filename, value);
}
describe('Metro build cache', () => {
let root: string;
beforeEach(async () => {
jest.resetAllMocks();
root = await fs.mkdtemp(path.join(os.tmpdir(), 'metro-test-'));
jest.mocked(downloadCacheAsync).mockRejectedValue(new Error('cache miss'));
});
it('does nothing unless enabled', async () => {
expect(await restoreMetroCacheAsync({ ...options, env: {}, cacheDirectory: root })).toEqual({});
await saveMetroCacheAsync({ ...options, env: {} });
expect(downloadCacheAsync).not.toHaveBeenCalled();
expect(uploadCacheAsync).not.toHaveBeenCalled();
});
it('starts with empty stores after a miss and preserves them on repeated restore steps', async () => {
await writeEntry(path.join(root, 'output'), 'aa/bb.mp', 'old output');
await writeEntry(path.join(root, 'restored'), 'aa/cc.mp', 'old restored');
const cacheEnv = await restoreMetroCacheAsync({ ...options, cacheDirectory: root });
expect(await fs.readdir(cacheEnv.EXPO_METRO_CACHE_OUTPUT_DIR)).toEqual([]);
expect(await fs.readdir(cacheEnv.EXPO_METRO_CACHE_RESTORE_DIR)).toEqual([]);
await writeEntry(cacheEnv.EXPO_METRO_CACHE_OUTPUT_DIR, 'aa/bb.mp', 'first bundle');
await restoreMetroCacheAsync({
...options,
env: { ...options.env, ...cacheEnv },
cacheDirectory: root,
});
expect(await fs.readFile(path.join(root, 'output/aa/bb.mp'), 'utf8')).toBe('first bundle');
expect(downloadCacheAsync).toHaveBeenCalledTimes(1);
});
it('archives only completed output entries and restores into a different job directory', async () => {
const cacheEnv = await restoreMetroCacheAsync({ ...options, cacheDirectory: root });
await writeEntry(cacheEnv.EXPO_METRO_CACHE_OUTPUT_DIR, 'aa/bb.mp', 'reused');
await writeEntry(cacheEnv.EXPO_METRO_CACHE_OUTPUT_DIR, 'aa/cc.mp', 'new');
await writeEntry(cacheEnv.EXPO_METRO_CACHE_OUTPUT_DIR, 'aa/bb.tmp123.mp', 'incomplete');
await writeEntry(cacheEnv.EXPO_METRO_CACHE_RESTORE_DIR, 'aa/dd.mp', 'unused');
const downloadPath = path.join(os.tmpdir(), 'download.tar.gz');
jest.mocked(uploadCacheAsync).mockImplementation(async ({ archivePath }) => {
await fs.copyFile(archivePath, downloadPath);
});
await saveMetroCacheAsync({ ...options, env: { ...options.env, ...cacheEnv } });
expect(uploadCacheAsync).toHaveBeenCalledWith(
expect.objectContaining({
key: 'metro-transform-v1',
force: true,
paths: ['metro-transform-cache-v1'],
})
);
const archivePath = downloadPath;
jest
.mocked(downloadCacheAsync)
.mockResolvedValue({ archivePath, matchedKey: 'metro-transform-v1' });
const next = await restoreMetroCacheAsync({
...options,
cacheDirectory: path.join(root, 'next-job'),
});
expect(await fs.readdir(path.join(next.EXPO_METRO_CACHE_RESTORE_DIR, 'aa'))).toEqual([
'bb.mp',
'cc.mp',
]);
expect(
await fs.readFile(path.join(next.EXPO_METRO_CACHE_RESTORE_DIR, 'aa/bb.mp'), 'utf8')
).toBe('reused');
expect(await fs.readdir(next.EXPO_METRO_CACHE_OUTPUT_DIR)).toEqual([]);
expect(downloadCacheAsync).toHaveBeenLastCalledWith(
expect.objectContaining({
key: 'metro-transform-v1',
paths: ['metro-transform-cache-v1'],
})
);
});
it('ignores archives with no usable entries and skips empty output from older Metro versions', async () => {
const source = path.join(root, 'source');
await writeEntry(source, 'other.txt', 'not a cache');
const archivePath = path.join(root, 'download.tar.gz');
await tar.create({ file: archivePath, cwd: source, gzip: true }, ['other.txt']);
jest.mocked(downloadCacheAsync).mockResolvedValue({ archivePath, matchedKey: 'key' });
const cacheEnv = await restoreMetroCacheAsync({
...options,
cacheDirectory: path.join(root, 'cache'),
});
expect(await fs.readdir(cacheEnv.EXPO_METRO_CACHE_RESTORE_DIR)).toEqual([]);
await saveMetroCacheAsync({ ...options, env: { ...options.env, ...cacheEnv } });
expect(uploadCacheAsync).not.toHaveBeenCalled();
});
it('discards a damaged archive and still allows a cold bundle', async () => {
const archivePath = path.join(root, 'broken.tar.gz');
await fs.writeFile(archivePath, 'not a tar archive');
jest.mocked(downloadCacheAsync).mockResolvedValue({ archivePath, matchedKey: 'key' });
const cacheEnv = await restoreMetroCacheAsync({
...options,
cacheDirectory: path.join(root, 'cache'),
});
expect(await fs.readdir(cacheEnv.EXPO_METRO_CACHE_RESTORE_DIR)).toEqual([]);
expect(await fs.readdir(cacheEnv.EXPO_METRO_CACHE_OUTPUT_DIR)).toEqual([]);
});
it.each([Platform.IOS, Platform.ANDROID, undefined])(
'passes cache directories between steps with platform=%s',
async platform => {
const globalCtx = createGlobalContextMock({
logger,
projectTargetDirectory: root,
staticContextContent: { job: { platform, secrets: options.secrets } },
});
globalCtx.updateEnv({ ...options.env, PRESERVED_ENV: 'value' });
const restoreStep = createRestoreBuildCacheFunction().createBuildStepFromFunctionCall(
globalCtx,
{}
);
await restoreStep.executeAsync();
expect(globalCtx.env.PRESERVED_ENV).toBe('value');
const output = globalCtx.env.EXPO_METRO_CACHE_OUTPUT_DIR!;
expect(path.isAbsolute(output)).toBe(true);
expect(globalCtx.env.EXPO_METRO_CACHE_RESTORE_DIR).not.toBe(output);
await writeEntry(output, 'aa/bb.mp', 'bundle result');
const saveStep = createSaveBuildCacheFunction(new Date()).createBuildStepFromFunctionCall(
globalCtx,
{}
);
await saveStep.executeAsync();
expect(uploadCacheAsync).toHaveBeenCalledWith(
expect.objectContaining({
force: true,
platform,
key: 'metro-transform-v1',
cacheVersion: 'metro-transform-v1',
})
);
}
);
it('does not fail the build if upload fails', async () => {
const cacheEnv = await restoreMetroCacheAsync({ ...options, cacheDirectory: root });
await writeEntry(cacheEnv.EXPO_METRO_CACHE_OUTPUT_DIR, 'aa/bb.mp', 'new');
jest.mocked(uploadCacheAsync).mockRejectedValue(new Error('network failure'));
await expect(
saveMetroCacheAsync({ ...options, env: { ...options.env, ...cacheEnv } })
).resolves.toBeUndefined();
await expect(fs.access(path.join(root, 'cache.tar.gz'))).rejects.toThrow();
});
});
@@ -4,9 +4,11 @@ import fs from 'fs';
import fetch from 'node-fetch';
import os from 'os';
import path from 'path';
import { Readable } from 'stream';
import { compressCacheAsync, uploadCacheAsync } from '../saveCache';
import { decompressCacheAsync } from '../restoreCache';
import { decompressCacheAsync, downloadCacheAsync } from '../restoreCache';
import { getCacheVersion } from '../../utils/cache';
jest.mock('node-fetch');
@@ -94,9 +96,10 @@ describe(uploadCacheAsync, () => {
});
it.each([
['normal upload', undefined, false],
['forced upload', true, true],
] as const)('sends force for %s', async (_name, force, expectedForce) => {
['normal upload', undefined, false, undefined],
['forced upload', true, true, undefined],
['portable Metro upload', true, true, 'metro-transform-v1'],
] as const)('sends force for %s', async (_name, force, expectedForce, cacheVersion) => {
const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'upload-cache-test-'));
const archivePath = path.join(tempDir, 'cache.tar.gz');
await fs.promises.writeFile(archivePath, 'cache archive');
@@ -134,6 +137,7 @@ describe(uploadCacheAsync, () => {
size: 13,
platform: Platform.ANDROID,
force,
cacheVersion,
});
expect(fetch).toHaveBeenCalledTimes(2);
@@ -146,6 +150,7 @@ describe(uploadCacheAsync, () => {
force: expectedForce,
key: 'cache-key',
size: 13,
version: cacheVersion ?? getCacheVersion(['/cache/path']),
});
expect(jest.mocked(fetch).mock.calls[1][0].toString()).toBe(
'https://storage.expo.test/cache'
@@ -155,3 +160,49 @@ describe(uploadCacheAsync, () => {
}
});
});
describe(downloadCacheAsync, () => {
it.each([Platform.IOS, Platform.ANDROID, undefined])(
'uses the same Metro version with platform=%s',
async platform => {
jest.mocked(fetch).mockReset();
jest
.mocked(fetch)
.mockResolvedValueOnce(
new Response(
JSON.stringify({
data: {
matchedKey: 'metro-transform-v1',
downloadUrl: 'https://storage.expo.test/cache.tar.gz',
},
})
)
)
.mockResolvedValueOnce(new Response(Readable.from(['cache archive'])));
await expect(
downloadCacheAsync({
logger: createLoggerMock(),
jobId: 'job-id',
expoApiServerURL: 'https://api.expo.test',
robotAccessToken: 'token',
paths: ['metro-transform-cache-v1'],
key: 'metro-transform-v1',
cacheVersion: 'metro-transform-v1',
keyPrefixes: [],
platform,
})
).resolves.toMatchObject({ matchedKey: 'metro-transform-v1' });
expect(fetch).toHaveBeenCalledTimes(2);
const [url, request] = jest.mocked(fetch).mock.calls[0];
expect(url.toString()).toBe(
`https://api.expo.test/v2/${platform ? 'turtle-builds/caches' : 'turtle-caches'}/download`
);
expect(JSON.parse(request!.body as string)).toEqual({
[platform ? 'buildId' : 'jobRunId']: 'job-id',
version: 'metro-transform-v1',
key: 'metro-transform-v1',
keyPrefixes: [],
});
}
);
});
@@ -0,0 +1,139 @@
import { Platform } from '@expo/eas-build-job';
import { bunyan } from '@expo/logger';
import fg from 'fast-glob';
import fs from 'fs/promises';
import nullthrows from 'nullthrows';
import path from 'path';
import * as tar from 'tar';
import { downloadCacheAsync } from './restoreCache';
import { uploadCacheAsync } from './saveCache';
import { TurtleFetchError } from '../../utils/turtleFetch';
// Enabled with EAS_METRO_CACHE=1. Requires BuildCacheStore support in @expo/metro-config.
// The server scopes this key by project and branch, with default-branch fallback.
// Metro validates individual transforms, so the archive can span platforms and hosts.
// Stable logical paths keep the cache version independent of job directories.
const CACHE_PATHS = ['metro-transform-cache-v1'];
const CACHE_ENTRY = /^[0-9a-f]{2}\/[0-9a-f]+\.mp$/;
type CacheOptions = {
logger: bunyan;
platform?: Platform;
env: Record<string, string | undefined>;
secrets?: { robotAccessToken?: string };
};
function getCacheRequest({ platform, env, secrets, logger }: CacheOptions) {
return {
logger,
platform,
jobId: nullthrows(env.EAS_BUILD_ID, 'EAS_BUILD_ID is not set'),
expoApiServerURL: nullthrows(env.__API_SERVER_URL, '__API_SERVER_URL is not set'),
robotAccessToken: nullthrows(secrets?.robotAccessToken, 'Robot access token is required'),
key: 'metro-transform-v1',
paths: CACHE_PATHS,
cacheVersion: 'metro-transform-v1',
};
}
export async function restoreMetroCacheAsync(
options: CacheOptions & { cacheDirectory: string }
): Promise<Record<string, string>> {
const { env, logger, cacheDirectory } = options;
if (env.EAS_METRO_CACHE !== '1') {
return {};
}
const output = path.resolve(cacheDirectory, 'output');
const restored = path.resolve(cacheDirectory, 'restored');
const cacheEnv = {
EXPO_METRO_CACHE_OUTPUT_DIR: output,
EXPO_METRO_CACHE_RESTORE_DIR: restored,
};
// Repeated restore steps must keep results from earlier bundle commands.
if (env.EXPO_METRO_CACHE_OUTPUT_DIR === output && env.EXPO_METRO_CACHE_RESTORE_DIR === restored) {
return cacheEnv;
}
let archivePath: string | undefined;
try {
await fs.rm(cacheDirectory, { recursive: true, force: true });
await fs.mkdir(output, { recursive: true });
await fs.mkdir(restored, { recursive: true });
} catch (err) {
logger.warn({ err }, 'Failed to prepare Metro cache directories');
return {};
}
try {
const result = await downloadCacheAsync({ ...getCacheRequest(options), keyPrefixes: [] });
archivePath = result.archivePath;
await tar.extract({
file: archivePath,
cwd: restored,
strict: true,
// Only accept binary store entries, without temporary files or links.
filter: (entryPath, entry) =>
CACHE_ENTRY.test(entryPath) && 'type' in entry && entry.type === 'File',
});
logger.info('Restored Metro transform cache');
} catch (err) {
try {
await fs.rm(restored, { recursive: true, force: true });
await fs.mkdir(restored, { recursive: true });
} catch (cleanupError) {
logger.warn({ err: cleanupError }, 'Failed to discard incomplete Metro cache');
return {};
}
if (err instanceof TurtleFetchError && err.response?.status === 404) {
logger.info('No Metro transform cache found');
} else {
logger.warn({ err }, 'Failed to restore Metro transform cache');
}
} finally {
if (archivePath) {
await fs.rm(archivePath, { force: true }).catch(err => {
logger.warn({ err }, 'Failed to remove Metro cache archive');
});
}
}
return cacheEnv;
}
export async function saveMetroCacheAsync(options: CacheOptions): Promise<void> {
const { env, logger } = options;
const output = env.EXPO_METRO_CACHE_OUTPUT_DIR;
if (env.EAS_METRO_CACHE !== '1' || !output || !env.EXPO_METRO_CACHE_RESTORE_DIR) {
return;
}
let archivePath: string | undefined;
try {
const files = (
await fg('[0-9a-f][0-9a-f]/*.mp', { cwd: output, followSymbolicLinks: false })
).filter(file => CACHE_ENTRY.test(file));
if (files.length === 0) {
logger.info('No Metro transform cache entries to save');
return;
}
// Run after bundling processes exit. Archive only output entries: reused and
// new transforms. Unused restored entries are excluded from the next archive.
archivePath = path.join(path.dirname(output), 'cache.tar.gz');
await tar.create({ file: archivePath, cwd: output, gzip: true }, files);
const { size } = await fs.stat(archivePath);
await uploadCacheAsync({
...getCacheRequest(options),
archivePath,
size,
// Refresh the working set even when the lockfile does not change.
force: true,
});
logger.info(`Saved ${files.length} Metro transform cache entries`);
} catch (err) {
logger.warn({ err }, 'Failed to save Metro transform cache');
} finally {
if (archivePath) {
await fs.rm(archivePath, { force: true }).catch(err => {
logger.warn({ err }, 'Failed to remove Metro cache archive');
});
}
}
}
@@ -13,6 +13,7 @@ import os from 'os';
import path from 'path';
import { sendCcacheStatsAsync } from './ccacheStats';
import { restoreMetroCacheAsync } from './metroBuildCache';
import { decompressCacheAsync, downloadCacheAsync, downloadPublicCacheAsync } from './restoreCache';
import {
CcacheBuildTarget,
@@ -53,12 +54,25 @@ export function createRestoreBuildCacheFunction(): BuildFunction {
const platform =
(inputs.platform.value as Platform | undefined) ??
stepCtx.global.staticContext.job.platform;
if (!platform || ![Platform.ANDROID, Platform.IOS].includes(platform)) {
if (platform && ![Platform.ANDROID, Platform.IOS].includes(platform)) {
throw new Error(
`Unsupported platform: ${platform}. Platform must be "${Platform.ANDROID}" or "${Platform.IOS}"`
);
}
const cacheEnv = await restoreMetroCacheAsync({
logger,
platform: stepCtx.global.staticContext.job.platform,
cacheDirectory: path.join(stepCtx.global.stepsInternalBuildDirectory, 'metro-cache'),
env,
secrets: stepCtx.global.staticContext.job.secrets,
});
stepCtx.global.updateEnv({ ...stepCtx.global.env, ...cacheEnv });
if (!platform) {
return;
}
const target: CcacheBuildTarget =
platform === Platform.IOS
? {
@@ -113,6 +113,7 @@ export async function downloadCacheAsync({
expoApiServerURL,
robotAccessToken,
paths,
cacheVersion,
key,
keyPrefixes,
platform,
@@ -122,6 +123,7 @@ export async function downloadCacheAsync({
expoApiServerURL: string;
robotAccessToken: string;
paths: string[];
cacheVersion?: string;
key: string;
keyPrefixes: string[];
platform: Platform | undefined;
@@ -134,13 +136,13 @@ export async function downloadCacheAsync({
? {
buildId: jobId,
key,
version: getCacheVersion(paths),
version: cacheVersion ?? getCacheVersion(paths),
keyPrefixes,
}
: {
jobRunId: jobId,
key,
version: getCacheVersion(paths),
version: cacheVersion ?? getCacheVersion(paths),
keyPrefixes,
},
headers: {
@@ -13,6 +13,7 @@ import nullthrows from 'nullthrows';
import os from 'os';
import path from 'path';
import { saveMetroCacheAsync } from './metroBuildCache';
import { compressCacheAsync, uploadCacheAsync } from './saveCache';
import { formatBytes } from '../../utils/artifacts';
import {
@@ -51,12 +52,23 @@ export function createSaveBuildCacheFunction(evictUsedBefore: Date): BuildFuncti
const platform =
(inputs.platform.value as Platform | undefined) ??
stepCtx.global.staticContext.job.platform;
if (!platform || ![Platform.ANDROID, Platform.IOS].includes(platform)) {
if (platform && ![Platform.ANDROID, Platform.IOS].includes(platform)) {
throw new Error(
`Unsupported platform: ${platform}. Platform must be "${Platform.ANDROID}" or "${Platform.IOS}"`
);
}
await saveMetroCacheAsync({
logger,
platform: stepCtx.global.staticContext.job.platform,
env,
secrets: stepCtx.global.staticContext.job.secrets,
});
if (!platform) {
return;
}
const target: CcacheBuildTarget =
platform === Platform.IOS
? {
@@ -99,6 +99,7 @@ export async function uploadCacheAsync({
expoApiServerURL,
robotAccessToken,
paths,
cacheVersion,
key,
archivePath,
size,
@@ -110,6 +111,7 @@ export async function uploadCacheAsync({
expoApiServerURL: string;
robotAccessToken: string;
paths: string[];
cacheVersion?: string;
key: string;
archivePath: string;
size: number;
@@ -127,14 +129,14 @@ export async function uploadCacheAsync({
? JSON.stringify({
buildId: jobId,
key,
version: getCacheVersion(paths),
version: cacheVersion ?? getCacheVersion(paths),
size,
force,
})
: JSON.stringify({
jobRunId: jobId,
key,
version: getCacheVersion(paths),
version: cacheVersion ?? getCacheVersion(paths),
size,
force,
}),