Compare commits

...
11 changed files with 1070 additions and 51 deletions

No files matched your search

@@ -0,0 +1,296 @@
#include <dlfcn.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/*
* Xcode loads one plugin that implements the llcas_* API. This small shim loads
* Apple's local CAS plugin and forwards every API call to it. The only exception
* is the "remote-service-path" option: Xcode needs that option to schedule C and
* Objective-C cache tasks, but Apple's local plugin does not need a remote service.
* The shim accepts and discards that option. No cache data leaves the machine.
*/
typedef void *llcas_cas_options_t;
typedef void *llcas_cas_t;
typedef void *llcas_cancellable_t;
typedef uint32_t llcas_lookup_result_t;
typedef struct {
const uint8_t *data;
size_t size;
} llcas_digest_t;
typedef struct {
const void *data;
size_t size;
} llcas_data_t;
typedef struct {
uint64_t opaque;
} llcas_objectid_t;
typedef struct {
uint64_t opaque;
} llcas_loaded_object_t;
typedef struct {
uint64_t opaque_b;
uint64_t opaque_e;
} llcas_object_refs_t;
typedef void (*llcas_cas_load_object_cb)(void *, llcas_lookup_result_t,
llcas_loaded_object_t, char *);
typedef void (*llcas_actioncache_get_cb)(void *, llcas_lookup_result_t,
llcas_objectid_t, char *);
typedef void (*llcas_actioncache_put_cb)(void *, bool, char *);
static void *apple_plugin_handle;
static void *apple_plugin_symbol(const char *name) {
if (apple_plugin_handle == NULL) {
const char *path = getenv("EAS_XCODE_LOCAL_CAS_APPLE_PLUGIN");
if (path == NULL || path[0] == '\0') {
path = "/Applications/Xcode.app/Contents/Developer/usr/lib/"
"libToolchainCASPlugin.dylib";
}
apple_plugin_handle = dlopen(path, RTLD_NOW | RTLD_LOCAL);
if (apple_plugin_handle == NULL) {
fprintf(stderr, "EAS local CAS plugin could not load %s: %s\n", path,
dlerror());
abort();
}
}
void *symbol = dlsym(apple_plugin_handle, name);
if (symbol == NULL) {
fprintf(stderr, "EAS local CAS plugin could not find %s: %s\n", name,
dlerror());
abort();
}
return symbol;
}
#define APPLE_PLUGIN(name, type) ((type)apple_plugin_symbol(#name))
void llcas_get_plugin_version(uint32_t *major, uint32_t *minor) {
typedef void (*fn_t)(uint32_t *, uint32_t *);
APPLE_PLUGIN(llcas_get_plugin_version, fn_t)(major, minor);
}
void llcas_string_dispose(char *value) {
typedef void (*fn_t)(char *);
APPLE_PLUGIN(llcas_string_dispose, fn_t)(value);
}
void llcas_cancellable_cancel(llcas_cancellable_t token) {
typedef void (*fn_t)(llcas_cancellable_t);
APPLE_PLUGIN(llcas_cancellable_cancel, fn_t)(token);
}
void llcas_cancellable_dispose(llcas_cancellable_t token) {
typedef void (*fn_t)(llcas_cancellable_t);
APPLE_PLUGIN(llcas_cancellable_dispose, fn_t)(token);
}
llcas_cas_options_t llcas_cas_options_create(void) {
typedef llcas_cas_options_t (*fn_t)(void);
return APPLE_PLUGIN(llcas_cas_options_create, fn_t)();
}
void llcas_cas_options_dispose(llcas_cas_options_t options) {
typedef void (*fn_t)(llcas_cas_options_t);
APPLE_PLUGIN(llcas_cas_options_dispose, fn_t)(options);
}
void llcas_cas_options_set_client_version(llcas_cas_options_t options,
uint32_t major, uint32_t minor) {
typedef void (*fn_t)(llcas_cas_options_t, uint32_t, uint32_t);
APPLE_PLUGIN(llcas_cas_options_set_client_version, fn_t)(options, major, minor);
}
void llcas_cas_options_set_ondisk_path(llcas_cas_options_t options,
const char *path) {
typedef void (*fn_t)(llcas_cas_options_t, const char *);
APPLE_PLUGIN(llcas_cas_options_set_ondisk_path, fn_t)(options, path);
}
bool llcas_cas_options_set_option(llcas_cas_options_t options, const char *name,
const char *value, char **error) {
if (name != NULL && strcmp(name, "remote-service-path") == 0) {
if (error != NULL) {
*error = NULL;
}
return false;
}
typedef bool (*fn_t)(llcas_cas_options_t, const char *, const char *,
char **);
return APPLE_PLUGIN(llcas_cas_options_set_option, fn_t)(options, name, value,
error);
}
llcas_cas_t llcas_cas_create(llcas_cas_options_t options, char **error) {
typedef llcas_cas_t (*fn_t)(llcas_cas_options_t, char **);
return APPLE_PLUGIN(llcas_cas_create, fn_t)(options, error);
}
void llcas_cas_dispose(llcas_cas_t cas) {
typedef void (*fn_t)(llcas_cas_t);
APPLE_PLUGIN(llcas_cas_dispose, fn_t)(cas);
}
int64_t llcas_cas_get_ondisk_size(llcas_cas_t cas, char **error) {
typedef int64_t (*fn_t)(llcas_cas_t, char **);
return APPLE_PLUGIN(llcas_cas_get_ondisk_size, fn_t)(cas, error);
}
bool llcas_cas_set_ondisk_size_limit(llcas_cas_t cas, int64_t size,
char **error) {
typedef bool (*fn_t)(llcas_cas_t, int64_t, char **);
return APPLE_PLUGIN(llcas_cas_set_ondisk_size_limit, fn_t)(cas, size, error);
}
bool llcas_cas_prune_ondisk_data(llcas_cas_t cas, char **error) {
typedef bool (*fn_t)(llcas_cas_t, char **);
return APPLE_PLUGIN(llcas_cas_prune_ondisk_data, fn_t)(cas, error);
}
char *llcas_cas_get_hash_schema_name(llcas_cas_t cas) {
typedef char *(*fn_t)(llcas_cas_t);
return APPLE_PLUGIN(llcas_cas_get_hash_schema_name, fn_t)(cas);
}
uint32_t llcas_digest_parse(llcas_cas_t cas, const char *printed,
uint8_t *bytes, size_t bytes_size, char **error) {
typedef uint32_t (*fn_t)(llcas_cas_t, const char *, uint8_t *, size_t,
char **);
return APPLE_PLUGIN(llcas_digest_parse, fn_t)(cas, printed, bytes,
bytes_size, error);
}
bool llcas_digest_print(llcas_cas_t cas, llcas_digest_t digest, char **printed,
char **error) {
typedef bool (*fn_t)(llcas_cas_t, llcas_digest_t, char **, char **);
return APPLE_PLUGIN(llcas_digest_print, fn_t)(cas, digest, printed, error);
}
bool llcas_cas_get_objectid(llcas_cas_t cas, llcas_digest_t digest,
llcas_objectid_t *object, char **error) {
typedef bool (*fn_t)(llcas_cas_t, llcas_digest_t, llcas_objectid_t *,
char **);
return APPLE_PLUGIN(llcas_cas_get_objectid, fn_t)(cas, digest, object, error);
}
llcas_digest_t llcas_objectid_get_digest(llcas_cas_t cas,
llcas_objectid_t object) {
typedef llcas_digest_t (*fn_t)(llcas_cas_t, llcas_objectid_t);
return APPLE_PLUGIN(llcas_objectid_get_digest, fn_t)(cas, object);
}
llcas_lookup_result_t llcas_cas_contains_object(llcas_cas_t cas,
llcas_objectid_t object,
bool globally, char **error) {
typedef llcas_lookup_result_t (*fn_t)(llcas_cas_t, llcas_objectid_t, bool,
char **);
return APPLE_PLUGIN(llcas_cas_contains_object, fn_t)(cas, object, globally,
error);
}
llcas_lookup_result_t llcas_cas_load_object(llcas_cas_t cas,
llcas_objectid_t object,
llcas_loaded_object_t *loaded,
char **error) {
typedef llcas_lookup_result_t (*fn_t)(llcas_cas_t, llcas_objectid_t,
llcas_loaded_object_t *, char **);
return APPLE_PLUGIN(llcas_cas_load_object, fn_t)(cas, object, loaded, error);
}
void llcas_cas_load_object_async(llcas_cas_t cas, llcas_objectid_t object,
void *context,
llcas_cas_load_object_cb callback,
llcas_cancellable_t *cancellable) {
typedef void (*fn_t)(llcas_cas_t, llcas_objectid_t, void *,
llcas_cas_load_object_cb, llcas_cancellable_t *);
APPLE_PLUGIN(llcas_cas_load_object_async,
fn_t)(cas, object, context, callback, cancellable);
}
bool llcas_cas_store_object(llcas_cas_t cas, llcas_data_t data,
const llcas_objectid_t *refs, size_t refs_count,
llcas_objectid_t *stored, char **error) {
typedef bool (*fn_t)(llcas_cas_t, llcas_data_t, const llcas_objectid_t *,
size_t, llcas_objectid_t *, char **);
return APPLE_PLUGIN(llcas_cas_store_object,
fn_t)(cas, data, refs, refs_count, stored, error);
}
llcas_data_t llcas_loaded_object_get_data(llcas_cas_t cas,
llcas_loaded_object_t object) {
typedef llcas_data_t (*fn_t)(llcas_cas_t, llcas_loaded_object_t);
return APPLE_PLUGIN(llcas_loaded_object_get_data, fn_t)(cas, object);
}
llcas_object_refs_t llcas_loaded_object_get_refs(llcas_cas_t cas,
llcas_loaded_object_t object) {
typedef llcas_object_refs_t (*fn_t)(llcas_cas_t, llcas_loaded_object_t);
return APPLE_PLUGIN(llcas_loaded_object_get_refs, fn_t)(cas, object);
}
size_t llcas_object_refs_get_count(llcas_cas_t cas, llcas_object_refs_t refs) {
typedef size_t (*fn_t)(llcas_cas_t, llcas_object_refs_t);
return APPLE_PLUGIN(llcas_object_refs_get_count, fn_t)(cas, refs);
}
llcas_objectid_t llcas_object_refs_get_id(llcas_cas_t cas,
llcas_object_refs_t refs,
size_t index) {
typedef llcas_objectid_t (*fn_t)(llcas_cas_t, llcas_object_refs_t, size_t);
return APPLE_PLUGIN(llcas_object_refs_get_id, fn_t)(cas, refs, index);
}
llcas_lookup_result_t llcas_actioncache_get_for_digest(llcas_cas_t cas,
llcas_digest_t digest,
llcas_objectid_t *result,
bool globally,
char **error) {
typedef llcas_lookup_result_t (*fn_t)(llcas_cas_t, llcas_digest_t,
llcas_objectid_t *, bool, char **);
return APPLE_PLUGIN(llcas_actioncache_get_for_digest,
fn_t)(cas, digest, result, globally, error);
}
void llcas_actioncache_get_for_digest_async(llcas_cas_t cas,
llcas_digest_t digest,
bool globally, void *context,
llcas_actioncache_get_cb callback,
llcas_cancellable_t *cancellable) {
typedef void (*fn_t)(llcas_cas_t, llcas_digest_t, bool, void *,
llcas_actioncache_get_cb, llcas_cancellable_t *);
APPLE_PLUGIN(llcas_actioncache_get_for_digest_async,
fn_t)(cas, digest, globally, context, callback, cancellable);
}
bool llcas_actioncache_put_for_digest(llcas_cas_t cas, llcas_digest_t digest,
llcas_objectid_t value, bool globally,
char **error) {
typedef bool (*fn_t)(llcas_cas_t, llcas_digest_t, llcas_objectid_t, bool,
char **);
return APPLE_PLUGIN(llcas_actioncache_put_for_digest,
fn_t)(cas, digest, value, globally, error);
}
void llcas_actioncache_put_for_digest_async(llcas_cas_t cas,
llcas_digest_t digest,
llcas_objectid_t value,
bool globally, void *context,
llcas_actioncache_put_cb callback,
llcas_cancellable_t *cancellable) {
typedef void (*fn_t)(llcas_cas_t, llcas_digest_t, llcas_objectid_t, bool,
void *, llcas_actioncache_put_cb, llcas_cancellable_t *);
APPLE_PLUGIN(llcas_actioncache_put_for_digest_async,
fn_t)(cas, digest, value, globally, context, callback,
cancellable);
}
+37 -17
View File
@@ -18,8 +18,12 @@ import { downloadApplicationArchiveAsync } from '../ios/resign';
import { resolveArtifactPath, resolveBuildConfiguration, resolveScheme } from '../ios/resolve';
import { Sentry } from '../sentry';
import { parseAndReportXcactivitylog } from '../steps/utils/ios/xcactivitylog';
import { cacheStatsAsync, restoreCcacheAsync } from '../steps/functions/restoreBuildCache';
import { saveCcacheAsync } from '../steps/functions/saveBuildCache';
import {
cacheStatsAsync,
restoreCcacheAsync,
restoreXcodeCompilationCacheAsync,
} from '../steps/functions/restoreBuildCache';
import { saveCcacheAsync, saveXcodeCompilationCacheAsync } from '../steps/functions/saveBuildCache';
import { uploadApplicationArchive } from '../utils/artifacts';
import {
configureExpoUpdatesIfInstalledAsync,
@@ -100,13 +104,21 @@ async function buildInnerAsync(
return;
}
await ctx.cacheManager?.restoreCache(ctx);
await restoreCcacheAsync({
logger: ctx.logger,
workingDirectory,
platform: ctx.job.platform,
env: ctx.env,
secrets: ctx.job.secrets,
});
await Promise.all([
restoreCcacheAsync({
logger: ctx.logger,
workingDirectory,
platform: ctx.job.platform,
env: ctx.env,
secrets: ctx.job.secrets,
}),
restoreXcodeCompilationCacheAsync({
logger: ctx.logger,
workingDirectory,
env: ctx.env,
secrets: ctx.job.secrets,
}),
]);
});
await ctx.runBuildPhase(BuildPhase.INSTALL_PODS, async () => {
@@ -242,14 +254,22 @@ async function buildInnerAsync(
return;
}
await ctx.cacheManager?.saveCache(ctx);
await saveCcacheAsync({
logger: ctx.logger,
workingDirectory,
platform: ctx.job.platform,
evictUsedBefore,
env: ctx.env,
secrets: ctx.job.secrets,
});
await Promise.all([
saveCcacheAsync({
logger: ctx.logger,
workingDirectory,
platform: ctx.job.platform,
evictUsedBefore,
env: ctx.env,
secrets: ctx.job.secrets,
}),
saveXcodeCompilationCacheAsync({
logger: ctx.logger,
workingDirectory,
env: ctx.env,
secrets: ctx.job.secrets,
}),
]);
});
await ctx.runBuildPhase(BuildPhase.CACHE_STATS, async () => {
@@ -0,0 +1,161 @@
import spawn from '@expo/turtle-spawn';
import { vol } from 'memfs';
import os from 'os';
import path from 'path';
import { createMockLogger } from '../../__tests__/utils/logger';
import {
XCODE_COMPILATION_CACHE_ENV,
XCODE_COMPILATION_CACHE_RELATIVE_PATH,
compressXcodeCompilationCacheAsync,
decompressXcodeCompilationCacheAsync,
prepareXcodeCompilationCacheEnvAsync,
} from '../compilationCache';
jest.mock('@expo/turtle-spawn', () => ({
__esModule: true,
default: jest.fn(),
}));
const DEVELOPER_DIRECTORY = '/Applications/Xcode.app/Contents/Developer';
const APPLE_PLUGIN_PATH = path.join(DEVELOPER_DIRECTORY, 'usr/lib/libToolchainCASPlugin.dylib');
const BUNDLED_SHIM_PATH = path.resolve(
__dirname,
'../../../bin/libeas_xcode_local_cas_plugin.dylib'
);
describe(prepareXcodeCompilationCacheEnvAsync, () => {
test('does nothing without the explicit opt-in', async () => {
const result = await prepareXcodeCompilationCacheEnvAsync({
derivedDataPath: '/project/ios/build',
env: {},
logger: createMockLogger(),
});
expect(result).toEqual({});
expect(spawn).not.toHaveBeenCalled();
});
test('enables compilation caching after validating the bundled arm64 shim', async () => {
vol.fromJSON({
[BUNDLED_SHIM_PATH]: 'bundled shim',
[APPLE_PLUGIN_PATH]: 'Apple plugin',
});
vol.mkdirSync(os.tmpdir(), { recursive: true });
jest.mocked(spawn).mockResolvedValue({} as any);
const result = await prepareXcodeCompilationCacheEnvAsync({
derivedDataPath: '/project/ios/build',
env: {
[XCODE_COMPILATION_CACHE_ENV]: '1',
DEVELOPER_DIR: DEVELOPER_DIRECTORY,
},
logger: createMockLogger(),
});
expect(spawn).toHaveBeenCalledWith(
'xcrun',
expect.arrayContaining([
'llvm-cas',
`--fcas-plugin-path=${BUNDLED_SHIM_PATH}`,
'--fcas-plugin-option=remote-service-path=/dev/null',
'--ingest',
BUNDLED_SHIM_PATH,
]),
expect.objectContaining({
env: expect.objectContaining({
EAS_XCODE_LOCAL_CAS_APPLE_PLUGIN: APPLE_PLUGIN_PATH,
}),
})
);
expect(result).toEqual({
EAS_XCODE_LOCAL_CAS_APPLE_PLUGIN: APPLE_PLUGIN_PATH,
GYM_XCARGS: `COMPILATION_CACHE_ENABLE_CACHING=YES COMPILATION_CACHE_ENABLE_DIAGNOSTIC_REMARKS=YES COMPILATION_CACHE_ENABLE_PLUGIN=YES COMPILATION_CACHE_PLUGIN_PATH=${BUNDLED_SHIM_PATH} COMPILATION_CACHE_REMOTE_SERVICE_PATH=/dev/null`,
GYM_DERIVED_DATA_PATH: '/project/ios/build',
});
});
test('does not enable compilation caching when the bundled shim is missing', async () => {
const logger = createMockLogger();
vol.fromJSON({
[APPLE_PLUGIN_PATH]: 'Apple plugin',
});
const result = await prepareXcodeCompilationCacheEnvAsync({
derivedDataPath: '/project/ios/build',
env: {
[XCODE_COMPILATION_CACHE_ENV]: '1',
DEVELOPER_DIR: DEVELOPER_DIRECTORY,
},
logger,
});
expect(spawn).not.toHaveBeenCalled();
expect(result).toEqual({});
expect(logger.warn).toHaveBeenCalledWith(
expect.objectContaining({ err: expect.any(Error) }),
expect.stringContaining('compilation caching is disabled')
);
});
test('does not enable compilation caching when the bundled shim is incompatible', async () => {
const logger = createMockLogger();
vol.fromJSON({
[BUNDLED_SHIM_PATH]: 'bundled shim',
[APPLE_PLUGIN_PATH]: 'Apple plugin',
});
vol.mkdirSync(os.tmpdir(), { recursive: true });
jest.mocked(spawn).mockRejectedValue(new Error('incompatible plugin'));
const result = await prepareXcodeCompilationCacheEnvAsync({
derivedDataPath: '/project/ios/build',
env: {
[XCODE_COMPILATION_CACHE_ENV]: '1',
DEVELOPER_DIR: DEVELOPER_DIRECTORY,
},
logger,
});
expect(result).toEqual({});
expect(logger.warn).toHaveBeenCalledWith(
expect.objectContaining({ err: expect.any(Error) }),
expect.stringContaining('compilation caching is disabled')
);
});
});
describe('Xcode compilation cache archives', () => {
test('uses system tar to preserve sparse files', async () => {
const workingDirectory = '/working-directory';
vol.fromJSON({
[path.join(workingDirectory, XCODE_COMPILATION_CACHE_RELATIVE_PATH, 'cache.db')]: 'cache',
});
vol.mkdirSync(os.tmpdir(), { recursive: true });
jest.mocked(spawn).mockResolvedValue({} as any);
const { archivePath } = await compressXcodeCompilationCacheAsync({
workingDirectory,
env: {},
logger: createMockLogger(),
});
expect(spawn).toHaveBeenCalledWith(
'tar',
['-czf', archivePath, '-C', workingDirectory, XCODE_COMPILATION_CACHE_RELATIVE_PATH],
expect.any(Object)
);
await decompressXcodeCompilationCacheAsync({
archivePath,
workingDirectory,
env: {},
logger: createMockLogger(),
});
expect(spawn).toHaveBeenCalledWith(
'tar',
['-xzSf', archivePath, '-C', workingDirectory],
expect.any(Object)
);
});
});
@@ -6,6 +6,7 @@ import path from 'path';
import { createTestIosJob } from '../../__tests__/utils/job';
import { createMockLogger } from '../../__tests__/utils/logger';
import { BuildContext } from '../../context';
import * as CompilationCache from '../compilationCache';
import { Credentials } from '../credentials/manager';
import { DistributionType } from '../credentials/provisioningProfile';
import { runFastlaneGym } from '../fastlane';
@@ -23,13 +24,19 @@ const WORKING_DIR = '/workingdir';
// BuildContext.getReactNativeProjectDirectory() nests the RN project under `${workingdir}/build`.
const IOS_DIR = path.join(WORKING_DIR, 'build', 'ios');
function makeIosBuildContext({ simulator }: { simulator: boolean }): BuildContext<Ios.Job> {
function makeIosBuildContext({
simulator,
env = {},
}: {
simulator: boolean;
env?: Record<string, string>;
}): BuildContext<Ios.Job> {
const job: Ios.Job = { ...createTestIosJob(), simulator };
return new BuildContext(job, {
workingdir: WORKING_DIR,
logger: createMockLogger(),
logBuffer: { getLogs: () => [], getPhaseLogs: () => [] },
env: { __API_SERVER_URL: 'http://api.expo.test' },
env: { __API_SERVER_URL: 'http://api.expo.test', ...env },
uploadArtifact: jest.fn(),
});
}
@@ -71,6 +78,10 @@ describe(runFastlaneGym, () => {
(spawn as jest.Mock).mockClear();
});
afterEach(() => {
jest.restoreAllMocks();
});
describe.each([
{
name: 'archive',
@@ -125,4 +136,68 @@ describe(runFastlaneGym, () => {
workspacePath: IOS_DIR,
});
});
it('enables local Swift, C, and Objective-C compilation caching', async () => {
jest.spyOn(CompilationCache, 'prepareXcodeCompilationCacheEnvAsync').mockResolvedValue({
GYM_XCARGS:
'OTHER_SWIFT_FLAGS=-warnings-as-errors COMPILATION_CACHE_ENABLE_CACHING=YES COMPILATION_CACHE_ENABLE_DIAGNOSTIC_REMARKS=YES COMPILATION_CACHE_ENABLE_PLUGIN=YES COMPILATION_CACHE_PLUGIN_PATH=/workingdir/bin/libeas_xcode_local_cas_plugin.dylib COMPILATION_CACHE_REMOTE_SERVICE_PATH=/dev/null',
EAS_XCODE_LOCAL_CAS_APPLE_PLUGIN:
'/Applications/Xcode.app/Contents/Developer/usr/lib/libToolchainCASPlugin.dylib',
GYM_DERIVED_DATA_PATH: path.join(IOS_DIR, 'build'),
});
const ctx = makeIosBuildContext({
simulator: false,
env: {
EAS_BUILD_XCODE_COMPILATION_CACHE: '1',
GYM_XCARGS: 'OTHER_SWIFT_FLAGS=-warnings-as-errors',
},
});
await runFastlaneGym(ctx, {
scheme: 'App',
credentials: ARCHIVE_CREDENTIALS,
entitlements: null,
});
expect(spawn).toHaveBeenCalledWith(
'fastlane',
['gym'],
expect.objectContaining({
env: expect.objectContaining({
GYM_XCARGS:
'OTHER_SWIFT_FLAGS=-warnings-as-errors COMPILATION_CACHE_ENABLE_CACHING=YES COMPILATION_CACHE_ENABLE_DIAGNOSTIC_REMARKS=YES COMPILATION_CACHE_ENABLE_PLUGIN=YES COMPILATION_CACHE_PLUGIN_PATH=/workingdir/bin/libeas_xcode_local_cas_plugin.dylib COMPILATION_CACHE_REMOTE_SERVICE_PATH=/dev/null',
EAS_XCODE_LOCAL_CAS_APPLE_PLUGIN:
'/Applications/Xcode.app/Contents/Developer/usr/lib/libToolchainCASPlugin.dylib',
GYM_DERIVED_DATA_PATH: path.join(IOS_DIR, 'build'),
}),
})
);
});
it('does not enable compilation caching when the local plugin cannot be prepared', async () => {
const ctx = makeIosBuildContext({
simulator: false,
env: {
EAS_BUILD_XCODE_COMPILATION_CACHE: '1',
},
});
await runFastlaneGym(ctx, {
scheme: 'App',
credentials: ARCHIVE_CREDENTIALS,
entitlements: null,
});
expect(spawn).toHaveBeenCalledWith(
'fastlane',
['gym'],
expect.objectContaining({
env: expect.not.objectContaining({ GYM_XCARGS: expect.any(String) }),
})
);
expect(ctx.logger.warn).toHaveBeenCalledWith(
expect.objectContaining({ err: expect.any(Error) }),
expect.stringContaining('compilation caching is disabled')
);
});
});
@@ -0,0 +1,194 @@
import { Env, Platform } from '@expo/eas-build-job';
import { bunyan } from '@expo/logger';
import spawn from '@expo/turtle-spawn';
import { createHash } from 'crypto';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
import { CACHE_KEY_PREFIX_BY_PLATFORM, generateDefaultBuildCacheKeyAsync } from '../utils/cacheKey';
export const XCODE_COMPILATION_CACHE_ENV = 'EAS_BUILD_XCODE_COMPILATION_CACHE';
export const XCODE_COMPILATION_CACHE_RELATIVE_PATH = 'ios/build/CompilationCache.noindex';
export function getXcodeCompilationCachePath(derivedDataPath: string): string {
return path.join(derivedDataPath, 'CompilationCache.noindex');
}
async function prepareXcodeLocalCompilationCachePluginAsync({
env,
logger,
}: {
env: Env;
logger: bunyan;
}): Promise<{ pluginPath: string; applePluginPath: string }> {
const bundledPluginPath = path.resolve(
__dirname,
'../../bin/libeas_xcode_local_cas_plugin.dylib'
);
const developerDirectory =
env.DEVELOPER_DIR ??
(
await spawn('xcode-select', ['--print-path'], {
env,
logger,
})
).stdout.trim();
const applePluginPath = path.join(
developerDirectory,
'usr',
'lib',
'libToolchainCASPlugin.dylib'
);
if (!(await fs.pathExists(applePluginPath))) {
throw new Error(`The Xcode CAS plugin is missing at ${applePluginPath}.`);
}
if (!(await fs.pathExists(bundledPluginPath))) {
throw new Error(`The EAS local CAS plugin is missing at ${bundledPluginPath}.`);
}
const validationDirectory = await fs.mkdtemp(
path.join(os.tmpdir(), 'eas-xcode-local-cache-validation-')
);
try {
await spawn(
'xcrun',
[
'llvm-cas',
`--fcas-plugin-path=${bundledPluginPath}`,
'--fcas-plugin-option=remote-service-path=/dev/null',
`--cas=${path.join(validationDirectory, 'cas')}`,
'--ingest',
bundledPluginPath,
],
{
env: { ...env, EAS_XCODE_LOCAL_CAS_APPLE_PLUGIN: applePluginPath },
logger,
}
);
} finally {
await fs.remove(validationDirectory);
}
return { pluginPath: bundledPluginPath, applePluginPath };
}
export async function compressXcodeCompilationCacheAsync({
workingDirectory,
env,
logger,
}: {
workingDirectory: string;
env: Env;
logger: bunyan;
}): Promise<{ archivePath: string }> {
const cachePath = path.join(workingDirectory, XCODE_COMPILATION_CACHE_RELATIVE_PATH);
if (!(await fs.pathExists(cachePath))) {
throw new Error(`The Xcode compilation cache does not exist at ${cachePath}.`);
}
const archiveDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'xcode-compilation-cache-'));
const archivePath = path.join(archiveDirectory, 'cache.tar.gz');
await spawn(
'tar',
['-czf', archivePath, '-C', workingDirectory, XCODE_COMPILATION_CACHE_RELATIVE_PATH],
{ env, logger }
);
return { archivePath };
}
export async function decompressXcodeCompilationCacheAsync({
archivePath,
workingDirectory,
env,
logger,
}: {
archivePath: string;
workingDirectory: string;
env: Env;
logger: bunyan;
}): Promise<void> {
await fs.ensureDir(workingDirectory);
await spawn('tar', ['-xzSf', archivePath, '-C', workingDirectory], { env, logger });
}
export async function generateXcodeCompilationCacheKeyAsync({
workingDirectory,
env,
logger,
}: {
workingDirectory: string;
env: Env;
logger: bunyan;
}): Promise<{ key: string; keyPrefix: string; xcodeVersion: string }> {
const xcodeVersion = (
await spawn('xcodebuild', ['-version'], {
env,
logger,
})
).stdout.trim();
const xcodeVersionHash = createHash('sha256').update(xcodeVersion).digest('hex').slice(0, 16);
const ccacheKey = await generateDefaultBuildCacheKeyAsync(workingDirectory, Platform.IOS);
const dependencyHash = ccacheKey.slice(CACHE_KEY_PREFIX_BY_PLATFORM[Platform.IOS].length);
const keyPrefix = `ios-xcode-compilation-cache-${xcodeVersionHash}-`;
return {
key: `${keyPrefix}${dependencyHash}`,
keyPrefix,
xcodeVersion: xcodeVersion.replace(/\s+/g, ' '),
};
}
export async function prepareXcodeCompilationCacheEnvAsync({
derivedDataPath,
env,
logger,
}: {
derivedDataPath: string;
env: Env;
logger: bunyan;
}): Promise<Env> {
if (env[XCODE_COMPILATION_CACHE_ENV] !== '1') {
return {};
}
let plugin: { pluginPath: string; applePluginPath: string };
try {
plugin = await prepareXcodeLocalCompilationCachePluginAsync({
env,
logger,
});
logger.info(`The EAS local Xcode CAS plugin is ready at ${plugin.pluginPath}.`);
} catch (error: any) {
logger.warn(
{ err: error },
'The EAS local Xcode CAS plugin is missing or incompatible. Xcode compilation caching is disabled.'
);
return {};
}
logger.info(
`Xcode compilation caching is enabled. The cache directory is ${getXcodeCompilationCachePath(derivedDataPath)}.`
);
const pluginXcargs = [
'COMPILATION_CACHE_ENABLE_PLUGIN=YES',
`COMPILATION_CACHE_PLUGIN_PATH=${plugin.pluginPath}`,
// A non-empty remote service path makes Xcode schedule C, Objective-C,
// precompiled-module, and precompiled-header cache tasks. The EAS plugin
// consumes this option and never opens the path or performs remote I/O.
'COMPILATION_CACHE_REMOTE_SERVICE_PATH=/dev/null',
].join(' ');
return {
GYM_XCARGS: [
env.GYM_XCARGS,
'COMPILATION_CACHE_ENABLE_CACHING=YES COMPILATION_CACHE_ENABLE_DIAGNOSTIC_REMARKS=YES',
pluginXcargs,
]
.filter(Boolean)
.join(' '),
GYM_DERIVED_DATA_PATH: derivedDataPath,
EAS_XCODE_LOCAL_CAS_APPLE_PLUGIN: plugin.applePluginPath,
};
}
+7 -1
View File
@@ -7,6 +7,7 @@ import os from 'os';
import path from 'path';
import type { Credentials } from './credentials/manager';
import * as CompilationCache from './compilationCache';
import { createFastfileForResigningBuild } from './fastfile';
import { createGymfileForArchiveBuild, createGymfileForSimulatorBuild } from './gymfile';
import { isTVOS } from './tvos';
@@ -46,13 +47,18 @@ export async function runFastlaneGym<TJob extends Ios.Job>(
if (ctx.skipNativeBuild) {
throw new SkipNativeBuildError('Skipping fastlane build');
}
const compilationCacheEnv = await CompilationCache.prepareXcodeCompilationCacheEnvAsync({
derivedDataPath,
env: ctx.env,
logger: ctx.logger,
});
const buildLogger = new XcodeBuildLogger(ctx.logger, ctx.getReactNativeProjectDirectory());
void buildLogger.watchLogFiles(ctx.buildLogsDirectory);
try {
await runFastlane(['gym'], {
cwd: workspacePath,
logger: ctx.logger,
env: { ...ctx.env, ...extraEnv },
env: { ...ctx.env, ...extraEnv, ...compilationCacheEnv },
});
} finally {
await buildLogger.flush();
@@ -0,0 +1,110 @@
import { Platform } from '@expo/eas-build-job';
import { vol } from 'memfs';
import { createMockLogger } from '../../../__tests__/utils/logger';
import {
XCODE_COMPILATION_CACHE_ENV,
XCODE_COMPILATION_CACHE_RELATIVE_PATH,
compressXcodeCompilationCacheAsync,
decompressXcodeCompilationCacheAsync,
generateXcodeCompilationCacheKeyAsync,
} from '../../../ios/compilationCache';
import { downloadCacheAsync } from '../restoreCache';
import { restoreXcodeCompilationCacheAsync } from '../restoreBuildCache';
import { uploadCacheAsync } from '../saveCache';
import { saveXcodeCompilationCacheAsync } from '../saveBuildCache';
jest.mock('../restoreCache', () => ({
downloadCacheAsync: jest.fn(),
}));
jest.mock('../saveCache', () => ({
uploadCacheAsync: jest.fn(),
}));
jest.mock('../../../ios/compilationCache', () => ({
XCODE_COMPILATION_CACHE_ENV: 'EAS_BUILD_XCODE_COMPILATION_CACHE',
XCODE_COMPILATION_CACHE_RELATIVE_PATH: 'ios/build/CompilationCache.noindex',
compressXcodeCompilationCacheAsync: jest.fn(),
decompressXcodeCompilationCacheAsync: jest.fn(),
generateXcodeCompilationCacheKeyAsync: jest.fn(),
}));
const WORKING_DIRECTORY = '/working-directory';
const ENV = {
[XCODE_COMPILATION_CACHE_ENV]: '1',
EAS_USE_CACHE: '1',
EAS_BUILD_ID: 'build-id',
__API_SERVER_URL: 'https://api.expo.test',
};
const SECRETS = { robotAccessToken: 'robot-access-token' };
describe('Xcode compilation build cache', () => {
beforeEach(() => {
jest.mocked(generateXcodeCompilationCacheKeyAsync).mockResolvedValue({
key: 'ios-xcode-compilation-cache-version-dependencies',
keyPrefix: 'ios-xcode-compilation-cache-version-',
xcodeVersion: 'Xcode 26.0 Build version 17A000',
});
});
test('restores through the active build cache service', async () => {
jest.mocked(downloadCacheAsync).mockResolvedValue({
archivePath: '/tmp/cache.tar.gz',
matchedKey: 'ios-xcode-compilation-cache-version-dependencies',
});
await restoreXcodeCompilationCacheAsync({
logger: createMockLogger(),
workingDirectory: WORKING_DIRECTORY,
env: ENV,
secrets: SECRETS,
});
expect(downloadCacheAsync).toHaveBeenCalledWith(
expect.objectContaining({
jobId: 'build-id',
key: 'ios-xcode-compilation-cache-version-dependencies',
keyPrefixes: ['ios-xcode-compilation-cache-version-'],
paths: [XCODE_COMPILATION_CACHE_RELATIVE_PATH],
platform: Platform.IOS,
})
);
expect(decompressXcodeCompilationCacheAsync).toHaveBeenCalledWith(
expect.objectContaining({
archivePath: '/tmp/cache.tar.gz',
workingDirectory: WORKING_DIRECTORY,
})
);
});
test('saves through the active build cache service', async () => {
vol.fromJSON({ '/tmp/xcode-cache.tar.gz': 'cache' });
jest.mocked(compressXcodeCompilationCacheAsync).mockResolvedValue({
archivePath: '/tmp/xcode-cache.tar.gz',
});
await saveXcodeCompilationCacheAsync({
logger: createMockLogger(),
workingDirectory: WORKING_DIRECTORY,
env: ENV,
secrets: SECRETS,
});
expect(compressXcodeCompilationCacheAsync).toHaveBeenCalledWith(
expect.objectContaining({
workingDirectory: WORKING_DIRECTORY,
})
);
expect(uploadCacheAsync).toHaveBeenCalledWith(
expect.objectContaining({
jobId: 'build-id',
key: 'ios-xcode-compilation-cache-version-dependencies',
paths: [XCODE_COMPILATION_CACHE_RELATIVE_PATH],
platform: Platform.IOS,
size: 5,
})
);
});
});
@@ -20,6 +20,12 @@ import {
getCcachePath,
} from '../../utils/cacheKey';
import { Datadog } from '../../datadog';
import {
XCODE_COMPILATION_CACHE_ENV,
XCODE_COMPILATION_CACHE_RELATIVE_PATH,
decompressXcodeCompilationCacheAsync,
generateXcodeCompilationCacheKeyAsync,
} from '../../ios/compilationCache';
import { GRADLE_CACHE_KEY_PREFIX, generateGradleCacheKeyAsync } from '../../utils/gradleCacheKey';
import { TurtleFetchError, turtleFetch } from '../../utils/turtleFetch';
@@ -48,26 +54,91 @@ export function createRestoreBuildCacheFunction(): BuildFunction {
);
}
await restoreCcacheAsync({
logger,
workingDirectory,
platform,
env,
secrets: stepCtx.global.staticContext.job.secrets,
});
if (platform === Platform.ANDROID) {
await restoreGradleCacheAsync({
await Promise.all([
restoreCcacheAsync({
logger,
workingDirectory,
platform,
env,
secrets: stepCtx.global.staticContext.job.secrets,
});
}
}),
platform === Platform.IOS
? restoreXcodeCompilationCacheAsync({
logger,
workingDirectory,
env,
secrets: stepCtx.global.staticContext.job.secrets,
})
: restoreGradleCacheAsync({
logger,
workingDirectory,
env,
secrets: stepCtx.global.staticContext.job.secrets,
}),
]);
},
});
}
export async function restoreXcodeCompilationCacheAsync({
logger,
workingDirectory,
env,
secrets,
}: {
logger: bunyan;
workingDirectory: string;
env: Record<string, string | undefined>;
secrets?: { robotAccessToken?: string };
}): Promise<void> {
const buildCacheEnabled =
env.EAS_RESTORE_CACHE === '1' || (env.EAS_USE_CACHE === '1' && env.EAS_RESTORE_CACHE !== '0');
if (!buildCacheEnabled || env[XCODE_COMPILATION_CACHE_ENV] !== '1') {
return;
}
try {
const robotAccessToken = nullthrows(
secrets?.robotAccessToken,
'Robot access token is required for cache operations'
);
const expoApiServerURL = nullthrows(env.__API_SERVER_URL, '__API_SERVER_URL is not set');
const jobId = nullthrows(env.EAS_BUILD_ID, 'EAS_BUILD_ID is not set');
const { key, keyPrefix, xcodeVersion } = await generateXcodeCompilationCacheKeyAsync({
workingDirectory,
env,
logger,
});
logger.info(`Restoring Xcode compilation cache for ${xcodeVersion} with key: ${key}`);
const { archivePath, matchedKey } = await downloadCacheAsync({
logger,
jobId,
expoApiServerURL,
robotAccessToken,
paths: [XCODE_COMPILATION_CACHE_RELATIVE_PATH],
key,
keyPrefixes: [keyPrefix],
platform: Platform.IOS,
});
await decompressXcodeCompilationCacheAsync({
archivePath,
workingDirectory,
env,
logger,
});
logger.info(
`Xcode compilation cache restored ${matchedKey === key ? '(direct hit)' : '(prefix match)'}`
);
} catch (err: unknown) {
if (err instanceof TurtleFetchError && err.response?.status === 404) {
logger.info('No Xcode compilation cache found for this key');
} else {
logger.warn({ err }, 'Failed to restore Xcode compilation cache');
}
}
}
export function createCacheStatsBuildFunction(): BuildFunction {
return new BuildFunction({
namespace: 'eas',
@@ -14,6 +14,12 @@ import os from 'os';
import path from 'path';
import { compressCacheAsync, uploadCacheAsync } from './saveCache';
import {
XCODE_COMPILATION_CACHE_ENV,
XCODE_COMPILATION_CACHE_RELATIVE_PATH,
compressXcodeCompilationCacheAsync,
generateXcodeCompilationCacheKeyAsync,
} from '../../ios/compilationCache';
import { formatBytes } from '../../utils/artifacts';
import { generateDefaultBuildCacheKeyAsync, getCcachePath } from '../../utils/cacheKey';
import { generateGradleCacheKeyAsync } from '../../utils/gradleCacheKey';
@@ -43,27 +49,87 @@ export function createSaveBuildCacheFunction(evictUsedBefore: Date): BuildFuncti
);
}
await saveCcacheAsync({
logger,
workingDirectory,
platform,
evictUsedBefore,
env,
secrets: stepCtx.global.staticContext.job.secrets,
});
if (platform === Platform.ANDROID) {
await saveGradleCacheAsync({
await Promise.all([
saveCcacheAsync({
logger,
workingDirectory,
platform,
evictUsedBefore,
env,
secrets: stepCtx.global.staticContext.job.secrets,
});
}
}),
platform === Platform.IOS
? saveXcodeCompilationCacheAsync({
logger,
workingDirectory,
env,
secrets: stepCtx.global.staticContext.job.secrets,
})
: saveGradleCacheAsync({
logger,
workingDirectory,
env,
secrets: stepCtx.global.staticContext.job.secrets,
}),
]);
},
});
}
export async function saveXcodeCompilationCacheAsync({
logger,
workingDirectory,
env,
secrets,
}: {
logger: bunyan;
workingDirectory: string;
env: Record<string, string | undefined>;
secrets?: { robotAccessToken?: string };
}): Promise<void> {
const buildCacheEnabled =
env.EAS_SAVE_CACHE === '1' || (env.EAS_USE_CACHE === '1' && env.EAS_SAVE_CACHE !== '0');
if (!buildCacheEnabled || env[XCODE_COMPILATION_CACHE_ENV] !== '1') {
return;
}
try {
const { key, xcodeVersion } = await generateXcodeCompilationCacheKeyAsync({
workingDirectory,
env,
logger,
});
logger.info(`Saving Xcode compilation cache for ${xcodeVersion} with key: ${key}`);
const jobId = nullthrows(env.EAS_BUILD_ID, 'EAS_BUILD_ID is not set');
const robotAccessToken = nullthrows(
secrets?.robotAccessToken,
'Robot access token is required for cache operations'
);
const expoApiServerURL = nullthrows(env.__API_SERVER_URL, '__API_SERVER_URL is not set');
const { archivePath } = await compressXcodeCompilationCacheAsync({
workingDirectory,
env,
logger,
});
const { size } = await fs.promises.stat(archivePath);
logger.info(`Xcode compilation cache archive size: ${formatBytes(size)}`);
await uploadCacheAsync({
logger,
jobId,
expoApiServerURL,
robotAccessToken,
archivePath,
key,
paths: [XCODE_COMPILATION_CACHE_RELATIVE_PATH],
size,
platform: Platform.IOS,
});
} catch (err) {
logger.error({ err }, 'Failed to save Xcode compilation cache');
}
}
export async function saveCcacheAsync({
logger,
workingDirectory,
@@ -5,6 +5,7 @@ import path from 'path';
import { COMMON_FASTLANE_ENV } from '../../../common/fastlane';
import { XcodeBuildLogger } from '../../../common/xcpretty';
import * as CompilationCache from '../../../ios/compilationCache';
export async function runFastlaneGym({
workingDir,
@@ -19,14 +20,23 @@ export async function runFastlaneGym({
env: BuildStepEnv;
extraEnv?: BuildStepEnv;
}): Promise<void> {
const workspacePath = path.join(workingDir, 'ios');
const derivedDataPath = path.join(workspacePath, 'build');
// When caching is enabled, this also sets GYM_DERIVED_DATA_PATH so Fastlane
// and the cache restore/save operations use this exact directory.
const compilationCacheEnv = await CompilationCache.prepareXcodeCompilationCacheEnvAsync({
derivedDataPath,
env,
logger,
});
const buildLogger = new XcodeBuildLogger(logger, workingDir);
void buildLogger.watchLogFiles(buildLogsDirectory);
try {
await runFastlane(['gym'], {
cwd: path.join(workingDir, 'ios'),
cwd: workspacePath,
logger,
env,
extraEnv,
extraEnv: { ...extraEnv, ...compilationCacheEnv },
});
} finally {
await buildLogger.flush();
+14 -4
View File
@@ -103,8 +103,8 @@ fi
if [[ "$PLATFORM" == "ios" ]]; then
record_sim_package_dir="$ROOT_DIR/packages/build-tools/resources/record-sim"
record_sim_bin_dir="$target_root_dir/packages/build-tools/bin"
mkdir -p "$record_sim_bin_dir"
build_tools_bin_dir="$target_root_dir/packages/build-tools/bin"
mkdir -p "$build_tools_bin_dir"
record_sim_bin_path=$(swift build \
-c release \
--package-path "$record_sim_package_dir" \
@@ -114,8 +114,18 @@ if [[ "$PLATFORM" == "ios" ]]; then
-c release \
--package-path "$record_sim_package_dir" \
--build-path "$record_sim_build_dir"
cp "$record_sim_bin_path/record-sim" "$record_sim_bin_dir/record-sim"
chmod +x "$record_sim_bin_dir/record-sim"
cp "$record_sim_bin_path/record-sim" "$build_tools_bin_dir/record-sim"
chmod +x "$build_tools_bin_dir/record-sim"
xcrun clang \
-arch arm64 \
-dynamiclib \
-O2 \
-Wall \
-Wextra \
-Werror \
-o "$build_tools_bin_dir/libeas_xcode_local_cas_plugin.dylib" \
"$ROOT_DIR/packages/build-tools/resources/xcode-local-cache-plugin/eas_local_cas_plugin.c"
# build plugin
pushd "$ROOT_DIR/packages/expo-cocoapods-proxy" >/dev/null 2>&1