Compare commits

..
Author SHA1 Message Date
Brent Vatne 72b8a78145 [worker] Apply log masking to uploaded Xcode build logs
Pass the job's log masks to the Xcode build log upload and mask the file
line by line while streaming it, including values written in the escaped
form Xcode uses for build settings.
2026-10-20 16:31:19 -07:00
51 changed files with 339 additions and 977 deletions

No files matched your search

@@ -1,5 +0,0 @@
---
'@expo/build-tools': patch
---
Retry the build lookup and the archive download of `eas/download_build` on transient failures, and name the failure cause in the build log.
-5
View File
@@ -1,5 +0,0 @@
---
'eas-cli': none
---
Make command descriptions in `eas --help` consistently lowercase without trailing periods.
-6
View File
@@ -1,6 +0,0 @@
---
'eas-cli': none
'@expo/build-tools': none
---
Bump Node.js to 22.23.4 to fix the CVEs in the [October 2026 Node.js security releases](https://nodejs.org/en/blog/vulnerability/october-2026-security-releases).
-5
View File
@@ -1,5 +0,0 @@
---
'@expo/build-tools': minor
---
Stop sandboxes after an optional maximum duration measured from readiness.
@@ -0,0 +1,5 @@
---
'@expo/worker': patch
---
Apply the job log's secret masking to uploaded Xcode build logs.
+1 -1
View File
@@ -6,4 +6,4 @@ MISE_NODE_COREPACK = "true"
actionlint = "1.7.12"
gcloud = "570.0.0"
jq = "1.8.1"
node = "22.23.4"
node = "22.23.3"
+1 -1
View File
@@ -26,7 +26,7 @@
"build:egress-guard": "resources/egress-guard/build.sh",
"test:egress-guard": "resources/egress-guard/tests/run-policy-tests.sh && resources/egress-guard/tests/run-guard-tests.sh",
"typecheck": "tsc",
"generate-appium-commands": "mise exec [email protected]3.4 -- node scripts/generate-appium-commands.js",
"generate-appium-commands": "mise exec [email protected]2.0 -- node scripts/generate-appium-commands.js",
"prepack": "rimraf dist \"*.tsbuildinfo\" && yarn gql && tsc -p tsconfig.build.json",
"jest-unit": "jest --config jest/unit-config.ts",
"jest-integration": "jest --config jest/integration-config.ts",
@@ -11,7 +11,6 @@ import * as tar from 'tar';
import { createGlobalContextMock } from '../../../__tests__/utils/context';
import { createMockLogger } from '../../../__tests__/utils/logger';
import { Datadog } from '../../../datadog';
import * as files from '../../../utils/files';
import { createDownloadBuildFunction, downloadBuildAsync } from '../downloadBuild';
@@ -31,17 +30,11 @@ const APPLICATION_ARCHIVE_URL = `https://expo.dev/artifacts/eas/${randomUUID()}.
function createMockGraphqlClient({
applicationArchiveUrl,
error,
transientError,
}: {
applicationArchiveUrl?: string | null;
error?: Error;
transientError?: Error;
}): Client {
const toPromise = jest.fn();
if (transientError) {
toPromise.mockResolvedValueOnce({ error: transientError, data: undefined });
}
toPromise.mockResolvedValue(
const toPromise = jest.fn().mockResolvedValue(
error
? { error, data: undefined }
: {
@@ -69,7 +62,7 @@ function createSuccessfulResponse({
url,
headers,
}: {
body: Buffer | Readable;
body: Buffer;
url: string;
headers?: Record<string, string>;
}): Response {
@@ -79,7 +72,7 @@ function createSuccessfulResponse({
return {
ok: true,
body: body instanceof Readable ? body : Readable.from(body),
body: Readable.from(body),
url,
headers: {
get: (name: string): string | null => normalizedHeaders[name.toLowerCase()] ?? null,
@@ -87,47 +80,6 @@ function createSuccessfulResponse({
} as unknown as Response;
}
function createUndiciNetworkError(code: string): CombinedError {
return new CombinedError({
networkError: Object.assign(new TypeError('fetch failed'), {
cause: Object.assign(new Error(`read ${code}`), { code }),
}),
});
}
const { FetchError: ActualFetchError } = jest.requireActual(
'node-fetch'
) as typeof import('node-fetch');
function createSocketFetchError(url: string, code: string): Error {
return new ActualFetchError(`request to ${url} failed, reason: connect ${code}`, 'system', {
code,
} as any);
}
function useFakeBackoffTimers(): void {
// Streams and fs callbacks need the real nextTick and setImmediate; only the backoff timers are faked.
jest.useFakeTimers({ doNotFake: ['nextTick', 'setImmediate', 'queueMicrotask'] });
}
const MAX_IO_TURNS_BEFORE_BACKOFF = 1000;
async function waitForBackoffTimerAsync(): Promise<void> {
for (let turns = 0; jest.getTimerCount() === 0; turns++) {
if (turns === MAX_IO_TURNS_BEFORE_BACKOFF) {
throw new Error('No backoff timer was scheduled.');
}
await new Promise(resolve => setImmediate(resolve));
}
}
async function fireBackoffTimersAsync(count: number): Promise<void> {
for (let i = 0; i < count; i++) {
await waitForBackoffTimerAsync();
await jest.advanceTimersToNextTimerAsync();
}
}
async function createFlatAppTarGzBufferAsync(): Promise<Buffer> {
const sourceDirectory = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'flat-app-source-'));
const archivePath = path.join(os.tmpdir(), `${randomUUID()}.tar.gz`);
@@ -156,11 +108,6 @@ async function createFlatAppTarGzBufferAsync(): Promise<Buffer> {
}
}
afterEach(() => {
jest.useRealTimers();
jest.restoreAllMocks();
});
describe('downloadBuild', () => {
it('downloads from applicationArchiveUrl returned by GraphQL', async () => {
const buildId = randomUUID();
@@ -518,13 +465,13 @@ describe('downloadBuild', () => {
expect(jest.mocked(fetch)).not.toHaveBeenCalled();
});
it('throws SystemError with the cause code after retrying a failing build lookup', async () => {
useFakeBackoffTimers();
it('throws SystemError when GraphQL request fails with a network error', async () => {
const buildId = randomUUID();
const graphqlClient = createMockGraphqlClient({
error: createUndiciNetworkError('ECONNRESET'),
error: new CombinedError({
networkError: new Error('Network request failed'),
}),
});
const datadogLog = jest.spyOn(Datadog, 'log');
const promise = downloadBuildAsync({
logger: createLogger({ name: 'test' }),
@@ -533,81 +480,19 @@ describe('downloadBuild', () => {
robotAccessToken: null,
extensions: ['app'],
});
const rejection = expect(promise).rejects.toBeInstanceOf(SystemError);
await fireBackoffTimersAsync(5);
await rejection;
await expect(promise).rejects.toBeInstanceOf(SystemError);
await expect(promise).rejects.toMatchObject({
message: `Could not fetch build ${buildId}: [Network] fetch failed (ECONNRESET)`,
});
expect(graphqlClient.query).toHaveBeenCalledTimes(6);
expect(datadogLog).toHaveBeenCalledWith('download_build.build_lookup failed on attempt 6', {
event: 'network_retry',
operation: 'download_build.build_lookup',
outcome: 'failed',
attempts: '6',
code: 'ECONNRESET',
message: `Could not fetch build ${buildId}: [Network] Network request failed`,
});
expect(jest.mocked(fetch)).not.toHaveBeenCalled();
});
it.each([
['a network error', createUndiciNetworkError('ECONNRESET'), 'ECONNRESET'],
[
'a transient GraphQL error',
new CombinedError({
graphQLErrors: [
{
message: 'Too many requests',
extensions: { isTransient: true, errorCode: 'TOO_MANY_REQUESTS' },
},
],
response: { status: 200 },
}),
'TOO_MANY_REQUESTS',
],
])('downloads after a build lookup that failed once with %s', async (_, transientError, code) => {
useFakeBackoffTimers();
const buildId = randomUUID();
const graphqlClient = createMockGraphqlClient({
applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
transientError,
});
jest
.mocked(fetch)
.mockResolvedValue(
createSuccessfulResponse({ body: Buffer.from('hello'), url: 'https://cdn.test/app.apk' })
);
const datadogLog = jest.spyOn(Datadog, 'log');
const promise = downloadBuildAsync({
logger: createLogger({ name: 'test' }),
buildId,
graphqlClient,
robotAccessToken: null,
extensions: ['apk'],
});
await fireBackoffTimersAsync(1);
const { artifactPath } = await promise;
expect(path.basename(artifactPath)).toBe('app.apk');
expect(graphqlClient.query).toHaveBeenCalledTimes(2);
expect(datadogLog).toHaveBeenCalledWith('download_build.build_lookup recovered on attempt 2', {
event: 'network_retry',
operation: 'download_build.build_lookup',
outcome: 'recovered',
attempts: '2',
code,
});
});
it('throws SystemError when GraphQL request returns 5xx', async () => {
useFakeBackoffTimers();
const buildId = randomUUID();
const graphqlClient = createMockGraphqlClient({
error: new CombinedError({
graphQLErrors: [{ message: 'Internal Server Error' }],
error: Object.assign(new Error('Internal Server Error'), {
response: { status: 500 },
}),
});
@@ -619,14 +504,12 @@ describe('downloadBuild', () => {
robotAccessToken: null,
extensions: ['app'],
});
const rejection = expect(promise).rejects.toBeInstanceOf(SystemError);
await fireBackoffTimersAsync(5);
await rejection;
await expect(promise).rejects.toBeInstanceOf(SystemError);
await expect(promise).rejects.toMatchObject({
message: `Could not fetch build ${buildId}: [GraphQL] Internal Server Error`,
message: `Could not fetch build ${buildId}: Internal Server Error`,
});
expect(graphqlClient.query).toHaveBeenCalledTimes(6);
expect(jest.mocked(fetch)).not.toHaveBeenCalled();
});
@@ -651,193 +534,10 @@ describe('downloadBuild', () => {
errorCode: 'EAS_DOWNLOAD_BUILD_FETCH_FAILED',
message: `Could not fetch build ${buildId}: [GraphQL] Build not found`,
});
expect(graphqlClient.query).toHaveBeenCalledTimes(1);
expect(jest.mocked(fetch)).not.toHaveBeenCalled();
});
it('downloads after an archive request that failed once with a socket error', async () => {
useFakeBackoffTimers();
const graphqlClient = createMockGraphqlClient({});
jest
.mocked(fetch)
.mockRejectedValueOnce(createSocketFetchError(APPLICATION_ARCHIVE_URL, 'ECONNRESET'))
.mockResolvedValueOnce(
createSuccessfulResponse({ body: Buffer.from('hello'), url: 'https://cdn.test/app.apk' })
);
const promise = downloadBuildAsync({
logger: createLogger({ name: 'test' }),
applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
graphqlClient,
robotAccessToken: null,
extensions: ['apk'],
});
await fireBackoffTimersAsync(1);
const { artifactPath } = await promise;
expect(path.basename(artifactPath)).toBe('app.apk');
expect(jest.mocked(fetch)).toHaveBeenCalledTimes(2);
});
it('throws SystemError with the cause code after retrying a failing build archive request', async () => {
useFakeBackoffTimers();
const graphqlClient = createMockGraphqlClient({
applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
});
jest
.mocked(fetch)
.mockRejectedValue(createSocketFetchError(APPLICATION_ARCHIVE_URL, 'ETIMEDOUT'));
const datadogLog = jest.spyOn(Datadog, 'log');
const promise = downloadBuildAsync({
logger: createLogger({ name: 'test' }),
buildId: randomUUID(),
graphqlClient,
robotAccessToken: null,
extensions: ['apk'],
});
const rejection = expect(promise).rejects.toBeInstanceOf(SystemError);
await fireBackoffTimersAsync(5);
await rejection;
await expect(promise).rejects.toMatchObject({
message: `Could not download the application archive: request to ${APPLICATION_ARCHIVE_URL} failed, reason: connect ETIMEDOUT`,
});
expect(jest.mocked(fetch)).toHaveBeenCalledTimes(6);
expect(datadogLog).toHaveBeenCalledWith('download_build.archive_download failed on attempt 6', {
event: 'network_retry',
operation: 'download_build.archive_download',
outcome: 'failed',
attempts: '6',
code: 'ETIMEDOUT',
});
});
function createBrokenBody(code: string): Readable {
return Readable.from(
(async function* () {
yield Buffer.from('he');
throw Object.assign(new Error(`body failed with ${code}`), { code });
})()
);
}
it.each(['ERR_STREAM_PREMATURE_CLOSE', 'ECONNRESET'])(
'downloads again when the archive body breaks off with %s after the headers arrived',
async code => {
useFakeBackoffTimers();
const graphqlClient = createMockGraphqlClient({});
const brokenBody = createBrokenBody(code);
jest
.mocked(fetch)
.mockResolvedValueOnce(
createSuccessfulResponse({ body: brokenBody, url: 'https://cdn.test/app.apk' })
)
.mockResolvedValueOnce(
createSuccessfulResponse({ body: Buffer.from('hello'), url: 'https://cdn.test/app.apk' })
);
const datadogLog = jest.spyOn(Datadog, 'log');
const promise = downloadBuildAsync({
logger: createLogger({ name: 'test' }),
applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
graphqlClient,
robotAccessToken: null,
extensions: ['apk'],
});
await fireBackoffTimersAsync(1);
const { artifactPath } = await promise;
expect(await fs.promises.readFile(artifactPath, 'utf-8')).toBe('hello');
expect(jest.mocked(fetch)).toHaveBeenCalledTimes(2);
expect(datadogLog).toHaveBeenCalledWith(
'download_build.archive_download recovered on attempt 2',
expect.objectContaining({ outcome: 'recovered', code })
);
}
);
it('throws SystemError when the build archive body keeps breaking off', async () => {
useFakeBackoffTimers();
const graphqlClient = createMockGraphqlClient({
applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
});
jest.mocked(fetch).mockImplementation(async () =>
createSuccessfulResponse({
body: createBrokenBody('ECONNRESET'),
url: 'https://cdn.test/app.apk',
})
);
const datadogLog = jest.spyOn(Datadog, 'log');
const promise = downloadBuildAsync({
logger: createLogger({ name: 'test' }),
buildId: randomUUID(),
graphqlClient,
robotAccessToken: null,
extensions: ['apk'],
});
const rejection = expect(promise).rejects.toBeInstanceOf(SystemError);
await fireBackoffTimersAsync(5);
await rejection;
await expect(promise).rejects.toMatchObject({
message: 'Could not download the application archive: body failed with ECONNRESET',
});
expect(jest.mocked(fetch)).toHaveBeenCalledTimes(6);
expect(datadogLog).toHaveBeenCalledWith(
'download_build.archive_download failed on attempt 6',
expect.objectContaining({ outcome: 'failed', code: 'ECONNRESET' })
);
});
it('does not start another archive attempt after an abort during the backoff', async () => {
useFakeBackoffTimers();
const graphqlClient = createMockGraphqlClient({});
jest
.mocked(fetch)
.mockRejectedValueOnce(createSocketFetchError(APPLICATION_ARCHIVE_URL, 'ECONNRESET'));
const controller = new AbortController();
const reason = new Error('step timed out');
const promise = downloadBuildAsync({
logger: createLogger({ name: 'test' }),
applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
graphqlClient,
robotAccessToken: null,
extensions: ['apk'],
signal: controller.signal,
});
const rejection = expect(promise).rejects.toBe(reason);
await waitForBackoffTimerAsync();
controller.abort(reason);
await jest.advanceTimersToNextTimerAsync();
await rejection;
expect(jest.mocked(fetch)).toHaveBeenCalledTimes(1);
});
it('rethrows the raw error of a failing user-supplied archive URL after the retries', async () => {
useFakeBackoffTimers();
const graphqlClient = createMockGraphqlClient({});
jest
.mocked(fetch)
.mockRejectedValue(createSocketFetchError(APPLICATION_ARCHIVE_URL, 'ENOTFOUND'));
const promise = downloadBuildAsync({
logger: createLogger({ name: 'test' }),
applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
graphqlClient,
robotAccessToken: null,
extensions: ['apk'],
});
const rejection = expect(promise).rejects.toBeInstanceOf(ActualFetchError);
await fireBackoffTimersAsync(5);
await rejection;
expect(jest.mocked(fetch)).toHaveBeenCalledTimes(6);
});
it('should throw an error if no matching files are found', async () => {
const graphqlClient = createMockGraphqlClient({
applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
@@ -905,21 +605,6 @@ describe('createDownloadBuildFunction', () => {
};
}
function createApplicationResponse(buildId: string): Response {
return new ActualResponse(
JSON.stringify({
data: {
build: {
id: buildId,
gitCommitHash: null,
applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
},
},
}),
{ status: 200 }
);
}
function createStep(ctx: any, callInputs: Record<string, unknown>) {
return createDownloadBuildFunction(ctx).createBuildStepFromFunctionCall(
createGlobalContextMock({
@@ -935,7 +620,18 @@ describe('createDownloadBuildFunction', () => {
const graphqlClient = createMockGraphqlClient({});
jest.mocked(fetch).mockImplementation(async url => {
if (url === workerApiUrl) {
return createApplicationResponse(buildId);
return new ActualResponse(
JSON.stringify({
data: {
build: {
id: buildId,
gitCommitHash: null,
applicationArchiveUrl: APPLICATION_ARCHIVE_URL,
},
},
}),
{ status: 200 }
);
}
return {
ok: false,
@@ -965,33 +661,6 @@ describe('createDownloadBuildFunction', () => {
expect(graphqlClient.query).not.toHaveBeenCalled();
});
it('retries the worker API lookup after a 5xx response', async () => {
useFakeBackoffTimers();
const buildId = randomUUID();
const graphqlClient = createMockGraphqlClient({});
jest
.mocked(fetch)
.mockResolvedValueOnce(new ActualResponse('Service Unavailable', { status: 503 }))
.mockResolvedValueOnce(createApplicationResponse(buildId))
.mockResolvedValueOnce(
createSuccessfulResponse({ body: Buffer.from('hello'), url: 'https://cdn.test/app.apk' })
);
const execution = createStep(createSessionCtx(graphqlClient), {
build_id: buildId,
extensions: ['apk'],
}).executeAsync();
await fireBackoffTimersAsync(1);
await execution;
expect(jest.mocked(fetch)).toHaveBeenCalledTimes(3);
expect(jest.mocked(fetch).mock.calls.map(([url]) => url)).toEqual([
workerApiUrl,
workerApiUrl,
APPLICATION_ARCHIVE_URL,
]);
});
it('does not call the worker API for an application_archive_url input', async () => {
const graphqlClient = createMockGraphqlClient({});
jest.mocked(fetch).mockResolvedValue({
@@ -1,4 +1,4 @@
import { SystemError, UserError } from '@expo/eas-build-job';
import { SystemError } from '@expo/eas-build-job';
import { Client, fetchExchange } from '@urql/core';
import * as sandboxDaemon from '../../utils/sandboxDaemon';
@@ -6,90 +6,6 @@ import * as sandboxDaemon from '../../utils/sandboxDaemon';
import { createStartSandboxBuildFunction, markSandboxReadyAsync } from '../startSandbox';
describe('sandbox build functions', () => {
it.each([
{ completion: 'duration', duration: 0.02 },
{ completion: 'cancellation', duration: 60 },
{ completion: 'duration', duration: 0.0001 },
])(
'stops after $completion with $duration seconds only after readiness',
async ({ completion, duration }) => {
const controller = new AbortController();
let ready!: () => void;
const stopAsync = jest.fn(async () => {});
const start = jest.spyOn(sandboxDaemon, 'startSandboxDaemonAsync').mockResolvedValue({
ready: new Promise<void>(resolve => {
ready = resolve;
}),
stopAsync,
});
const mutation = jest.fn(() => ({ toPromise: async () => ({ data: {} }) }));
const logger = { info: jest.fn(), child: jest.fn() };
logger.child.mockReturnValue(logger);
const fn = createStartSandboxBuildFunction({
mcpServerUrl: 'ws://localhost:8787',
graphqlClient: { mutation },
} as any);
try {
const pending = fn.fn!({ logger } as any, {
inputs: {
sandbox_id: { value: 'sandbox-id' },
max_duration_seconds: { value: duration },
},
outputs: {},
env: { __EAS_SANDBOX_MCP_TOKEN: 'token' },
signal: controller.signal,
});
await new Promise(resolve => setTimeout(resolve, 40));
expect(mutation).not.toHaveBeenCalled();
expect(stopAsync).not.toHaveBeenCalled();
ready();
if (completion === 'cancellation') {
await new Promise(resolve => setImmediate(resolve));
expect(stopAsync).not.toHaveBeenCalled();
controller.abort();
}
await pending;
expect(mutation).toHaveBeenCalledTimes(1);
expect(stopAsync).toHaveBeenCalledTimes(1);
const timeoutMessage = `Sandbox reached its maximum runtime of ${duration} seconds. Stopping.`;
if (completion === 'duration') {
expect(logger.info).toHaveBeenCalledWith(timeoutMessage);
} else {
expect(logger.info).not.toHaveBeenCalledWith(timeoutMessage);
}
} finally {
start.mockRestore();
}
}
);
it.each([0, -1, Infinity, NaN])(
'rejects invalid maximum duration %s before starting the daemon',
async duration => {
const start = jest.spyOn(sandboxDaemon, 'startSandboxDaemonAsync');
const fn = createStartSandboxBuildFunction({
mcpServerUrl: 'ws://localhost:8787',
} as any);
try {
const result = fn.fn!({} as any, {
inputs: {
sandbox_id: { value: 'sandbox-id' },
max_duration_seconds: { value: duration },
},
outputs: {},
env: { __EAS_SANDBOX_MCP_TOKEN: 'token' },
});
await expect(result).rejects.toBeInstanceOf(UserError);
await expect(result).rejects.toThrow(
`Invalid sandbox maximum duration: ${duration}. Provide a finite number greater than zero, or omit max_duration_seconds.`
);
expect(start).not.toHaveBeenCalled();
} finally {
start.mockRestore();
}
}
);
it('aborts an active ready request and keeps its authentication headers', async () => {
const controller = new AbortController();
let requestStarted!: () => void;
@@ -7,7 +7,7 @@ import {
BuildStepInputValueTypeName,
BuildStepOutput,
} from '@expo/steps';
import { Client, CombinedError } from '@urql/core';
import { Client } from '@urql/core';
import contentDisposition from 'content-disposition';
import { glob } from 'fast-glob';
import { graphql } from 'gql.tada';
@@ -26,16 +26,10 @@ import { CustomBuildContext } from '../../customBuildContext';
import { formatBytes } from '../../utils/artifacts';
import { decompressTarAsync, isFileTarGzAsync } from '../../utils/files';
import { graphqlAbortContext } from '../../utils/graphqlAbort';
import {
describeError,
isTransferFailure,
isTransientGraphqlError,
} from '../../utils/networkErrors';
import { retryOnNetworkFailureAsync } from '../../utils/retryOnNetworkFailure';
import { retryOnDNSFailure } from '../../utils/retryOnDNSFailure';
import { pluralize } from '../../utils/strings';
import {
type DeviceRunSessionWorkerApi,
DeviceRunSessionWorkerApiError,
getDeviceRunSessionWorkerApi,
} from '../utils/deviceRunSessionWorkerApi';
@@ -158,34 +152,21 @@ async function fetchApplicationArchiveUrlAsync({
signal,
});
}
let data;
try {
data = await retryOnNetworkFailureAsync(
async () => {
const queryResult = await graphqlClient
.query(BUILD_BY_ID_QUERY, { buildId }, graphqlAbortContext(signal))
.toPromise();
if (queryResult.error) {
throw queryResult.error;
}
return queryResult.data;
},
{
operation: 'download_build.build_lookup',
signal,
isRetryable: error => error instanceof CombinedError && isTransientGraphqlError(error),
}
);
} catch (error) {
signal?.throwIfAborted();
const message = `Could not fetch build ${buildId}: ${describeError(error)}`;
throw error instanceof CombinedError && !isTransientGraphqlError(error)
? new UserError('EAS_DOWNLOAD_BUILD_FETCH_FAILED', message, { cause: error })
: new SystemError(message, { cause: error });
}
const result = await graphqlClient
.query(BUILD_BY_ID_QUERY, { buildId }, graphqlAbortContext(signal))
.toPromise();
signal?.throwIfAborted();
const applicationArchiveUrl = data?.builds.byId?.artifacts?.applicationArchiveUrl;
if (result.error) {
const { error } = result;
const message = `Could not fetch build ${buildId}: ${error.message}`;
throw error.networkError || result.error.response?.status >= 500
? new SystemError(message, { cause: error })
: new UserError('EAS_DOWNLOAD_BUILD_FETCH_FAILED', message, { cause: error });
}
const applicationArchiveUrl = result.data?.builds.byId?.artifacts?.applicationArchiveUrl;
if (!applicationArchiveUrl) {
throw new UserError(
'EAS_DOWNLOAD_BUILD_NO_APPLICATION_ARCHIVE',
@@ -214,18 +195,13 @@ async function fetchDeviceRunSessionApplicationArchiveUrlAsync({
}): Promise<string> {
let application;
try {
application = await retryOnNetworkFailureAsync(
() => deviceRunSessionWorkerApi.getApplicationAsync({ signal }),
{
operation: 'download_build.session_build_lookup',
signal,
isRetryable: error => error instanceof DeviceRunSessionWorkerApiError && error.isTransient,
}
);
application = await deviceRunSessionWorkerApi.getApplicationAsync({ signal });
} catch (err) {
signal?.throwIfAborted();
throw new SystemError(
`Could not fetch build ${buildId} for this device run session: ${describeError(err)}`,
`Could not fetch build ${buildId} for this device run session: ${
err instanceof Error ? err.message : String(err)
}`,
{ cause: err }
);
}
@@ -305,34 +281,18 @@ export async function downloadBuildAsync(
);
signal?.throwIfAborted();
let archivePath: string;
try {
archivePath = await retryOnNetworkFailureAsync(
async () => {
const response = await fetch(downloadUrl, { headers, signal });
if (!response.ok) {
const textResult = await asyncResult(response.text());
throw new Error(
`Unexpected response from server (${response.status}): ${textResult.value}`
);
}
const archiveFilename = resolveArchiveFilename({ response, extensions });
const attemptArchivePath = path.join(downloadDestinationDirectory, archiveFilename);
// On abort, node-fetch emits an error on the body stream, which fails the pipeline.
await streamPipeline(response.body, fs.createWriteStream(attemptArchivePath));
return attemptArchivePath;
},
{ operation: 'download_build.archive_download', signal, isRetryable: isTransferFailure }
);
} catch (error) {
signal?.throwIfAborted();
if (params.buildId && isTransferFailure(error)) {
throw new SystemError(`Could not download the application archive: ${describeError(error)}`, {
cause: error,
});
}
throw error;
const response = await retryOnDNSFailure(fetch)(downloadUrl, { headers, signal });
if (!response.ok) {
const textResult = await asyncResult(response.text());
throw new Error(`Unexpected response from server (${response.status}): ${textResult.value}`);
}
const archiveFilename = resolveArchiveFilename({ response, extensions });
const archivePath = path.join(downloadDestinationDirectory, archiveFilename);
// On abort, node-fetch emits an error on the body stream, which fails the pipeline.
await streamPipeline(response.body, fs.createWriteStream(archivePath));
signal?.throwIfAborted();
const { size } = await fs.promises.stat(archivePath);
@@ -1,4 +1,4 @@
import { SystemError, UserError } from '@expo/eas-build-job';
import { SystemError } from '@expo/eas-build-job';
import { BuildFunction, BuildStepInput, BuildStepInputValueTypeName } from '@expo/steps';
import { graphql } from 'gql.tada';
@@ -32,11 +32,6 @@ export function createStartSandboxBuildFunction(ctx: CustomBuildContext): BuildF
required: true,
allowedValueTypeName: BuildStepInputValueTypeName.STRING,
}),
BuildStepInput.createProvider({
id: 'max_duration_seconds',
required: false,
allowedValueTypeName: BuildStepInputValueTypeName.NUMBER,
}),
],
fn: async (stepCtx, { inputs, signal, env }) => {
// The daemon needs this credential, but shell commands do not.
@@ -49,18 +44,6 @@ export function createStartSandboxBuildFunction(ctx: CustomBuildContext): BuildF
throw new SystemError('MCP server URL is required to start the sandbox daemon.');
}
const sandboxId = String(inputs.sandbox_id.value);
const maxDurationSeconds = inputs.max_duration_seconds?.value;
if (
maxDurationSeconds !== undefined &&
(typeof maxDurationSeconds !== 'number' ||
!Number.isFinite(maxDurationSeconds) ||
maxDurationSeconds <= 0)
) {
throw new UserError(
'EAS_SANDBOX_INVALID_DURATION',
`Invalid sandbox maximum duration: ${String(maxDurationSeconds)}. Provide a finite number greater than zero, or omit max_duration_seconds.`
);
}
const daemon = await startSandboxDaemonAsync({
credential: sandboxToken,
serverUrl: mcpServerUrl,
@@ -80,21 +63,7 @@ export function createStartSandboxBuildFunction(ctx: CustomBuildContext): BuildF
stepCtx.logger.info('Sandbox daemon started.');
await withLogPhaseAsync(stepCtx.logger, AWAIT_SANDBOX_STEP_NAME, async () => {
const timeoutSignal =
maxDurationSeconds === undefined
? undefined
: AbortSignal.timeout(Math.ceil(maxDurationSeconds * 1000));
await waitForCompletionAsync(
AbortSignal.any([
...(signal ? [signal] : []),
...(timeoutSignal ? [timeoutSignal] : []),
])
);
if (timeoutSignal?.aborted) {
stepCtx.logger.info(
`Sandbox reached its maximum runtime of ${maxDurationSeconds} seconds. Stopping.`
);
}
await waitForCancellationAsync(signal);
});
} finally {
await daemon.stopAsync();
@@ -120,11 +89,11 @@ export async function markSandboxReadyAsync(
}
}
async function waitForCompletionAsync(signal: AbortSignal): Promise<void> {
if (signal.aborted) {
async function waitForCancellationAsync(signal?: AbortSignal): Promise<void> {
if (signal?.aborted) {
return;
}
await new Promise<void>(resolve => {
signal.addEventListener('abort', () => resolve(), { once: true });
signal?.addEventListener('abort', () => resolve(), { once: true });
});
}
@@ -330,8 +330,7 @@ describe(DeviceRunSessionWorkerApi, () => {
});
it('reports a request that failed before www responded', async () => {
const reason = new Error('socket hang up');
jest.mocked(turtleFetch).mockRejectedValueOnce(reason);
jest.mocked(turtleFetch).mockRejectedValueOnce(new Error('socket hang up'));
const error = await createApi()
.getSessionAsync()
@@ -339,7 +338,6 @@ describe(DeviceRunSessionWorkerApi, () => {
expect(error).toBeInstanceOf(DeviceRunSessionWorkerApiError);
expect(error.status).toBeUndefined();
expect(error.cause).toBe(reason);
expect(error.message).toContain('failed before www responded: socket hang up');
});
@@ -88,16 +88,11 @@ export class DeviceRunSessionWorkerApiError extends Error {
constructor(
message: string,
/** The HTTP status of the response, or undefined when no response arrived. */
readonly status: number | undefined,
options?: ErrorOptions
readonly status: number | undefined
) {
super(message, options);
super(message);
this.name = 'DeviceRunSessionWorkerApiError';
}
get isTransient(): boolean {
return this.status === undefined || this.status >= 500;
}
}
/**
@@ -223,8 +218,7 @@ export class DeviceRunSessionWorkerApi {
`${routeDescription} failed before www responded: ${
reason instanceof Error ? reason.message : String(reason)
}. Check the network connection of the worker.`,
undefined,
{ cause: reason }
undefined
);
}
const response = responseResult.value;
@@ -1,16 +1,4 @@
import { CombinedError } from '@urql/core';
import {
describeError,
findErrorCode,
findGraphqlErrorCode,
isConnectFailure,
isConnectionInterruptedError,
isTransferFailure,
isTransientGraphqlError,
} from '../networkErrors';
const { FetchError } = jest.requireActual('node-fetch') as typeof import('node-fetch');
import { isConnectFailure, isConnectionInterruptedError } from '../networkErrors';
it.each([
'EAI_AGAIN',
@@ -46,77 +34,3 @@ it.each(['ETIMEDOUT', 'CERT_HAS_EXPIRED', undefined])('does not automatically re
expect(isConnectFailure(error)).toBe(false);
expect(isConnectionInterruptedError(error)).toBe(false);
});
it('finds the errno code on the cause of an undici fetch failure', () => {
const error = new CombinedError({
networkError: new TypeError('fetch failed', {
cause: Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }),
}),
});
expect(findErrorCode(error)).toBe('ECONNRESET');
expect(findErrorCode(new Error('Request failed'))).toBeUndefined();
expect(findErrorCode('Request failed')).toBeUndefined();
});
it('finds the www error code on a GraphQL error', () => {
const error = new CombinedError({
graphQLErrors: [
{ message: 'Too many requests', extensions: { errorCode: 'TOO_MANY_REQUESTS' } },
],
});
expect(findGraphqlErrorCode(error)).toBe('TOO_MANY_REQUESTS');
expect(
findGraphqlErrorCode(new CombinedError({ graphQLErrors: ['Build not found'] }))
).toBeUndefined();
expect(findGraphqlErrorCode(new Error('Request failed'))).toBeUndefined();
});
it.each([
['Request failed', 'ECONNRESET', 'Request failed (ECONNRESET)'],
['read ECONNRESET', 'ECONNRESET', 'read ECONNRESET'],
['Request failed', undefined, 'Request failed'],
])('describes %p with code %p as %p', (message, code, expected) => {
expect(describeError(Object.assign(new Error(message), { code }))).toBe(expected);
});
it('treats a missing response, a 5xx response or an isTransient GraphQL error as transient', () => {
expect(
isTransientGraphqlError(new CombinedError({ networkError: new Error('fetch failed') }))
).toBe(true);
expect(
isTransientGraphqlError(
new CombinedError({ graphQLErrors: ['Internal Server Error'], response: { status: 503 } })
)
).toBe(true);
expect(
isTransientGraphqlError(
new CombinedError({
graphQLErrors: [{ message: 'Database unavailable', extensions: { isTransient: true } }],
response: { status: 200 },
})
)
).toBe(true);
expect(
isTransientGraphqlError(
new CombinedError({ graphQLErrors: ['Build not found'], response: { status: 200 } })
)
).toBe(false);
});
it.each([
[
'a node-fetch socket failure',
new FetchError('request failed', 'system', { code: 'ECONNRESET' } as any),
true,
],
['a node-fetch body size failure', new FetchError('too large', 'max-size'), false],
['a body reset', Object.assign(new Error('aborted'), { code: 'ECONNRESET' }), true],
[
'a body closed early',
Object.assign(new Error('closed'), { code: 'ERR_STREAM_PREMATURE_CLOSE' }),
true,
],
['an HTTP error response', new Error('Unexpected response from server (500)'), false],
])('classifies %s as a transfer failure or not', (_label, error, expected) => {
expect(isTransferFailure(error)).toBe(expected);
});
@@ -1,5 +1,3 @@
import { CombinedError } from '@urql/core';
const CONNECT_FAILURE_CODES = new Set([
'EAI_AGAIN',
'ECONNREFUSED',
@@ -32,50 +30,3 @@ export function isConnectionInterruptedError(error: unknown): boolean {
}
return false;
}
/** Errno code of the error or its causes; undici keeps it on cause, urql on networkError. */
export function findErrorCode(error: unknown): string | undefined {
let current: unknown = error;
while (current instanceof Error) {
if ('code' in current && typeof current.code === 'string') {
return current.code;
}
current = current instanceof CombinedError ? current.networkError : current.cause;
}
return undefined;
}
export function findGraphqlErrorCode(error: unknown): string | undefined {
if (!(error instanceof CombinedError)) {
return undefined;
}
const code = error.graphQLErrors.at(0)?.extensions.errorCode;
return typeof code === 'string' ? code : undefined;
}
export function describeError(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
const code = findErrorCode(error);
return code && !message.includes(code) ? `${message} (${code})` : message;
}
export function isTransientGraphqlError(error: CombinedError): boolean {
return (
Boolean(error.networkError) ||
error.response?.status >= 500 ||
error.graphQLErrors.some(graphqlError => graphqlError.extensions.isTransient === true)
);
}
// node-fetch throws type 'system' for a socket failure before the headers; after them it forwards
// ECONNRESET or ERR_STREAM_PREMATURE_CLOSE to the body. HTTP error responses never throw.
export function isTransferFailure(error: unknown): boolean {
return (
(error instanceof Error &&
error.name === 'FetchError' &&
'type' in error &&
error.type === 'system') ||
isConnectionInterruptedError(error) ||
findErrorCode(error) === 'ERR_STREAM_PREMATURE_CLOSE'
);
}
@@ -1,53 +0,0 @@
import { Datadog } from '../datadog';
import { findErrorCode, findGraphqlErrorCode } from './networkErrors';
import { promiseRetryWithCondition } from './promiseRetryWithCondition';
export async function retryOnNetworkFailureAsync<T>(
attemptAsync: () => Promise<T>,
{
operation,
signal,
isRetryable,
}: {
operation: string;
signal?: AbortSignal;
isRetryable: (error: unknown) => boolean;
}
): Promise<T> {
const shouldRetry = (error: unknown): boolean => !signal?.aborted && isRetryable(error);
let attempts = 1;
let lastError: unknown;
const report = (outcome: 'recovered' | 'failed', error: unknown): void => {
Datadog.log(`${operation} ${outcome} on attempt ${attempts}`, {
event: 'network_retry',
operation,
outcome,
attempts: String(attempts),
code: findErrorCode(error) ?? findGraphqlErrorCode(error) ?? 'unknown',
});
};
try {
const result = await promiseRetryWithCondition(
async () => {
signal?.throwIfAborted();
return await attemptAsync();
},
shouldRetry,
{ retries: 5, minTimeout: 1_000, factor: 2 },
({ attemptNumber, error }) => {
attempts = attemptNumber;
lastError = error;
}
)();
if (attempts > 1) {
report('recovered', lastError);
}
return result;
} catch (error) {
if (attempts > 1 && !signal?.aborted) {
report('failed', error);
}
throw error;
}
}
+1 -1
View File
@@ -252,7 +252,7 @@
},
"update": {
"node": {
"version": "22.23.4"
"version": "22.23.3"
},
"s3": {
"templates": {
+1 -1
View File
@@ -33,7 +33,7 @@ const PROJECT_PAGES: Record<string, string> = {
export default class Browse extends EasCommand {
static override description =
'transition from the terminal to the web browser to view and interact with your project on https://expo.dev';
'Transition from the terminal to the web browser to view and interact with your project on https://expo.dev';
static override args = {
page: Args.string({
@@ -13,7 +13,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
import { pollForBackgroundJobReceiptAsync } from '../../utils/pollForBackgroundJobReceiptAsync';
export default class ChannelDelete extends EasCommand {
static override description = 'delete a channel';
static override description = 'Delete a channel';
static override args = {
name: Args.string({
@@ -50,7 +50,7 @@ type ChannelRolloutArgsAndFlags = {
Partial<CreateRolloutNonInteractiveOptions>;
export default class ChannelRollout extends EasCommand {
static override description = 'roll a new branch out on a channel incrementally';
static override description = 'Roll a new branch out on a channel incrementally.';
static override args = {
channel: Args.string({
@@ -7,7 +7,7 @@ import { SetUpBuildCredentialsCommandAction } from '../../credentials/manager/Se
import { selectPlatformAsync } from '../../platform';
export default class InitializeBuildCredentials extends EasCommand {
static override description = 'set up credentials for building your project';
static override description = 'Set up credentials for building your project.';
static override flags = {
platform: Flags.option({
@@ -22,7 +22,7 @@ interface DeployAliasDeleteFlags {
}
export default class WorkerAliasDelete extends EasCommand {
static override description = 'delete deployment aliases';
static override description = 'Delete deployment aliases.';
static override aliases = ['worker:alias:delete'];
static override state = 'preview';
@@ -39,7 +39,7 @@ interface RawDeployAliasFlags {
}
export default class WorkerAlias extends EasCommand {
static override description = 'assign deployment aliases';
static override description = 'Assign deployment aliases.';
static override aliases = ['worker:alias', 'deploy:promote'];
static override state = 'preview';
@@ -23,7 +23,7 @@ interface RawDeployDeleteFlags {
}
export default class WorkerDelete extends EasCommand {
static override description = 'delete a deployment';
static override description = 'Delete a deployment.';
static override aliases = ['worker:delete'];
static override state = 'preview';
+1 -1
View File
@@ -100,7 +100,7 @@ async function withSuppressedOutputAsync<T>(fn: () => Promise<T>): Promise<T> {
/* eslint-enable no-console */
export default class Go extends EasCommand {
static override description = 'create a custom Expo Go and submit to TestFlight';
static override description = 'Create a custom Expo Go and submit to TestFlight';
static override hidden = true;
static override flags = {
@@ -22,7 +22,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
export default class ObserveEvent extends EasCommand {
static override description =
'display a single Observe event (metric, log, or error) by its ID. IDs are included in event data when the `--json` flag is passed to `eas observe:session`, `eas observe:metrics`, or `eas observe:events`';
'display a single Observe event (metric, log, or error) by its ID. IDs are included in event data when the `--json` flag is passed to `eas observe:session`, `eas observe:metrics`, or `eas observe:events`.';
static override args = {
id: Args.string({
@@ -36,7 +36,7 @@ const DEFAULT_EVENTS_LIMIT = 10;
export default class ObserveEvents extends EasCommand {
static override description =
'display individual events emitted by the app via `logEvent`, filtered by the event name in the argument. With no arguments, a list of the available event names and associated event counts is returned';
'display individual events emitted by the app via `logEvent`, filtered by the event name in the argument. With no arguments, a list of the available event names and associated event counts is returned.';
static override args = {
eventName: Args.string({
+1 -1
View File
@@ -134,7 +134,7 @@ export default class New extends EasCommand {
static override aliases = ['new'];
static override description =
'create a new project configured with Expo Application Services (EAS)';
'Create a new project configured with Expo Application Services (EAS)';
static override args = {
path: Args.string({
@@ -35,7 +35,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
export default class SimulatorControl extends EasCommand {
static override aliases = ['sim:control'];
static override description =
'[EXPERIMENTAL] run a serve-sim action on a remote iOS simulator session on EAS. Run without an action to list the actions the session supports. To save a screenshot locally, run `eas simulator:control screenshot.capture --output ./screenshot.png`';
'[EXPERIMENTAL] run a serve-sim action on a remote iOS simulator session on EAS. Run without an action to list the actions the session supports. To save a screenshot locally, run `eas simulator:control screenshot.capture --output ./screenshot.png`.';
static override strict = false;
static override examples = [
@@ -39,7 +39,7 @@ type DeviceRunSessionArtifact = DeviceRunSessionById['artifacts'][number];
export default class SimulatorGet extends EasCommand {
static override aliases = ['sim:get'];
static override description =
"[EXPERIMENTAL] get info about a remote simulator session on EAS by its simulator session ID. Preview links follow the session's saved sharing preference";
"[EXPERIMENTAL] get info about a remote simulator session on EAS by its simulator session ID. Preview links follow the session's saved sharing preference.";
static override flags = {
id: Flags.string({
@@ -83,7 +83,7 @@ export default class Simulator extends EasCommand {
static override aliases = ['simulator:start', 'sim:start'];
static override hiddenAliases = ['sim'];
static override description =
'[EXPERIMENTAL] start a remote simulator session on EAS and get instructions to connect to it. The default browser preview link requires sign-in';
'[EXPERIMENTAL] start a remote simulator session on EAS and get instructions to connect to it. The default browser preview link requires sign-in.';
static override flags = {
platform: Flags.option({
@@ -9,7 +9,7 @@ import { promptAsync } from '../../prompts';
export default class WorkflowRunCancel extends EasCommand {
static override description =
'cancel one or more workflow runs. If no workflow run IDs are provided, you will be prompted to select IN_PROGRESS runs to cancel';
'Cancel one or more workflow runs. If no workflow run IDs are provided, you will be prompted to select IN_PROGRESS runs to cancel.';
static override strict = false;
@@ -9,7 +9,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
export default class WorkflowList extends EasCommand {
static override hidden = true;
static override description = 'list workflows for the current project';
static override description = 'List workflows for the current project';
static override flags = {
...EasJsonOnlyFlag,
@@ -26,7 +26,7 @@ function printLogsForAllSteps(logs: JobLogs): void {
export default class WorkflowLogView extends EasCommand {
static override description =
'view logs for a workflow run, selecting a job and step to view. You can pass in either a workflow run ID or a job ID. If no ID is passed in, you will be prompted to select from recent workflow runs for the current project';
'view logs for a workflow run, selecting a job and step to view. You can pass in either a workflow run ID or a job ID. If no ID is passed in, you will be prompted to select from recent workflow runs for the current project.';
static override flags = {
...EasJsonOnlyFlag,
@@ -93,7 +93,7 @@ export function resolveWorkflowRunSshInput({
export default class WorkflowRun extends EasCommand {
static override description =
'run an EAS workflow. The entire local project directory will be packaged and uploaded to EAS servers for the workflow run, unless the --ref flag is used';
'run an EAS workflow. The entire local project directory will be packaged and uploaded to EAS servers for the workflow run, unless the --ref flag is used.';
static override args = {
file: Args.string({
@@ -32,7 +32,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
export default class WorkflowStatus extends EasCommand {
static override description =
'show the status of an existing workflow run. If no run ID is provided, you will be prompted to select from recent workflow runs for the current project';
'show the status of an existing workflow run. If no run ID is provided, you will be prompted to select from recent workflow runs for the current project.';
static override args = {
WORKFLOW_RUN_ID: Args.string({
@@ -59,7 +59,7 @@ const processedOutputs: (job: WorkflowJobResult) => FormatFieldsItem[] = job =>
export default class WorkflowView extends EasCommand {
static override description =
'view details for a workflow run, including jobs. If no run ID is provided, you will be prompted to select from recent workflow runs for the current project';
'view details for a workflow run, including jobs. If no run ID is provided, you will be prompted to select from recent workflow runs for the current project.';
static override flags = {
...EasJsonOnlyFlag,
-15
View File
@@ -42,18 +42,3 @@ Github action builds a docker image for every PR with any change in `infra/eas-b
### VM template
Github action builds a new vm template for every PR with any change in `infra/eas-build-worker-images/ios/{{ templateName }}`.
### Packaged startup
Existing services run `src/services/worker/dist/main.js`. In the worker archive,
that path points to `bin/bootstrap.cjs`. The bootstrap loads only Node built-ins
and starts `bin/start-worker.sh`. The shell then uses `exec` to start the normal
worker with the original Node binary, options, and arguments.
Put setup that must precede worker imports or Node startup options in the shell.
The bootstrap forwards SIGINT, SIGTERM, IPC messages, and the worker exit status.
It remains the supervised parent process; test service shutdown on both platforms
before deployment. A failure in the startup shell prevents the worker from
loading. Direct execution of `packages/worker/dist/main.js` bypasses this hook.
Run `yarn workspace @expo/worker test:startup` for the process-level checks.
-53
View File
@@ -1,53 +0,0 @@
// This entry point must only load Node built-ins before the startup shell runs.
const { spawn } = require('node:child_process');
const path = require('node:path');
const worker = spawn(
'/bin/bash',
[
path.join(__dirname, 'start-worker.sh'),
process.execPath,
...process.execArgv,
path.join(__dirname, '../dist/main.js'),
...process.argv.slice(2),
],
{ stdio: process.send ? ['inherit', 'inherit', 'inherit', 'ipc'] : 'inherit' }
);
const signals = ['SIGINT', 'SIGTERM'];
for (const signal of signals) {
process.on(signal, () => worker.kill(signal));
}
// Retain the readiness channel used by process managers in development.
if (process.send) {
worker.on('message', message => {
if (process.connected) {
process.send(message, () => {});
}
});
process.on('message', message => {
if (worker.connected) {
worker.send(message, () => {});
}
});
process.on('disconnect', () => {
if (worker.connected) {
worker.disconnect();
}
});
}
worker.on('error', error => {
process.stderr.write(`Worker startup failed: ${error.stack ?? error}\n`, () => process.exit(1));
});
worker.on('exit', (code, signal) => {
if (signal) {
for (const forwarded of signals) {
process.removeAllListeners(forwarded);
}
process.kill(process.pid, signal);
} else {
process.exit(code ?? 1);
}
});
-5
View File
@@ -1,5 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Startup setup belongs here, before the normal worker's Node process starts.
exec "$@"
+2 -3
View File
@@ -11,12 +11,11 @@
"compile": "tsc -p tsconfig.build.json",
"prepublishOnly": "npm run build",
"test": "yarn test:unit",
"test:unit": "yarn test:startup && ./resources/jest --config jest.config.unit.ts",
"test:unit": "./resources/jest --config jest.config.unit.ts",
"test:unit:watch": "./resources/jest --config jest.config.unit.ts --watch",
"test:integration": "./resources/jest --runInBand --config jest.config.integration.ts",
"test:integration:watch": "./resources/jest --runInBand --config jest.config.integration.ts --watchAll",
"typecheck": "tsc",
"test:startup": "node --test tests/*.test.cjs"
"typecheck": "tsc"
},
"dependencies": {
"@babel/core": "7.24.7",
+2 -2
View File
@@ -120,10 +120,10 @@ fi
# Create backward-compatible symlink for orchestrator
# The orchestrator expects ./src/services/worker/dist/main.js
# Route it through the packaged startup wrapper before loading worker modules.
# but the new structure has ./packages/worker/dist/main.js
# We're only symlinking the specific file to discover any other legacy path usage
mkdir -p "$target_root_dir/src/services/worker/dist"
ln -s ../../../../packages/worker/bin/bootstrap.cjs "$target_root_dir/src/services/worker/dist/main.js"
ln -s ../../../../packages/worker/dist/main.js "$target_root_dir/src/services/worker/dist/main.js"
tar zcf $OUTPUT_FILE -C $target_root_dir .
@@ -39,6 +39,7 @@ describe(createBuildContext.name, () => {
buildId: 'build-id',
buildLogger,
addBuildLogMask: jest.fn(),
createLogMaskingStream: jest.fn(),
};
beforeEach(() => {
+98 -1
View File
@@ -1,4 +1,5 @@
import { TransformCallback, Writable } from 'node:stream';
import { Readable, Transform, TransformCallback, Writable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { EnvironmentSecretType } from '@expo/eas-build-job';
import z from 'zod';
@@ -356,3 +357,99 @@ describe('logger', () => {
expect(outputStream.readableLength).toBe(0);
});
});
describe('createLogMaskingStream', () => {
async function maskAsync(stream: Transform, chunks: Buffer[]): Promise<Buffer> {
const output: Buffer[] = [];
await pipeline(
Readable.from(chunks),
stream,
new Writable({
write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
output.push(chunk);
callback();
},
})
);
return Buffer.concat(output);
}
it('masks secrets, secrets as Xcode prints them in build settings, and decoded secrets', async () => {
const { createLogMaskingStream } = await createBuildLoggerWithSecretsFilter({
environmentSecrets: [
{ name: 'API_KEY', value: 'ab cd=ef$gh', type: EnvironmentSecretType.STRING },
{
name: 'ANOTHER_SECRET_BASE64',
value: 'YW5vdGhlclNlY3JldA==',
type: EnvironmentSecretType.STRING,
},
],
});
const log = [
' export API_KEY\\=ab\\ cd\\=ef\\$gh',
'token is ab cd=ef$gh',
'decoded anotherSecret here',
'',
].join('\n');
const masked = await maskAsync(createLogMaskingStream(), [Buffer.from(log)]);
expect(masked.toString()).toBe(
[
` export API_KEY\\=${'*'.repeat(14)}`,
`token is ${'*'.repeat(11)}`,
`decoded ${'*'.repeat(13)} here`,
'',
].join('\n')
);
});
it('masks secrets split across chunks, including masks added after the logger was created', async () => {
const { createLogMaskingStream, addBuildLogMask } = await createBuildLoggerWithSecretsFilter(
{}
);
addBuildLogMask('leased-token');
const masked = await maskAsync(createLogMaskingStream(), [
Buffer.from('first line leased-'),
Buffer.from('token\nsecond '),
Buffer.from('leased-token without a newline'),
]);
expect(masked.toString()).toBe(
`first line ${'*'.repeat(12)}\nsecond ${'*'.repeat(12)} without a newline`
);
});
it('keeps everything except secrets byte for byte, including multibyte and invalid UTF-8', async () => {
const { createLogMaskingStream } = await createBuildLoggerWithSecretsFilter({
environmentSecrets: [
{ name: 'PASSWORD', value: 'pässwörd', type: EnvironmentSecretType.STRING },
],
});
const log = Buffer.concat([
Buffer.from('✓ Compiling café.swift\r\n'),
Buffer.from([0xff, 0xfe, 0x0a]),
Buffer.from('password: pässwörd\n'),
]);
// Split inside the multibyte characters.
const chunks = [
log.subarray(0, 1),
log.subarray(1, 20),
log.subarray(20, 40),
log.subarray(40),
];
const masked = await maskAsync(createLogMaskingStream(), chunks);
const passwordBytes = Buffer.byteLength('pässwörd');
expect(masked).toEqual(
Buffer.concat([
Buffer.from('✓ Compiling café.swift\r\n'),
Buffer.from([0xff, 0xfe, 0x0a]),
Buffer.from(`password: ${'*'.repeat(passwordBytes)}\n`),
])
);
expect(masked.length).toBe(log.length);
});
});
@@ -115,6 +115,7 @@ describe('BuildService Datadog setup', () => {
createBuildLoggerWithSecretsFilterMock.mockResolvedValue({
logger: buildLogger,
addBuildLogMask: jest.fn(),
createLogMaskingStream: jest.fn(),
cleanUp: jest.fn(async () => {}),
logBuffer: { getLogs: () => [], getPhaseLogs: () => [] } as any,
outputStream: {} as any,
@@ -0,0 +1,82 @@
import { RemoteLoggerStream, uploadWithSignedUrl } from '@expo/build-tools';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
import { Readable } from 'stream';
import zlib from 'zlib';
import config from '../config';
import { uploadXcodeBuildLogs } from '../ios/xcodeLogs';
import { createTextMaskingStream } from '../logger';
// These tests use real fs and directories in os.tmpdir(), since memfs read streams can't be closed.
jest.unmock('fs');
jest.unmock('node:fs');
const logger = { warn: jest.fn() } as any;
async function readAllAsync(stream: Readable): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.from(chunk));
}
return Buffer.concat(chunks);
}
describe(uploadXcodeBuildLogs, () => {
const originalGcs = { ...config.loggers.gcs };
let tmpDir: string;
beforeEach(async () => {
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'xcode-logs-test-'));
config.loggers.gcs.signedUploadUrlForXcodeBuildLogs = {
url: 'https://gcs.expo.test/xcode-logs',
headers: {},
};
});
afterEach(async () => {
Object.assign(config.loggers.gcs, originalGcs);
jest.mocked(uploadWithSignedUrl).mockReset();
await fs.remove(tmpDir);
});
it.each([
['no compression', null, (body: Buffer) => body],
['gzip', RemoteLoggerStream.CompressionMethod.GZIP, zlib.gunzipSync],
['brotli', RemoteLoggerStream.CompressionMethod.BR, zlib.brotliDecompressSync],
])('uploads the log with secrets masked (%s)', async (_name, compressionMethod, decompress) => {
config.loggers.gcs.compressionMethod = compressionMethod as any;
const logsPath = path.join(tmpDir, 'xcodebuild.log');
await fs.writeFile(logsPath, ' export API_KEY\\=ab\\ cd\\=ef\\$gh\n** BUILD SUCCEEDED **\n');
let uploaded: Buffer | undefined;
jest.mocked(uploadWithSignedUrl).mockImplementation(async ({ srcGeneratorAsync }) => {
uploaded = await readAllAsync((await srcGeneratorAsync()) as Readable);
return 'https://gcs.expo.test/xcode-logs';
});
await uploadXcodeBuildLogs(logger, logsPath, () =>
createTextMaskingStream(new Set(['ab cd=ef$gh']))
);
expect(decompress(uploaded!).toString()).toBe(
` export API_KEY\\=${'*'.repeat(14)}\n** BUILD SUCCEEDED **\n`
);
});
it.each([
['no compression', null],
['gzip', RemoteLoggerStream.CompressionMethod.GZIP],
])('fails the upload when the log cannot be read (%s)', async (_name, compressionMethod) => {
config.loggers.gcs.compressionMethod = compressionMethod as any;
jest.mocked(uploadWithSignedUrl).mockImplementation(async ({ srcGeneratorAsync }) => {
await readAllAsync((await srcGeneratorAsync()) as Readable);
return 'https://gcs.expo.test/xcode-logs';
});
// A directory can be stat'ed but not read.
await expect(
uploadXcodeBuildLogs(logger, tmpDir, () => createTextMaskingStream(new Set(['secret'])))
).rejects.toThrow('EISDIR');
});
});
+4 -1
View File
@@ -2,6 +2,7 @@ import { BuildContext, BuildContextOptions, LogBuffer, RuntimeSettings } from '@
import { Job, ManagedArtifactType, Metadata, Platform } from '@expo/eas-build-job';
import { bunyan } from '@expo/logger';
import assert from 'assert';
import { Transform } from 'stream';
import { GCSCacheManager } from './CacheManager';
import config from './config';
@@ -27,6 +28,7 @@ export async function createBuildContext<TJob extends Job>({
buildId,
buildLogger,
addBuildLogMask,
createLogMaskingStream,
}: {
job: TJob;
logBuffer: LogBuffer;
@@ -36,6 +38,7 @@ export async function createBuildContext<TJob extends Job>({
buildId: string;
buildLogger: bunyan;
addBuildLogMask: (secret: string) => void;
createLogMaskingStream: () => Transform;
}): Promise<BuildContext<TJob>> {
const childLogger = buildLogger.child({ buildId });
await RuntimeSettings.loadAsync({
@@ -51,7 +54,7 @@ export async function createBuildContext<TJob extends Job>({
switch (type) {
case ManagedArtifactType.XCODE_BUILD_LOGS: {
if (job.platform === Platform.IOS) {
await uploadXcodeBuildLogs(logger, paths[0]);
await uploadXcodeBuildLogs(logger, paths[0], createLogMaskingStream);
return { filename: null };
}
throw new Error('Uploading Xcode logs in non-iOS builds is not supported');
+14 -3
View File
@@ -3,14 +3,18 @@ import { bunyan } from '@expo/logger';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
import { Readable, pipeline } from 'stream';
import { Readable, Transform, pipeline } from 'stream';
import { promisify } from 'util';
import { v4 as uuid } from 'uuid';
import zlib from 'zlib';
import config from '../config';
export async function uploadXcodeBuildLogs(logger: bunyan, logsPath: string): Promise<void> {
export async function uploadXcodeBuildLogs(
logger: bunyan,
logsPath: string,
createLogMaskingStream: () => Transform
): Promise<void> {
if (!config.loggers.gcs.signedUploadUrlForXcodeBuildLogs) {
logger.warn('GCS Presigned URL for Xcode logs was not specified, skipping upload');
} else {
@@ -18,8 +22,15 @@ export async function uploadXcodeBuildLogs(logger: bunyan, logsPath: string): Pr
signedUrl: config.loggers.gcs.signedUploadUrlForXcodeBuildLogs,
srcGeneratorAsync: async () => {
const stat = await fs.stat(logsPath);
// Masking replaces each byte of a secret with one byte, so the size stays the same.
// pipeline passes read errors on to the masked stream, which fails the upload.
const maskedStream = pipeline(
fs.createReadStream(logsPath),
createLogMaskingStream(),
() => {}
);
const { stream } = await compress(
{ stream: fs.createReadStream(logsPath), length: stat.size },
{ stream: maskedStream, length: stat.size },
config.loggers.gcs.compressionMethod
);
return stream;
+37
View File
@@ -102,6 +102,40 @@ function createTransformStream(secretPatterns: ReadonlySet<string>): Transform {
});
}
// Characters that Xcode escapes with a backslash when it prints build settings.
const XCODE_ESCAPED_CHARACTERS = /[\t !"#$&'()*;<=>?[\\\]^`{|}~]/g;
/**
* Masks secrets in a text file's contents, line by line, including secrets written the way Xcode
* prints build settings. Masking works on bytes, so everything else stays unchanged.
*/
export function createTextMaskingStream(secretPatterns: ReadonlySet<string>): Transform {
const patterns = [
...new Set(
[...secretPatterns].flatMap(secret => [
secret,
secret.replace(XCODE_ESCAPED_CHARACTERS, '\\$&'),
])
),
].map(pattern => Buffer.from(pattern).toString('latin1'));
const maskLine = (line: string): string =>
patterns.some(pattern => line.includes(pattern)) ? maskSecrets(line, patterns) : line;
let partialLine = '';
return new Transform({
transform(chunk: Buffer, _encoding: BufferEncoding, callback: TransformCallback) {
const lines = (partialLine + chunk.toString('latin1')).split('\n');
partialLine = lines.pop() ?? '';
callback(
null,
lines.length > 0 ? Buffer.from(`${lines.map(maskLine).join('\n')}\n`, 'latin1') : undefined
);
},
flush(callback: TransformCallback) {
callback(null, partialLine ? Buffer.from(maskLine(partialLine), 'latin1') : undefined);
},
});
}
function maskSecrets(message: string, secrets: Iterable<string>): string {
const characters = message.split('');
for (const secret of secrets) {
@@ -170,6 +204,7 @@ function createDiscardStream(): Writable {
export async function createBuildLoggerWithSecretsFilter(secrets: Job['secrets']): Promise<{
logger: bunyan;
addBuildLogMask: (secret: string) => void;
createLogMaskingStream: () => Transform;
cleanUp: () => Promise<void>;
logBuffer: WorkerLogBuffer;
outputStream: Readable;
@@ -239,6 +274,8 @@ export async function createBuildLoggerWithSecretsFilter(secrets: Job['secrets']
return {
logger: buildLogger,
addBuildLogMask,
// Reads the masks when the stream is created, so it includes masks added during the job.
createLogMaskingStream: () => createTextMaskingStream(secretPatterns),
cleanUp: async () => {
await Promise.all([remoteLoggerStream?.cleanUp?.(), httpLogStream?.cleanUp?.()]);
},
+2
View File
@@ -275,6 +275,7 @@ export default class BuildService {
const {
logger: buildLogger,
addBuildLogMask,
createLogMaskingStream,
cleanUp,
logBuffer,
} = await createBuildLoggerWithSecretsFilter(job.secrets ?? {});
@@ -315,6 +316,7 @@ export default class BuildService {
buildId: this.buildId,
buildLogger,
addBuildLogMask,
createLogMaskingStream,
});
this.buildContext = ctx;
-100
View File
@@ -1,100 +0,0 @@
const assert = require('node:assert/strict');
const { spawn } = require('node:child_process');
const { once } = require('node:events');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
function archive(t, main, shell) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'worker-startup-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const worker = path.join(root, 'packages/worker');
fs.mkdirSync(path.join(worker, 'dist'), { recursive: true });
fs.cpSync(path.join(__dirname, '../bin'), path.join(worker, 'bin'), { recursive: true });
fs.writeFileSync(path.join(worker, 'dist/main.js'), main);
if (shell) {
fs.writeFileSync(path.join(worker, 'bin/start-worker.sh'), shell);
}
// Exercise the path that existing images run, including Node's symlink resolution.
const legacy = path.join(root, 'src/services/worker/dist/main.js');
fs.mkdirSync(path.dirname(legacy), { recursive: true });
fs.symlinkSync('../../../../packages/worker/bin/bootstrap.cjs', legacy);
return legacy;
}
function launch(t, entry, args = []) {
const child = spawn(process.execPath, [entry, ...args], {
stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
});
t.after(() => {
if (child.exitCode === null && child.signalCode === null) {
child.kill('SIGKILL');
}
});
return child;
}
test(
'startup settings reach the worker before its first module runs',
{ timeout: 10000 },
async t => {
const entry = archive(
t,
'process.send({ ready: true, args: process.argv.slice(2), setting: process.env.STARTUP_TEST }); process.on("message", () => process.exit(7));',
'#!/bin/bash\nset -euo pipefail\nexport STARTUP_TEST=installed\nexec "$@"\n'
);
const child = launch(t, entry, ['argument with spaces']);
const exited = once(child, 'exit');
assert.deepEqual((await once(child, 'message'))[0], {
ready: true,
args: ['argument with spaces'],
setting: 'installed',
});
child.send('stop');
assert.deepEqual(await exited, [7, null]);
}
);
for (const signal of ['SIGINT', 'SIGTERM']) {
test(`forwards ${signal} and preserves termination status`, { timeout: 10000 }, async t => {
const entry = archive(t, 'process.send("ready"); setInterval(() => {}, 1000);');
const child = launch(t, entry);
const exited = once(child, 'exit');
await once(child, 'message');
child.kill(signal);
assert.deepEqual(await exited, [null, signal]);
});
}
test('startup failure prevents loading the worker', { timeout: 10000 }, async t => {
const entry = archive(
t,
'process.send("must not start");',
'#!/bin/bash\nset -euo pipefail\nexit 42\nexec "$@"\n'
);
const child = launch(t, entry);
const messages = [];
child.on('message', message => messages.push(message));
assert.deepEqual(await once(child, 'exit'), [42, null]);
assert.deepEqual(messages, []);
});
test('runs without IPC and retains stdout, stderr, and exit code', { timeout: 10000 }, async t => {
const entry = archive(
t,
'console.log("worker output"); console.error("worker error"); process.exit(9);'
);
const child = spawn(process.execPath, [entry], { stdio: ['ignore', 'pipe', 'pipe'] });
let stdout = '';
let stderr = '';
child.stdout.on('data', data => {
stdout += data;
});
child.stderr.on('data', data => {
stderr += data;
});
assert.deepEqual(await once(child, 'close'), [9, null]);
assert.equal(stdout, 'worker output\n');
assert.equal(stderr, 'worker error\n');
});