mirror of
https://github.com/expo/eas-cli.git
synced 2026-11-14 17:50:32 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
72b8a78145 |
No files matched your search
@@ -1,5 +0,0 @@
|
||||
---
|
||||
'eas-cli': none
|
||||
---
|
||||
|
||||
Make command descriptions in `eas --help` consistently lowercase without trailing periods.
|
||||
@@ -1,6 +0,0 @@
|
||||
---
|
||||
'eas-cli': none
|
||||
'@expo/build-tools': none
|
||||
---
|
||||
|
||||
Bump Node.js to 22.23.4 to fix the CVEs in the [October 2026 Node.js security releases](https://nodejs.org/en/blog/vulnerability/october-2026-security-releases).
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
'@expo/build-tools': minor
|
||||
---
|
||||
|
||||
Stop sandboxes after an optional maximum duration measured from readiness.
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
'@expo/worker': patch
|
||||
---
|
||||
|
||||
Apply the job log's secret masking to uploaded Xcode build logs.
|
||||
@@ -6,4 +6,4 @@ MISE_NODE_COREPACK = "true"
|
||||
actionlint = "1.7.12"
|
||||
gcloud = "570.0.0"
|
||||
jq = "1.8.1"
|
||||
node = "22.23.4"
|
||||
node = "22.23.3"
|
||||
@@ -26,7 +26,7 @@
|
||||
"build:egress-guard": "resources/egress-guard/build.sh",
|
||||
"test:egress-guard": "resources/egress-guard/tests/run-policy-tests.sh && resources/egress-guard/tests/run-guard-tests.sh",
|
||||
"typecheck": "tsc",
|
||||
"generate-appium-commands": "mise exec [email protected]3.4 -- node scripts/generate-appium-commands.js",
|
||||
"generate-appium-commands": "mise exec [email protected]2.0 -- node scripts/generate-appium-commands.js",
|
||||
"prepack": "rimraf dist \"*.tsbuildinfo\" && yarn gql && tsc -p tsconfig.build.json",
|
||||
"jest-unit": "jest --config jest/unit-config.ts",
|
||||
"jest-integration": "jest --config jest/integration-config.ts",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { SystemError, UserError } from '@expo/eas-build-job';
|
||||
import { SystemError } from '@expo/eas-build-job';
|
||||
import { Client, fetchExchange } from '@urql/core';
|
||||
|
||||
import * as sandboxDaemon from '../../utils/sandboxDaemon';
|
||||
@@ -6,90 +6,6 @@ import * as sandboxDaemon from '../../utils/sandboxDaemon';
|
||||
import { createStartSandboxBuildFunction, markSandboxReadyAsync } from '../startSandbox';
|
||||
|
||||
describe('sandbox build functions', () => {
|
||||
it.each([
|
||||
{ completion: 'duration', duration: 0.02 },
|
||||
{ completion: 'cancellation', duration: 60 },
|
||||
{ completion: 'duration', duration: 0.0001 },
|
||||
])(
|
||||
'stops after $completion with $duration seconds only after readiness',
|
||||
async ({ completion, duration }) => {
|
||||
const controller = new AbortController();
|
||||
let ready!: () => void;
|
||||
const stopAsync = jest.fn(async () => {});
|
||||
const start = jest.spyOn(sandboxDaemon, 'startSandboxDaemonAsync').mockResolvedValue({
|
||||
ready: new Promise<void>(resolve => {
|
||||
ready = resolve;
|
||||
}),
|
||||
stopAsync,
|
||||
});
|
||||
const mutation = jest.fn(() => ({ toPromise: async () => ({ data: {} }) }));
|
||||
const logger = { info: jest.fn(), child: jest.fn() };
|
||||
logger.child.mockReturnValue(logger);
|
||||
const fn = createStartSandboxBuildFunction({
|
||||
mcpServerUrl: 'ws://localhost:8787',
|
||||
graphqlClient: { mutation },
|
||||
} as any);
|
||||
try {
|
||||
const pending = fn.fn!({ logger } as any, {
|
||||
inputs: {
|
||||
sandbox_id: { value: 'sandbox-id' },
|
||||
max_duration_seconds: { value: duration },
|
||||
},
|
||||
outputs: {},
|
||||
env: { __EAS_SANDBOX_MCP_TOKEN: 'token' },
|
||||
signal: controller.signal,
|
||||
});
|
||||
await new Promise(resolve => setTimeout(resolve, 40));
|
||||
expect(mutation).not.toHaveBeenCalled();
|
||||
expect(stopAsync).not.toHaveBeenCalled();
|
||||
ready();
|
||||
if (completion === 'cancellation') {
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
expect(stopAsync).not.toHaveBeenCalled();
|
||||
controller.abort();
|
||||
}
|
||||
await pending;
|
||||
expect(mutation).toHaveBeenCalledTimes(1);
|
||||
expect(stopAsync).toHaveBeenCalledTimes(1);
|
||||
const timeoutMessage = `Sandbox reached its maximum runtime of ${duration} seconds. Stopping.`;
|
||||
if (completion === 'duration') {
|
||||
expect(logger.info).toHaveBeenCalledWith(timeoutMessage);
|
||||
} else {
|
||||
expect(logger.info).not.toHaveBeenCalledWith(timeoutMessage);
|
||||
}
|
||||
} finally {
|
||||
start.mockRestore();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
it.each([0, -1, Infinity, NaN])(
|
||||
'rejects invalid maximum duration %s before starting the daemon',
|
||||
async duration => {
|
||||
const start = jest.spyOn(sandboxDaemon, 'startSandboxDaemonAsync');
|
||||
const fn = createStartSandboxBuildFunction({
|
||||
mcpServerUrl: 'ws://localhost:8787',
|
||||
} as any);
|
||||
try {
|
||||
const result = fn.fn!({} as any, {
|
||||
inputs: {
|
||||
sandbox_id: { value: 'sandbox-id' },
|
||||
max_duration_seconds: { value: duration },
|
||||
},
|
||||
outputs: {},
|
||||
env: { __EAS_SANDBOX_MCP_TOKEN: 'token' },
|
||||
});
|
||||
await expect(result).rejects.toBeInstanceOf(UserError);
|
||||
await expect(result).rejects.toThrow(
|
||||
`Invalid sandbox maximum duration: ${duration}. Provide a finite number greater than zero, or omit max_duration_seconds.`
|
||||
);
|
||||
expect(start).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
start.mockRestore();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
it('aborts an active ready request and keeps its authentication headers', async () => {
|
||||
const controller = new AbortController();
|
||||
let requestStarted!: () => void;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { SystemError, UserError } from '@expo/eas-build-job';
|
||||
import { SystemError } from '@expo/eas-build-job';
|
||||
import { BuildFunction, BuildStepInput, BuildStepInputValueTypeName } from '@expo/steps';
|
||||
import { graphql } from 'gql.tada';
|
||||
|
||||
@@ -32,11 +32,6 @@ export function createStartSandboxBuildFunction(ctx: CustomBuildContext): BuildF
|
||||
required: true,
|
||||
allowedValueTypeName: BuildStepInputValueTypeName.STRING,
|
||||
}),
|
||||
BuildStepInput.createProvider({
|
||||
id: 'max_duration_seconds',
|
||||
required: false,
|
||||
allowedValueTypeName: BuildStepInputValueTypeName.NUMBER,
|
||||
}),
|
||||
],
|
||||
fn: async (stepCtx, { inputs, signal, env }) => {
|
||||
// The daemon needs this credential, but shell commands do not.
|
||||
@@ -49,18 +44,6 @@ export function createStartSandboxBuildFunction(ctx: CustomBuildContext): BuildF
|
||||
throw new SystemError('MCP server URL is required to start the sandbox daemon.');
|
||||
}
|
||||
const sandboxId = String(inputs.sandbox_id.value);
|
||||
const maxDurationSeconds = inputs.max_duration_seconds?.value;
|
||||
if (
|
||||
maxDurationSeconds !== undefined &&
|
||||
(typeof maxDurationSeconds !== 'number' ||
|
||||
!Number.isFinite(maxDurationSeconds) ||
|
||||
maxDurationSeconds <= 0)
|
||||
) {
|
||||
throw new UserError(
|
||||
'EAS_SANDBOX_INVALID_DURATION',
|
||||
`Invalid sandbox maximum duration: ${String(maxDurationSeconds)}. Provide a finite number greater than zero, or omit max_duration_seconds.`
|
||||
);
|
||||
}
|
||||
const daemon = await startSandboxDaemonAsync({
|
||||
credential: sandboxToken,
|
||||
serverUrl: mcpServerUrl,
|
||||
@@ -80,21 +63,7 @@ export function createStartSandboxBuildFunction(ctx: CustomBuildContext): BuildF
|
||||
stepCtx.logger.info('Sandbox daemon started.');
|
||||
|
||||
await withLogPhaseAsync(stepCtx.logger, AWAIT_SANDBOX_STEP_NAME, async () => {
|
||||
const timeoutSignal =
|
||||
maxDurationSeconds === undefined
|
||||
? undefined
|
||||
: AbortSignal.timeout(Math.ceil(maxDurationSeconds * 1000));
|
||||
await waitForCompletionAsync(
|
||||
AbortSignal.any([
|
||||
...(signal ? [signal] : []),
|
||||
...(timeoutSignal ? [timeoutSignal] : []),
|
||||
])
|
||||
);
|
||||
if (timeoutSignal?.aborted) {
|
||||
stepCtx.logger.info(
|
||||
`Sandbox reached its maximum runtime of ${maxDurationSeconds} seconds. Stopping.`
|
||||
);
|
||||
}
|
||||
await waitForCancellationAsync(signal);
|
||||
});
|
||||
} finally {
|
||||
await daemon.stopAsync();
|
||||
@@ -120,11 +89,11 @@ export async function markSandboxReadyAsync(
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForCompletionAsync(signal: AbortSignal): Promise<void> {
|
||||
if (signal.aborted) {
|
||||
async function waitForCancellationAsync(signal?: AbortSignal): Promise<void> {
|
||||
if (signal?.aborted) {
|
||||
return;
|
||||
}
|
||||
await new Promise<void>(resolve => {
|
||||
signal.addEventListener('abort', () => resolve(), { once: true });
|
||||
signal?.addEventListener('abort', () => resolve(), { once: true });
|
||||
});
|
||||
}
|
||||
@@ -252,7 +252,7 @@
|
||||
},
|
||||
"update": {
|
||||
"node": {
|
||||
"version": "22.23.4"
|
||||
"version": "22.23.3"
|
||||
},
|
||||
"s3": {
|
||||
"templates": {
|
||||
|
||||
@@ -3,56 +3,8 @@ import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import fs from 'fs-extra';
|
||||
import nock from 'nock';
|
||||
|
||||
import { computeFileMd5Base64Async, uploadWithPresignedPostWithRetryAsync } from '../uploads';
|
||||
|
||||
describe(uploadWithPresignedPostWithRetryAsync, () => {
|
||||
const presignedPost = {
|
||||
url: 'https://storage.example.com/upload',
|
||||
fields: { key: 'object-key' },
|
||||
};
|
||||
const contents = Array.from({ length: 50000 }, (_, i) => `line ${i}\n`).join('');
|
||||
let tmpDir: string;
|
||||
let file: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'eas-cli-presigned-post-test-'));
|
||||
file = path.join(tmpDir, 'bundle.js');
|
||||
await fs.writeFile(file, contents);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await fs.remove(tmpDir);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
nock.cleanAll();
|
||||
});
|
||||
|
||||
it('sends the whole file and reports activity while it uploads', async () => {
|
||||
let body = '';
|
||||
nock('https://storage.example.com')
|
||||
.post('/upload', requestBody => {
|
||||
body = requestBody;
|
||||
return true;
|
||||
})
|
||||
.reply(204);
|
||||
const onAssetUploadBegin = jest.fn();
|
||||
|
||||
await uploadWithPresignedPostWithRetryAsync(file, presignedPost, onAssetUploadBegin);
|
||||
|
||||
expect(body).toContain('filename="bundle.js"');
|
||||
expect(body).toContain(contents);
|
||||
expect(onAssetUploadBegin.mock.calls.length).toBeGreaterThan(2);
|
||||
expect(onAssetUploadBegin).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ file, fileSize: contents.length, uploadedBytes: 0 })
|
||||
);
|
||||
expect(onAssetUploadBegin).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({ file, fileSize: contents.length, uploadedBytes: contents.length })
|
||||
);
|
||||
});
|
||||
});
|
||||
import { computeFileMd5Base64Async } from '../uploads';
|
||||
|
||||
describe(computeFileMd5Base64Async, () => {
|
||||
let tmpDir: string;
|
||||
|
||||
@@ -33,7 +33,7 @@ const PROJECT_PAGES: Record<string, string> = {
|
||||
|
||||
export default class Browse extends EasCommand {
|
||||
static override description =
|
||||
'transition from the terminal to the web browser to view and interact with your project on https://expo.dev';
|
||||
'Transition from the terminal to the web browser to view and interact with your project on https://expo.dev';
|
||||
|
||||
static override args = {
|
||||
page: Args.string({
|
||||
|
||||
@@ -13,7 +13,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
|
||||
import { pollForBackgroundJobReceiptAsync } from '../../utils/pollForBackgroundJobReceiptAsync';
|
||||
|
||||
export default class ChannelDelete extends EasCommand {
|
||||
static override description = 'delete a channel';
|
||||
static override description = 'Delete a channel';
|
||||
|
||||
static override args = {
|
||||
name: Args.string({
|
||||
|
||||
@@ -50,7 +50,7 @@ type ChannelRolloutArgsAndFlags = {
|
||||
Partial<CreateRolloutNonInteractiveOptions>;
|
||||
|
||||
export default class ChannelRollout extends EasCommand {
|
||||
static override description = 'roll a new branch out on a channel incrementally';
|
||||
static override description = 'Roll a new branch out on a channel incrementally.';
|
||||
|
||||
static override args = {
|
||||
channel: Args.string({
|
||||
|
||||
@@ -7,7 +7,7 @@ import { SetUpBuildCredentialsCommandAction } from '../../credentials/manager/Se
|
||||
import { selectPlatformAsync } from '../../platform';
|
||||
|
||||
export default class InitializeBuildCredentials extends EasCommand {
|
||||
static override description = 'set up credentials for building your project';
|
||||
static override description = 'Set up credentials for building your project.';
|
||||
|
||||
static override flags = {
|
||||
platform: Flags.option({
|
||||
|
||||
@@ -22,7 +22,7 @@ interface DeployAliasDeleteFlags {
|
||||
}
|
||||
|
||||
export default class WorkerAliasDelete extends EasCommand {
|
||||
static override description = 'delete deployment aliases';
|
||||
static override description = 'Delete deployment aliases.';
|
||||
static override aliases = ['worker:alias:delete'];
|
||||
static override state = 'preview';
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ interface RawDeployAliasFlags {
|
||||
}
|
||||
|
||||
export default class WorkerAlias extends EasCommand {
|
||||
static override description = 'assign deployment aliases';
|
||||
static override description = 'Assign deployment aliases.';
|
||||
static override aliases = ['worker:alias', 'deploy:promote'];
|
||||
static override state = 'preview';
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ interface RawDeployDeleteFlags {
|
||||
}
|
||||
|
||||
export default class WorkerDelete extends EasCommand {
|
||||
static override description = 'delete a deployment';
|
||||
static override description = 'Delete a deployment.';
|
||||
static override aliases = ['worker:delete'];
|
||||
static override state = 'preview';
|
||||
|
||||
|
||||
@@ -100,7 +100,7 @@ async function withSuppressedOutputAsync<T>(fn: () => Promise<T>): Promise<T> {
|
||||
/* eslint-enable no-console */
|
||||
|
||||
export default class Go extends EasCommand {
|
||||
static override description = 'create a custom Expo Go and submit to TestFlight';
|
||||
static override description = 'Create a custom Expo Go and submit to TestFlight';
|
||||
static override hidden = true;
|
||||
|
||||
static override flags = {
|
||||
|
||||
@@ -22,7 +22,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
|
||||
|
||||
export default class ObserveEvent extends EasCommand {
|
||||
static override description =
|
||||
'display a single Observe event (metric, log, or error) by its ID. IDs are included in event data when the `--json` flag is passed to `eas observe:session`, `eas observe:metrics`, or `eas observe:events`';
|
||||
'display a single Observe event (metric, log, or error) by its ID. IDs are included in event data when the `--json` flag is passed to `eas observe:session`, `eas observe:metrics`, or `eas observe:events`.';
|
||||
|
||||
static override args = {
|
||||
id: Args.string({
|
||||
|
||||
@@ -36,7 +36,7 @@ const DEFAULT_EVENTS_LIMIT = 10;
|
||||
|
||||
export default class ObserveEvents extends EasCommand {
|
||||
static override description =
|
||||
'display individual events emitted by the app via `logEvent`, filtered by the event name in the argument. With no arguments, a list of the available event names and associated event counts is returned';
|
||||
'display individual events emitted by the app via `logEvent`, filtered by the event name in the argument. With no arguments, a list of the available event names and associated event counts is returned.';
|
||||
|
||||
static override args = {
|
||||
eventName: Args.string({
|
||||
|
||||
@@ -134,7 +134,7 @@ export default class New extends EasCommand {
|
||||
static override aliases = ['new'];
|
||||
|
||||
static override description =
|
||||
'create a new project configured with Expo Application Services (EAS)';
|
||||
'Create a new project configured with Expo Application Services (EAS)';
|
||||
|
||||
static override args = {
|
||||
path: Args.string({
|
||||
|
||||
@@ -35,7 +35,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
|
||||
export default class SimulatorControl extends EasCommand {
|
||||
static override aliases = ['sim:control'];
|
||||
static override description =
|
||||
'[EXPERIMENTAL] run a serve-sim action on a remote iOS simulator session on EAS. Run without an action to list the actions the session supports. To save a screenshot locally, run `eas simulator:control screenshot.capture --output ./screenshot.png`';
|
||||
'[EXPERIMENTAL] run a serve-sim action on a remote iOS simulator session on EAS. Run without an action to list the actions the session supports. To save a screenshot locally, run `eas simulator:control screenshot.capture --output ./screenshot.png`.';
|
||||
static override strict = false;
|
||||
|
||||
static override examples = [
|
||||
|
||||
@@ -39,7 +39,7 @@ type DeviceRunSessionArtifact = DeviceRunSessionById['artifacts'][number];
|
||||
export default class SimulatorGet extends EasCommand {
|
||||
static override aliases = ['sim:get'];
|
||||
static override description =
|
||||
"[EXPERIMENTAL] get info about a remote simulator session on EAS by its simulator session ID. Preview links follow the session's saved sharing preference";
|
||||
"[EXPERIMENTAL] get info about a remote simulator session on EAS by its simulator session ID. Preview links follow the session's saved sharing preference.";
|
||||
|
||||
static override flags = {
|
||||
id: Flags.string({
|
||||
|
||||
@@ -83,7 +83,7 @@ export default class Simulator extends EasCommand {
|
||||
static override aliases = ['simulator:start', 'sim:start'];
|
||||
static override hiddenAliases = ['sim'];
|
||||
static override description =
|
||||
'[EXPERIMENTAL] start a remote simulator session on EAS and get instructions to connect to it. The default browser preview link requires sign-in';
|
||||
'[EXPERIMENTAL] start a remote simulator session on EAS and get instructions to connect to it. The default browser preview link requires sign-in.';
|
||||
|
||||
static override flags = {
|
||||
platform: Flags.option({
|
||||
|
||||
@@ -61,7 +61,6 @@ jest.mock('../../../graphql/queries/UpdateQuery');
|
||||
jest.mock('../../../prompts');
|
||||
jest.mock('../../../graphql/queries/EnvironmentVariablesQuery');
|
||||
jest.mock('../../../ora', () => ({
|
||||
isSpinnerEnabled: () => false,
|
||||
ora: () => ({
|
||||
start: () => ({ succeed: () => {}, fail: () => {}, stop: () => {} }),
|
||||
}),
|
||||
|
||||
@@ -3,7 +3,6 @@ import { EasJson, EasJsonAccessor, EasJsonUtils } from '@expo/eas-json';
|
||||
import { Errors, Flags } from '@oclif/core';
|
||||
import chalk from 'chalk';
|
||||
import nullthrows from 'nullthrows';
|
||||
import path from 'path';
|
||||
|
||||
import { ensureBranchExistsAsync } from '../../branch/queries';
|
||||
import { ensureRepoIsCleanAsync } from '../../build/utils/repository';
|
||||
@@ -29,7 +28,7 @@ import {
|
||||
} from '../../graphql/generated';
|
||||
import { PublishMutation } from '../../graphql/mutations/PublishMutation';
|
||||
import Log, { learnMore, link } from '../../log';
|
||||
import { isSpinnerEnabled, ora } from '../../ora';
|
||||
import { ora } from '../../ora';
|
||||
import { RequestedPlatform } from '../../platform';
|
||||
import { maybeUploadAssetMapAsync } from '../../project/maybeUploadAssetMapAsync';
|
||||
import {
|
||||
@@ -57,7 +56,6 @@ import {
|
||||
resolveInputDirectoryAsync,
|
||||
uploadAssetsAsync,
|
||||
} from '../../project/publish';
|
||||
import { createUploadProgressReporter } from '../../project/uploadProgressReporter';
|
||||
import { resolveWorkflowPerPlatformAsync } from '../../project/workflow';
|
||||
import { resolveUpdateGroupsSupersedingActiveRolloutsAsync } from '../../update/active-rollout';
|
||||
import { ensureEASUpdateIsConfiguredAsync } from '../../update/configure';
|
||||
@@ -391,41 +389,13 @@ export default class UpdatePublish extends EasCommand {
|
||||
// - Start with NO_ACTIVITY_TIMEOUT. 180 seconds is chosen because the cloud function that processes
|
||||
// uploaded assets has a timeout of 60 seconds and uploading can take some time on a slow connection.
|
||||
// - Each time one or more assets reports as ready, reset the timeout.
|
||||
// - Each time an asset upload begins or sends more bytes, reset the timeout. This includes retries.
|
||||
// - Each time an asset upload begins, reset the timeout. This includes retries.
|
||||
// - Start upload. Internally, uploadAssetsAsync uploads them all first and then checks for successful
|
||||
// processing every (5 + n) seconds with a linear backoff of n + 1 second.
|
||||
// - At the same time as upload is started, start timeout checker which checks every 1 second to see
|
||||
// if timeout has been reached. When timeout expires, send a cancellation signal to currently running
|
||||
// upload function call to instruct it to stop uploading or checking for successful processing.
|
||||
const NO_ACTIVITY_TIMEOUT = 180 * 1000; // 180 seconds
|
||||
let uploadCountText = 'Uploading...';
|
||||
let activeUploadLines: string[] = [];
|
||||
let unfinishedAssetPaths: string[] = [];
|
||||
const fullySentAssetPaths = new Set<string>();
|
||||
const updateAssetSpinnerText = (): void => {
|
||||
const processingAssetNames = unfinishedAssetPaths
|
||||
.filter(assetPath => fullySentAssetPaths.has(assetPath))
|
||||
.map(assetPath => path.basename(assetPath));
|
||||
assetSpinner.text = [
|
||||
uploadCountText,
|
||||
...activeUploadLines,
|
||||
...(processingAssetNames.length > 0
|
||||
? [
|
||||
`Waiting for the server to process ${processingAssetNames.slice(0, 3).join(', ')}` +
|
||||
(processingAssetNames.length > 3
|
||||
? ` and ${processingAssetNames.length - 3} more`
|
||||
: ''),
|
||||
]
|
||||
: []),
|
||||
].join('\n');
|
||||
};
|
||||
const reportUploadProgress = createUploadProgressReporter({
|
||||
isInteractive: isSpinnerEnabled(),
|
||||
onActiveUploadsChanged: lines => {
|
||||
activeUploadLines = lines;
|
||||
updateAssetSpinnerText();
|
||||
},
|
||||
});
|
||||
let lastUploadedStorageKeys = new Set<string>();
|
||||
let lastAssetUploadResults: {
|
||||
asset: RawAsset & { storageKey: string };
|
||||
@@ -452,23 +422,11 @@ export default class UpdatePublish extends EasCommand {
|
||||
|
||||
const totalAssets = assetUploadResults.length;
|
||||
const missingAssetCount = assetUploadResults.filter(a => !a.finished).length;
|
||||
uploadCountText = `Uploading (${totalAssets - missingAssetCount}/${totalAssets})`;
|
||||
unfinishedAssetPaths = assetUploadResults
|
||||
.filter(r => !r.finished)
|
||||
.map(r => r.asset.path);
|
||||
updateAssetSpinnerText();
|
||||
assetSpinner.text = `Uploading (${totalAssets - missingAssetCount}/${totalAssets})`;
|
||||
},
|
||||
progress => {
|
||||
() => {
|
||||
// when an upload is retried, reset the timeout as we know this will now need more time
|
||||
timeAtWhichToTimeout = Date.now() + NO_ACTIVITY_TIMEOUT; // reset timeout to NO_ACTIVITY_TIMEOUT
|
||||
if (progress) {
|
||||
if (progress.uploadedBytes >= progress.fileSize) {
|
||||
fullySentAssetPaths.add(progress.file);
|
||||
} else {
|
||||
fullySentAssetPaths.delete(progress.file);
|
||||
}
|
||||
reportUploadProgress(progress);
|
||||
}
|
||||
}
|
||||
),
|
||||
(async () => {
|
||||
|
||||
@@ -9,7 +9,7 @@ import { promptAsync } from '../../prompts';
|
||||
|
||||
export default class WorkflowRunCancel extends EasCommand {
|
||||
static override description =
|
||||
'cancel one or more workflow runs. If no workflow run IDs are provided, you will be prompted to select IN_PROGRESS runs to cancel';
|
||||
'Cancel one or more workflow runs. If no workflow run IDs are provided, you will be prompted to select IN_PROGRESS runs to cancel.';
|
||||
|
||||
static override strict = false;
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
|
||||
export default class WorkflowList extends EasCommand {
|
||||
static override hidden = true;
|
||||
|
||||
static override description = 'list workflows for the current project';
|
||||
static override description = 'List workflows for the current project';
|
||||
|
||||
static override flags = {
|
||||
...EasJsonOnlyFlag,
|
||||
|
||||
@@ -26,7 +26,7 @@ function printLogsForAllSteps(logs: JobLogs): void {
|
||||
|
||||
export default class WorkflowLogView extends EasCommand {
|
||||
static override description =
|
||||
'view logs for a workflow run, selecting a job and step to view. You can pass in either a workflow run ID or a job ID. If no ID is passed in, you will be prompted to select from recent workflow runs for the current project';
|
||||
'view logs for a workflow run, selecting a job and step to view. You can pass in either a workflow run ID or a job ID. If no ID is passed in, you will be prompted to select from recent workflow runs for the current project.';
|
||||
|
||||
static override flags = {
|
||||
...EasJsonOnlyFlag,
|
||||
|
||||
@@ -93,7 +93,7 @@ export function resolveWorkflowRunSshInput({
|
||||
|
||||
export default class WorkflowRun extends EasCommand {
|
||||
static override description =
|
||||
'run an EAS workflow. The entire local project directory will be packaged and uploaded to EAS servers for the workflow run, unless the --ref flag is used';
|
||||
'run an EAS workflow. The entire local project directory will be packaged and uploaded to EAS servers for the workflow run, unless the --ref flag is used.';
|
||||
|
||||
static override args = {
|
||||
file: Args.string({
|
||||
|
||||
@@ -32,7 +32,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
|
||||
|
||||
export default class WorkflowStatus extends EasCommand {
|
||||
static override description =
|
||||
'show the status of an existing workflow run. If no run ID is provided, you will be prompted to select from recent workflow runs for the current project';
|
||||
'show the status of an existing workflow run. If no run ID is provided, you will be prompted to select from recent workflow runs for the current project.';
|
||||
|
||||
static override args = {
|
||||
WORKFLOW_RUN_ID: Args.string({
|
||||
|
||||
@@ -59,7 +59,7 @@ const processedOutputs: (job: WorkflowJobResult) => FormatFieldsItem[] = job =>
|
||||
|
||||
export default class WorkflowView extends EasCommand {
|
||||
static override description =
|
||||
'view details for a workflow run, including jobs. If no run ID is provided, you will be prompted to select from recent workflow runs for the current project';
|
||||
'view details for a workflow run, including jobs. If no run ID is provided, you will be prompted to select from recent workflow runs for the current project.';
|
||||
|
||||
static override flags = {
|
||||
...EasJsonOnlyFlag,
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
import Log from '../../log';
|
||||
import { UploadProgress } from '../../uploads';
|
||||
import { createUploadProgressReporter } from '../uploadProgressReporter';
|
||||
|
||||
const MB = 1024 * 1024;
|
||||
|
||||
function makeProgress(file: string, uploadedBytes: number, elapsedMs: number): UploadProgress {
|
||||
return { file, fileSize: 7 * MB, uploadedBytes, elapsedMs };
|
||||
}
|
||||
|
||||
describe(createUploadProgressReporter, () => {
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('interactive', () => {
|
||||
it('shows a line for each active upload once it has run for 10 seconds', () => {
|
||||
const onActiveUploadsChanged = jest.fn();
|
||||
const report = createUploadProgressReporter({ isInteractive: true, onActiveUploadsChanged });
|
||||
|
||||
report(makeProgress('/dist/ios.hbc', MB, 5_000));
|
||||
expect(onActiveUploadsChanged).toHaveBeenLastCalledWith([]);
|
||||
|
||||
report(makeProgress('/dist/ios.hbc', MB, 20_000));
|
||||
report(makeProgress('/dist/android.hbc', 6 * MB, 10_000));
|
||||
|
||||
expect(onActiveUploadsChanged).toHaveBeenLastCalledWith([
|
||||
'ios.hbc: 1.0 MB of 7.0 MB, 51.2 KB/s, about 2 minutes left',
|
||||
'android.hbc: 6.0 MB of 7.0 MB, 614 KB/s',
|
||||
]);
|
||||
});
|
||||
|
||||
it('removes an upload once all of its bytes are sent', () => {
|
||||
const onActiveUploadsChanged = jest.fn();
|
||||
const report = createUploadProgressReporter({ isInteractive: true, onActiveUploadsChanged });
|
||||
|
||||
report(makeProgress('/dist/ios.hbc', MB, 20_000));
|
||||
report(makeProgress('/dist/ios.hbc', 7 * MB, 30_000));
|
||||
|
||||
expect(onActiveUploadsChanged).toHaveBeenLastCalledWith([]);
|
||||
});
|
||||
|
||||
it('shows at most 5 uploads', () => {
|
||||
const onActiveUploadsChanged = jest.fn();
|
||||
const report = createUploadProgressReporter({ isInteractive: true, onActiveUploadsChanged });
|
||||
|
||||
for (let i = 0; i < 7; i++) {
|
||||
report(makeProgress(`/dist/asset-${i}.png`, MB, 20_000));
|
||||
}
|
||||
|
||||
const lines = onActiveUploadsChanged.mock.calls.at(-1)?.[0];
|
||||
expect(lines).toHaveLength(6);
|
||||
expect(lines[5]).toBe('and 2 more uploads');
|
||||
});
|
||||
|
||||
it('does not log', () => {
|
||||
const warnSpy = jest.spyOn(Log, 'warn').mockImplementation(() => {});
|
||||
const logSpy = jest.spyOn(Log, 'log').mockImplementation(() => {});
|
||||
const report = createUploadProgressReporter({
|
||||
isInteractive: true,
|
||||
onActiveUploadsChanged: () => {},
|
||||
});
|
||||
|
||||
report(makeProgress('/dist/ios.hbc', MB, 20_000));
|
||||
|
||||
expect(warnSpy).not.toHaveBeenCalled();
|
||||
expect(logSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('non-interactive', () => {
|
||||
it('warns when an upload is slow and then logs its progress every minute', () => {
|
||||
let now = 0;
|
||||
jest.spyOn(Date, 'now').mockImplementation(() => now);
|
||||
const warnSpy = jest.spyOn(Log, 'warn').mockImplementation(() => {});
|
||||
const logSpy = jest.spyOn(Log, 'log').mockImplementation(() => {});
|
||||
const report = createUploadProgressReporter({
|
||||
isInteractive: false,
|
||||
onActiveUploadsChanged: () => {},
|
||||
});
|
||||
|
||||
report(makeProgress('/dist/ios.hbc', MB, 20_000));
|
||||
now = 30_000;
|
||||
report(makeProgress('/dist/ios.hbc', 2 * MB, 50_000));
|
||||
now = 60_000;
|
||||
report(makeProgress('/dist/ios.hbc', 3 * MB, 80_000));
|
||||
|
||||
expect(warnSpy).toHaveBeenCalledTimes(1);
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
'Uploading ios.hbc (7.0 MB) is slow (51.2 KB/s). It may take about 2 minutes.'
|
||||
);
|
||||
expect(logSpy).toHaveBeenCalledTimes(1);
|
||||
expect(logSpy).toHaveBeenCalledWith(
|
||||
'Still uploading ios.hbc: 3.0 MB of 7.0 MB, 38.4 KB/s, about 2 minutes left.'
|
||||
);
|
||||
});
|
||||
|
||||
it('warns again when a retried upload is still slow', () => {
|
||||
const warnSpy = jest.spyOn(Log, 'warn').mockImplementation(() => {});
|
||||
const report = createUploadProgressReporter({
|
||||
isInteractive: false,
|
||||
onActiveUploadsChanged: () => {},
|
||||
});
|
||||
|
||||
report(makeProgress('/dist/ios.hbc', MB, 20_000));
|
||||
report(makeProgress('/dist/ios.hbc', 0, 0));
|
||||
report(makeProgress('/dist/ios.hbc', MB, 20_000));
|
||||
|
||||
expect(warnSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not log when an upload is fast', () => {
|
||||
const warnSpy = jest.spyOn(Log, 'warn').mockImplementation(() => {});
|
||||
const logSpy = jest.spyOn(Log, 'log').mockImplementation(() => {});
|
||||
const report = createUploadProgressReporter({
|
||||
isInteractive: false,
|
||||
onActiveUploadsChanged: () => {},
|
||||
});
|
||||
|
||||
report(makeProgress('/dist/ios.hbc', 6 * MB, 10_000));
|
||||
|
||||
expect(warnSpy).not.toHaveBeenCalled();
|
||||
expect(logSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -44,7 +44,7 @@ import {
|
||||
truncateString as truncateUpdateMessage,
|
||||
updatePublishPlatformToAppPlatform,
|
||||
} from '../update/utils';
|
||||
import { PresignedPost, UploadProgress, uploadWithPresignedPostWithRetryAsync } from '../uploads';
|
||||
import { PresignedPost, uploadWithPresignedPostWithRetryAsync } from '../uploads';
|
||||
import {
|
||||
expoCommandAsync,
|
||||
shouldUseVersionedExpoCLI,
|
||||
@@ -496,7 +496,7 @@ export async function uploadAssetsAsync(
|
||||
onAssetUploadResultsChanged: (
|
||||
assetUploadResults: { asset: RawAsset & { storageKey: string }; finished: boolean }[]
|
||||
) => void,
|
||||
onAssetUploadBegin: (progress?: UploadProgress) => void
|
||||
onAssetUploadBegin: () => void
|
||||
): Promise<AssetUploadResult> {
|
||||
let assets: RawAsset[] = [];
|
||||
let platform: keyof FilteredCollectedAssets;
|
||||
@@ -552,7 +552,7 @@ export async function uploadAssetsAsync(
|
||||
// Sign URLs in chunks (endpoint caps at 100), then upload all from one pool so it stays saturated.
|
||||
const uploadBatchAsync = async (
|
||||
assetsToUpload: AssetWithStorageKey[],
|
||||
onBegin: (progress?: UploadProgress) => void
|
||||
onBegin: () => void
|
||||
): Promise<void> => {
|
||||
const signedAssets: { asset: AssetWithStorageKey; specification: string }[] = [];
|
||||
for (const assetChunk of chunk(assetsToUpload, 100)) {
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
import path from 'path';
|
||||
|
||||
import Log from '../log';
|
||||
import { UploadProgress } from '../uploads';
|
||||
import { formatBytes } from '../utils/files';
|
||||
|
||||
const MIN_UPLOAD_MS_BEFORE_REPORTING = 10_000;
|
||||
const SLOW_UPLOAD_SECONDS_LEFT = 60;
|
||||
const SLOW_UPLOAD_LOG_INTERVAL_MS = 60_000;
|
||||
const MAX_ACTIVE_UPLOAD_LINES = 5;
|
||||
|
||||
function getBytesPerSecond(progress: UploadProgress): number {
|
||||
return progress.uploadedBytes / (progress.elapsedMs / 1000);
|
||||
}
|
||||
|
||||
function getSecondsLeft(progress: UploadProgress): number {
|
||||
return (progress.fileSize - progress.uploadedBytes) / getBytesPerSecond(progress);
|
||||
}
|
||||
|
||||
function formatMinutesLeft(progress: UploadProgress): string {
|
||||
const minutes = Math.ceil(getSecondsLeft(progress) / 60);
|
||||
return minutes === 1 ? '1 minute' : `${minutes} minutes`;
|
||||
}
|
||||
|
||||
function formatUploadProgress(progress: UploadProgress): string {
|
||||
const message =
|
||||
`${path.basename(progress.file)}: ${formatBytes(progress.uploadedBytes)} of ${formatBytes(progress.fileSize)}, ` +
|
||||
`${formatBytes(getBytesPerSecond(progress))}/s`;
|
||||
return getSecondsLeft(progress) > SLOW_UPLOAD_SECONDS_LEFT
|
||||
? `${message}, about ${formatMinutesLeft(progress)} left`
|
||||
: message;
|
||||
}
|
||||
|
||||
function isSlowUpload(progress: UploadProgress): boolean {
|
||||
return (
|
||||
progress.elapsedMs >= MIN_UPLOAD_MS_BEFORE_REPORTING &&
|
||||
getSecondsLeft(progress) > SLOW_UPLOAD_SECONDS_LEFT
|
||||
);
|
||||
}
|
||||
|
||||
export function createUploadProgressReporter({
|
||||
isInteractive,
|
||||
onActiveUploadsChanged,
|
||||
}: {
|
||||
isInteractive: boolean;
|
||||
onActiveUploadsChanged: (lines: string[]) => void;
|
||||
}): (progress: UploadProgress) => void {
|
||||
const activeUploads = new Map<string, UploadProgress>();
|
||||
const lastSlowUploadLogTimes = new Map<string, number>();
|
||||
|
||||
return progress => {
|
||||
if (progress.uploadedBytes === 0) {
|
||||
lastSlowUploadLogTimes.delete(progress.file);
|
||||
}
|
||||
if (progress.uploadedBytes >= progress.fileSize) {
|
||||
activeUploads.delete(progress.file);
|
||||
} else {
|
||||
activeUploads.set(progress.file, progress);
|
||||
}
|
||||
|
||||
if (isInteractive) {
|
||||
const lines = [...activeUploads.values()]
|
||||
.filter(upload => upload.elapsedMs >= MIN_UPLOAD_MS_BEFORE_REPORTING)
|
||||
.map(formatUploadProgress);
|
||||
onActiveUploadsChanged(
|
||||
lines.length > MAX_ACTIVE_UPLOAD_LINES
|
||||
? [
|
||||
...lines.slice(0, MAX_ACTIVE_UPLOAD_LINES),
|
||||
`and ${lines.length - MAX_ACTIVE_UPLOAD_LINES} more uploads`,
|
||||
]
|
||||
: lines
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isSlowUpload(progress)) {
|
||||
return;
|
||||
}
|
||||
const lastLogTime = lastSlowUploadLogTimes.get(progress.file);
|
||||
if (lastLogTime === undefined) {
|
||||
Log.warn(
|
||||
`Uploading ${path.basename(progress.file)} (${formatBytes(progress.fileSize)}) is slow ` +
|
||||
`(${formatBytes(getBytesPerSecond(progress))}/s). It may take about ${formatMinutesLeft(progress)}.`
|
||||
);
|
||||
lastSlowUploadLogTimes.set(progress.file, Date.now());
|
||||
} else if (Date.now() - lastLogTime >= SLOW_UPLOAD_LOG_INTERVAL_MS) {
|
||||
Log.log(`Still uploading ${formatUploadProgress(progress)}.`);
|
||||
lastSlowUploadLogTimes.set(progress.file, Date.now());
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -20,13 +20,6 @@ export interface PresignedPost {
|
||||
fields: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface UploadProgress {
|
||||
file: string;
|
||||
fileSize: number;
|
||||
uploadedBytes: number;
|
||||
elapsedMs: number;
|
||||
}
|
||||
|
||||
export async function uploadFileAtPathToGCSAsync(
|
||||
graphqlClient: ExpoGraphqlClient,
|
||||
type: UploadSessionType,
|
||||
@@ -95,7 +88,7 @@ export async function uploadAppScopedFileAtPathToGCSAsync(
|
||||
export async function uploadWithPresignedPostWithRetryAsync(
|
||||
file: string,
|
||||
presignedPost: PresignedPost,
|
||||
onAssetUploadBegin: (progress?: UploadProgress) => void
|
||||
onAssetUploadBegin: () => void
|
||||
): Promise<Response> {
|
||||
return await promiseRetry(
|
||||
async retry => {
|
||||
@@ -103,7 +96,7 @@ export async function uploadWithPresignedPostWithRetryAsync(
|
||||
let response: Response;
|
||||
try {
|
||||
onAssetUploadBegin();
|
||||
response = await uploadWithPresignedPostAsync(file, presignedPost, onAssetUploadBegin);
|
||||
response = await uploadWithPresignedPostAsync(file, presignedPost);
|
||||
} catch (e: any) {
|
||||
return retry(e);
|
||||
}
|
||||
@@ -134,8 +127,7 @@ export async function uploadWithPresignedPostWithRetryAsync(
|
||||
|
||||
async function uploadWithPresignedPostAsync(
|
||||
file: string,
|
||||
presignedPost: PresignedPost,
|
||||
onProgress: (progress: UploadProgress) => void
|
||||
presignedPost: PresignedPost
|
||||
): Promise<Response> {
|
||||
const fileStat = await fs.stat(file);
|
||||
const fileSize = fileStat.size;
|
||||
@@ -143,15 +135,7 @@ async function uploadWithPresignedPostAsync(
|
||||
for (const [fieldKey, fieldValue] of Object.entries(presignedPost.fields)) {
|
||||
form.append(fieldKey, fieldValue);
|
||||
}
|
||||
const readStream = fs.createReadStream(file);
|
||||
form.append('file', readStream, { knownLength: fileSize });
|
||||
const startTime = Date.now();
|
||||
let uploadedBytes = 0;
|
||||
onProgress({ file, fileSize, uploadedBytes, elapsedMs: 0 });
|
||||
readStream.addListener('data', (chunk: Buffer | string) => {
|
||||
uploadedBytes += Buffer.byteLength(chunk);
|
||||
onProgress({ file, fileSize, uploadedBytes, elapsedMs: Date.now() - startTime });
|
||||
});
|
||||
form.append('file', fs.createReadStream(file), { knownLength: fileSize });
|
||||
const formHeaders = form.getHeaders();
|
||||
return await fetch(presignedPost.url, {
|
||||
method: 'POST',
|
||||
|
||||
@@ -39,6 +39,7 @@ describe(createBuildContext.name, () => {
|
||||
buildId: 'build-id',
|
||||
buildLogger,
|
||||
addBuildLogMask: jest.fn(),
|
||||
createLogMaskingStream: jest.fn(),
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { TransformCallback, Writable } from 'node:stream';
|
||||
import { Readable, Transform, TransformCallback, Writable } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { EnvironmentSecretType } from '@expo/eas-build-job';
|
||||
import z from 'zod';
|
||||
|
||||
@@ -356,3 +357,99 @@ describe('logger', () => {
|
||||
expect(outputStream.readableLength).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createLogMaskingStream', () => {
|
||||
async function maskAsync(stream: Transform, chunks: Buffer[]): Promise<Buffer> {
|
||||
const output: Buffer[] = [];
|
||||
await pipeline(
|
||||
Readable.from(chunks),
|
||||
stream,
|
||||
new Writable({
|
||||
write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
|
||||
output.push(chunk);
|
||||
callback();
|
||||
},
|
||||
})
|
||||
);
|
||||
return Buffer.concat(output);
|
||||
}
|
||||
|
||||
it('masks secrets, secrets as Xcode prints them in build settings, and decoded secrets', async () => {
|
||||
const { createLogMaskingStream } = await createBuildLoggerWithSecretsFilter({
|
||||
environmentSecrets: [
|
||||
{ name: 'API_KEY', value: 'ab cd=ef$gh', type: EnvironmentSecretType.STRING },
|
||||
{
|
||||
name: 'ANOTHER_SECRET_BASE64',
|
||||
value: 'YW5vdGhlclNlY3JldA==',
|
||||
type: EnvironmentSecretType.STRING,
|
||||
},
|
||||
],
|
||||
});
|
||||
const log = [
|
||||
' export API_KEY\\=ab\\ cd\\=ef\\$gh',
|
||||
'token is ab cd=ef$gh',
|
||||
'decoded anotherSecret here',
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
const masked = await maskAsync(createLogMaskingStream(), [Buffer.from(log)]);
|
||||
|
||||
expect(masked.toString()).toBe(
|
||||
[
|
||||
` export API_KEY\\=${'*'.repeat(14)}`,
|
||||
`token is ${'*'.repeat(11)}`,
|
||||
`decoded ${'*'.repeat(13)} here`,
|
||||
'',
|
||||
].join('\n')
|
||||
);
|
||||
});
|
||||
|
||||
it('masks secrets split across chunks, including masks added after the logger was created', async () => {
|
||||
const { createLogMaskingStream, addBuildLogMask } = await createBuildLoggerWithSecretsFilter(
|
||||
{}
|
||||
);
|
||||
addBuildLogMask('leased-token');
|
||||
|
||||
const masked = await maskAsync(createLogMaskingStream(), [
|
||||
Buffer.from('first line leased-'),
|
||||
Buffer.from('token\nsecond '),
|
||||
Buffer.from('leased-token without a newline'),
|
||||
]);
|
||||
|
||||
expect(masked.toString()).toBe(
|
||||
`first line ${'*'.repeat(12)}\nsecond ${'*'.repeat(12)} without a newline`
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps everything except secrets byte for byte, including multibyte and invalid UTF-8', async () => {
|
||||
const { createLogMaskingStream } = await createBuildLoggerWithSecretsFilter({
|
||||
environmentSecrets: [
|
||||
{ name: 'PASSWORD', value: 'pässwörd', type: EnvironmentSecretType.STRING },
|
||||
],
|
||||
});
|
||||
const log = Buffer.concat([
|
||||
Buffer.from('✓ Compiling café.swift\r\n'),
|
||||
Buffer.from([0xff, 0xfe, 0x0a]),
|
||||
Buffer.from('password: pässwörd\n'),
|
||||
]);
|
||||
// Split inside the multibyte characters.
|
||||
const chunks = [
|
||||
log.subarray(0, 1),
|
||||
log.subarray(1, 20),
|
||||
log.subarray(20, 40),
|
||||
log.subarray(40),
|
||||
];
|
||||
|
||||
const masked = await maskAsync(createLogMaskingStream(), chunks);
|
||||
|
||||
const passwordBytes = Buffer.byteLength('pässwörd');
|
||||
expect(masked).toEqual(
|
||||
Buffer.concat([
|
||||
Buffer.from('✓ Compiling café.swift\r\n'),
|
||||
Buffer.from([0xff, 0xfe, 0x0a]),
|
||||
Buffer.from(`password: ${'*'.repeat(passwordBytes)}\n`),
|
||||
])
|
||||
);
|
||||
expect(masked.length).toBe(log.length);
|
||||
});
|
||||
});
|
||||
@@ -115,6 +115,7 @@ describe('BuildService Datadog setup', () => {
|
||||
createBuildLoggerWithSecretsFilterMock.mockResolvedValue({
|
||||
logger: buildLogger,
|
||||
addBuildLogMask: jest.fn(),
|
||||
createLogMaskingStream: jest.fn(),
|
||||
cleanUp: jest.fn(async () => {}),
|
||||
logBuffer: { getLogs: () => [], getPhaseLogs: () => [] } as any,
|
||||
outputStream: {} as any,
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { RemoteLoggerStream, uploadWithSignedUrl } from '@expo/build-tools';
|
||||
import fs from 'fs-extra';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { Readable } from 'stream';
|
||||
import zlib from 'zlib';
|
||||
|
||||
import config from '../config';
|
||||
import { uploadXcodeBuildLogs } from '../ios/xcodeLogs';
|
||||
import { createTextMaskingStream } from '../logger';
|
||||
|
||||
// These tests use real fs and directories in os.tmpdir(), since memfs read streams can't be closed.
|
||||
jest.unmock('fs');
|
||||
jest.unmock('node:fs');
|
||||
|
||||
const logger = { warn: jest.fn() } as any;
|
||||
|
||||
async function readAllAsync(stream: Readable): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
describe(uploadXcodeBuildLogs, () => {
|
||||
const originalGcs = { ...config.loggers.gcs };
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'xcode-logs-test-'));
|
||||
config.loggers.gcs.signedUploadUrlForXcodeBuildLogs = {
|
||||
url: 'https://gcs.expo.test/xcode-logs',
|
||||
headers: {},
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
Object.assign(config.loggers.gcs, originalGcs);
|
||||
jest.mocked(uploadWithSignedUrl).mockReset();
|
||||
await fs.remove(tmpDir);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['no compression', null, (body: Buffer) => body],
|
||||
['gzip', RemoteLoggerStream.CompressionMethod.GZIP, zlib.gunzipSync],
|
||||
['brotli', RemoteLoggerStream.CompressionMethod.BR, zlib.brotliDecompressSync],
|
||||
])('uploads the log with secrets masked (%s)', async (_name, compressionMethod, decompress) => {
|
||||
config.loggers.gcs.compressionMethod = compressionMethod as any;
|
||||
const logsPath = path.join(tmpDir, 'xcodebuild.log');
|
||||
await fs.writeFile(logsPath, ' export API_KEY\\=ab\\ cd\\=ef\\$gh\n** BUILD SUCCEEDED **\n');
|
||||
let uploaded: Buffer | undefined;
|
||||
jest.mocked(uploadWithSignedUrl).mockImplementation(async ({ srcGeneratorAsync }) => {
|
||||
uploaded = await readAllAsync((await srcGeneratorAsync()) as Readable);
|
||||
return 'https://gcs.expo.test/xcode-logs';
|
||||
});
|
||||
|
||||
await uploadXcodeBuildLogs(logger, logsPath, () =>
|
||||
createTextMaskingStream(new Set(['ab cd=ef$gh']))
|
||||
);
|
||||
|
||||
expect(decompress(uploaded!).toString()).toBe(
|
||||
` export API_KEY\\=${'*'.repeat(14)}\n** BUILD SUCCEEDED **\n`
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['no compression', null],
|
||||
['gzip', RemoteLoggerStream.CompressionMethod.GZIP],
|
||||
])('fails the upload when the log cannot be read (%s)', async (_name, compressionMethod) => {
|
||||
config.loggers.gcs.compressionMethod = compressionMethod as any;
|
||||
jest.mocked(uploadWithSignedUrl).mockImplementation(async ({ srcGeneratorAsync }) => {
|
||||
await readAllAsync((await srcGeneratorAsync()) as Readable);
|
||||
return 'https://gcs.expo.test/xcode-logs';
|
||||
});
|
||||
|
||||
// A directory can be stat'ed but not read.
|
||||
await expect(
|
||||
uploadXcodeBuildLogs(logger, tmpDir, () => createTextMaskingStream(new Set(['secret'])))
|
||||
).rejects.toThrow('EISDIR');
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { BuildContext, BuildContextOptions, LogBuffer, RuntimeSettings } from '@
|
||||
import { Job, ManagedArtifactType, Metadata, Platform } from '@expo/eas-build-job';
|
||||
import { bunyan } from '@expo/logger';
|
||||
import assert from 'assert';
|
||||
import { Transform } from 'stream';
|
||||
|
||||
import { GCSCacheManager } from './CacheManager';
|
||||
import config from './config';
|
||||
@@ -27,6 +28,7 @@ export async function createBuildContext<TJob extends Job>({
|
||||
buildId,
|
||||
buildLogger,
|
||||
addBuildLogMask,
|
||||
createLogMaskingStream,
|
||||
}: {
|
||||
job: TJob;
|
||||
logBuffer: LogBuffer;
|
||||
@@ -36,6 +38,7 @@ export async function createBuildContext<TJob extends Job>({
|
||||
buildId: string;
|
||||
buildLogger: bunyan;
|
||||
addBuildLogMask: (secret: string) => void;
|
||||
createLogMaskingStream: () => Transform;
|
||||
}): Promise<BuildContext<TJob>> {
|
||||
const childLogger = buildLogger.child({ buildId });
|
||||
await RuntimeSettings.loadAsync({
|
||||
@@ -51,7 +54,7 @@ export async function createBuildContext<TJob extends Job>({
|
||||
switch (type) {
|
||||
case ManagedArtifactType.XCODE_BUILD_LOGS: {
|
||||
if (job.platform === Platform.IOS) {
|
||||
await uploadXcodeBuildLogs(logger, paths[0]);
|
||||
await uploadXcodeBuildLogs(logger, paths[0], createLogMaskingStream);
|
||||
return { filename: null };
|
||||
}
|
||||
throw new Error('Uploading Xcode logs in non-iOS builds is not supported');
|
||||
|
||||
@@ -3,14 +3,18 @@ import { bunyan } from '@expo/logger';
|
||||
import fs from 'fs-extra';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { Readable, pipeline } from 'stream';
|
||||
import { Readable, Transform, pipeline } from 'stream';
|
||||
import { promisify } from 'util';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
import zlib from 'zlib';
|
||||
|
||||
import config from '../config';
|
||||
|
||||
export async function uploadXcodeBuildLogs(logger: bunyan, logsPath: string): Promise<void> {
|
||||
export async function uploadXcodeBuildLogs(
|
||||
logger: bunyan,
|
||||
logsPath: string,
|
||||
createLogMaskingStream: () => Transform
|
||||
): Promise<void> {
|
||||
if (!config.loggers.gcs.signedUploadUrlForXcodeBuildLogs) {
|
||||
logger.warn('GCS Presigned URL for Xcode logs was not specified, skipping upload');
|
||||
} else {
|
||||
@@ -18,8 +22,15 @@ export async function uploadXcodeBuildLogs(logger: bunyan, logsPath: string): Pr
|
||||
signedUrl: config.loggers.gcs.signedUploadUrlForXcodeBuildLogs,
|
||||
srcGeneratorAsync: async () => {
|
||||
const stat = await fs.stat(logsPath);
|
||||
// Masking replaces each byte of a secret with one byte, so the size stays the same.
|
||||
// pipeline passes read errors on to the masked stream, which fails the upload.
|
||||
const maskedStream = pipeline(
|
||||
fs.createReadStream(logsPath),
|
||||
createLogMaskingStream(),
|
||||
() => {}
|
||||
);
|
||||
const { stream } = await compress(
|
||||
{ stream: fs.createReadStream(logsPath), length: stat.size },
|
||||
{ stream: maskedStream, length: stat.size },
|
||||
config.loggers.gcs.compressionMethod
|
||||
);
|
||||
return stream;
|
||||
|
||||
@@ -102,6 +102,40 @@ function createTransformStream(secretPatterns: ReadonlySet<string>): Transform {
|
||||
});
|
||||
}
|
||||
|
||||
// Characters that Xcode escapes with a backslash when it prints build settings.
|
||||
const XCODE_ESCAPED_CHARACTERS = /[\t !"#$&'()*;<=>?[\\\]^`{|}~]/g;
|
||||
|
||||
/**
|
||||
* Masks secrets in a text file's contents, line by line, including secrets written the way Xcode
|
||||
* prints build settings. Masking works on bytes, so everything else stays unchanged.
|
||||
*/
|
||||
export function createTextMaskingStream(secretPatterns: ReadonlySet<string>): Transform {
|
||||
const patterns = [
|
||||
...new Set(
|
||||
[...secretPatterns].flatMap(secret => [
|
||||
secret,
|
||||
secret.replace(XCODE_ESCAPED_CHARACTERS, '\\$&'),
|
||||
])
|
||||
),
|
||||
].map(pattern => Buffer.from(pattern).toString('latin1'));
|
||||
const maskLine = (line: string): string =>
|
||||
patterns.some(pattern => line.includes(pattern)) ? maskSecrets(line, patterns) : line;
|
||||
let partialLine = '';
|
||||
return new Transform({
|
||||
transform(chunk: Buffer, _encoding: BufferEncoding, callback: TransformCallback) {
|
||||
const lines = (partialLine + chunk.toString('latin1')).split('\n');
|
||||
partialLine = lines.pop() ?? '';
|
||||
callback(
|
||||
null,
|
||||
lines.length > 0 ? Buffer.from(`${lines.map(maskLine).join('\n')}\n`, 'latin1') : undefined
|
||||
);
|
||||
},
|
||||
flush(callback: TransformCallback) {
|
||||
callback(null, partialLine ? Buffer.from(maskLine(partialLine), 'latin1') : undefined);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function maskSecrets(message: string, secrets: Iterable<string>): string {
|
||||
const characters = message.split('');
|
||||
for (const secret of secrets) {
|
||||
@@ -170,6 +204,7 @@ function createDiscardStream(): Writable {
|
||||
export async function createBuildLoggerWithSecretsFilter(secrets: Job['secrets']): Promise<{
|
||||
logger: bunyan;
|
||||
addBuildLogMask: (secret: string) => void;
|
||||
createLogMaskingStream: () => Transform;
|
||||
cleanUp: () => Promise<void>;
|
||||
logBuffer: WorkerLogBuffer;
|
||||
outputStream: Readable;
|
||||
@@ -239,6 +274,8 @@ export async function createBuildLoggerWithSecretsFilter(secrets: Job['secrets']
|
||||
return {
|
||||
logger: buildLogger,
|
||||
addBuildLogMask,
|
||||
// Reads the masks when the stream is created, so it includes masks added during the job.
|
||||
createLogMaskingStream: () => createTextMaskingStream(secretPatterns),
|
||||
cleanUp: async () => {
|
||||
await Promise.all([remoteLoggerStream?.cleanUp?.(), httpLogStream?.cleanUp?.()]);
|
||||
},
|
||||
|
||||
@@ -275,6 +275,7 @@ export default class BuildService {
|
||||
const {
|
||||
logger: buildLogger,
|
||||
addBuildLogMask,
|
||||
createLogMaskingStream,
|
||||
cleanUp,
|
||||
logBuffer,
|
||||
} = await createBuildLoggerWithSecretsFilter(job.secrets ?? {});
|
||||
@@ -315,6 +316,7 @@ export default class BuildService {
|
||||
buildId: this.buildId,
|
||||
buildLogger,
|
||||
addBuildLogMask,
|
||||
createLogMaskingStream,
|
||||
});
|
||||
this.buildContext = ctx;
|
||||
|
||||
|
||||
Reference in new issue
Block a user