Compare commits

...
Author SHA1 Message Date
Gabe Debes c64f922588 [eas-cli] Keep update uploads alive while bytes are sent and report slow uploads 2026-10-28 09:16:26 -07:00
Ramon Claudio a0fe3ff2c3 [eas-cli][build-tools] Bump Node.js to 22.23.4 (#4714)
* [eas-cli] Bump Node.js to 22.23.4

* [eas-cli] Add changeset

* [build-tools] Bump Node.js to 22.23.4
2026-10-21 18:38:36 -04:00
Brent VatneandClaude Opus 5.5 fb383d818b [eas-cli] make command descriptions lowercase without trailing periods (#4385)
* [eas-cli] make command descriptions lowercase without trailing periods

Most command descriptions in eas --help already start lowercase and end
without a period; align the remaining 11 capitalized and 12 period-terminated
ones with that style. Proper nouns, [EXPERIMENTAL] prefixes, and later
paragraphs of multi-line descriptions are unchanged.

* [eas-cli] Drop trailing periods from simulator command descriptions

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

---------

Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
2026-10-21 14:27:33 -07:00
Stanisław Chmiela c07c5edba9 [build-tools] Add max_duration_seconds to sandboxes (#4702)
# Why

I want to set up billing and limits for sandboxes so we need to be able to say how long can a sandbox run for.

# How

Added simple `max_duration_seconds` like we have in agent runs and device run sessions.

# Test Plan

CI should pass.
2026-10-21 17:18:26 +02:00
35 changed files with 496 additions and 41 deletions

No files matched your search

+5
View File
@@ -0,0 +1,5 @@
---
'eas-cli': none
---
Make command descriptions in `eas --help` consistently lowercase without trailing periods.
+6
View File
@@ -0,0 +1,6 @@
---
'eas-cli': none
'@expo/build-tools': none
---
Bump Node.js to 22.23.4 to fix the CVEs in the [October 2026 Node.js security releases](https://nodejs.org/en/blog/vulnerability/october-2026-security-releases).
+5
View File
@@ -0,0 +1,5 @@
---
'@expo/build-tools': minor
---
Stop sandboxes after an optional maximum duration measured from readiness.
+1 -1
View File
@@ -6,4 +6,4 @@ MISE_NODE_COREPACK = "true"
actionlint = "1.7.12"
gcloud = "570.0.0"
jq = "1.8.1"
node = "22.23.3"
node = "22.23.4"
+1 -1
View File
@@ -26,7 +26,7 @@
"build:egress-guard": "resources/egress-guard/build.sh",
"test:egress-guard": "resources/egress-guard/tests/run-policy-tests.sh && resources/egress-guard/tests/run-guard-tests.sh",
"typecheck": "tsc",
"generate-appium-commands": "mise exec [email protected]2.0 -- node scripts/generate-appium-commands.js",
"generate-appium-commands": "mise exec [email protected]3.4 -- node scripts/generate-appium-commands.js",
"prepack": "rimraf dist \"*.tsbuildinfo\" && yarn gql && tsc -p tsconfig.build.json",
"jest-unit": "jest --config jest/unit-config.ts",
"jest-integration": "jest --config jest/integration-config.ts",
@@ -1,4 +1,4 @@
import { SystemError } from '@expo/eas-build-job';
import { SystemError, UserError } from '@expo/eas-build-job';
import { Client, fetchExchange } from '@urql/core';
import * as sandboxDaemon from '../../utils/sandboxDaemon';
@@ -6,6 +6,90 @@ import * as sandboxDaemon from '../../utils/sandboxDaemon';
import { createStartSandboxBuildFunction, markSandboxReadyAsync } from '../startSandbox';
describe('sandbox build functions', () => {
it.each([
{ completion: 'duration', duration: 0.02 },
{ completion: 'cancellation', duration: 60 },
{ completion: 'duration', duration: 0.0001 },
])(
'stops after $completion with $duration seconds only after readiness',
async ({ completion, duration }) => {
const controller = new AbortController();
let ready!: () => void;
const stopAsync = jest.fn(async () => {});
const start = jest.spyOn(sandboxDaemon, 'startSandboxDaemonAsync').mockResolvedValue({
ready: new Promise<void>(resolve => {
ready = resolve;
}),
stopAsync,
});
const mutation = jest.fn(() => ({ toPromise: async () => ({ data: {} }) }));
const logger = { info: jest.fn(), child: jest.fn() };
logger.child.mockReturnValue(logger);
const fn = createStartSandboxBuildFunction({
mcpServerUrl: 'ws://localhost:8787',
graphqlClient: { mutation },
} as any);
try {
const pending = fn.fn!({ logger } as any, {
inputs: {
sandbox_id: { value: 'sandbox-id' },
max_duration_seconds: { value: duration },
},
outputs: {},
env: { __EAS_SANDBOX_MCP_TOKEN: 'token' },
signal: controller.signal,
});
await new Promise(resolve => setTimeout(resolve, 40));
expect(mutation).not.toHaveBeenCalled();
expect(stopAsync).not.toHaveBeenCalled();
ready();
if (completion === 'cancellation') {
await new Promise(resolve => setImmediate(resolve));
expect(stopAsync).not.toHaveBeenCalled();
controller.abort();
}
await pending;
expect(mutation).toHaveBeenCalledTimes(1);
expect(stopAsync).toHaveBeenCalledTimes(1);
const timeoutMessage = `Sandbox reached its maximum runtime of ${duration} seconds. Stopping.`;
if (completion === 'duration') {
expect(logger.info).toHaveBeenCalledWith(timeoutMessage);
} else {
expect(logger.info).not.toHaveBeenCalledWith(timeoutMessage);
}
} finally {
start.mockRestore();
}
}
);
it.each([0, -1, Infinity, NaN])(
'rejects invalid maximum duration %s before starting the daemon',
async duration => {
const start = jest.spyOn(sandboxDaemon, 'startSandboxDaemonAsync');
const fn = createStartSandboxBuildFunction({
mcpServerUrl: 'ws://localhost:8787',
} as any);
try {
const result = fn.fn!({} as any, {
inputs: {
sandbox_id: { value: 'sandbox-id' },
max_duration_seconds: { value: duration },
},
outputs: {},
env: { __EAS_SANDBOX_MCP_TOKEN: 'token' },
});
await expect(result).rejects.toBeInstanceOf(UserError);
await expect(result).rejects.toThrow(
`Invalid sandbox maximum duration: ${duration}. Provide a finite number greater than zero, or omit max_duration_seconds.`
);
expect(start).not.toHaveBeenCalled();
} finally {
start.mockRestore();
}
}
);
it('aborts an active ready request and keeps its authentication headers', async () => {
const controller = new AbortController();
let requestStarted!: () => void;
@@ -1,4 +1,4 @@
import { SystemError } from '@expo/eas-build-job';
import { SystemError, UserError } from '@expo/eas-build-job';
import { BuildFunction, BuildStepInput, BuildStepInputValueTypeName } from '@expo/steps';
import { graphql } from 'gql.tada';
@@ -32,6 +32,11 @@ export function createStartSandboxBuildFunction(ctx: CustomBuildContext): BuildF
required: true,
allowedValueTypeName: BuildStepInputValueTypeName.STRING,
}),
BuildStepInput.createProvider({
id: 'max_duration_seconds',
required: false,
allowedValueTypeName: BuildStepInputValueTypeName.NUMBER,
}),
],
fn: async (stepCtx, { inputs, signal, env }) => {
// The daemon needs this credential, but shell commands do not.
@@ -44,6 +49,18 @@ export function createStartSandboxBuildFunction(ctx: CustomBuildContext): BuildF
throw new SystemError('MCP server URL is required to start the sandbox daemon.');
}
const sandboxId = String(inputs.sandbox_id.value);
const maxDurationSeconds = inputs.max_duration_seconds?.value;
if (
maxDurationSeconds !== undefined &&
(typeof maxDurationSeconds !== 'number' ||
!Number.isFinite(maxDurationSeconds) ||
maxDurationSeconds <= 0)
) {
throw new UserError(
'EAS_SANDBOX_INVALID_DURATION',
`Invalid sandbox maximum duration: ${String(maxDurationSeconds)}. Provide a finite number greater than zero, or omit max_duration_seconds.`
);
}
const daemon = await startSandboxDaemonAsync({
credential: sandboxToken,
serverUrl: mcpServerUrl,
@@ -63,7 +80,21 @@ export function createStartSandboxBuildFunction(ctx: CustomBuildContext): BuildF
stepCtx.logger.info('Sandbox daemon started.');
await withLogPhaseAsync(stepCtx.logger, AWAIT_SANDBOX_STEP_NAME, async () => {
await waitForCancellationAsync(signal);
const timeoutSignal =
maxDurationSeconds === undefined
? undefined
: AbortSignal.timeout(Math.ceil(maxDurationSeconds * 1000));
await waitForCompletionAsync(
AbortSignal.any([
...(signal ? [signal] : []),
...(timeoutSignal ? [timeoutSignal] : []),
])
);
if (timeoutSignal?.aborted) {
stepCtx.logger.info(
`Sandbox reached its maximum runtime of ${maxDurationSeconds} seconds. Stopping.`
);
}
});
} finally {
await daemon.stopAsync();
@@ -89,11 +120,11 @@ export async function markSandboxReadyAsync(
}
}
async function waitForCancellationAsync(signal?: AbortSignal): Promise<void> {
if (signal?.aborted) {
async function waitForCompletionAsync(signal: AbortSignal): Promise<void> {
if (signal.aborted) {
return;
}
await new Promise<void>(resolve => {
signal?.addEventListener('abort', () => resolve(), { once: true });
signal.addEventListener('abort', () => resolve(), { once: true });
});
}
+1 -1
View File
@@ -252,7 +252,7 @@
},
"update": {
"node": {
"version": "22.23.3"
"version": "22.23.4"
},
"s3": {
"templates": {
+49 -1
View File
@@ -3,8 +3,56 @@ import os from 'node:os';
import path from 'node:path';
import fs from 'fs-extra';
import nock from 'nock';
import { computeFileMd5Base64Async } from '../uploads';
import { computeFileMd5Base64Async, uploadWithPresignedPostWithRetryAsync } from '../uploads';
describe(uploadWithPresignedPostWithRetryAsync, () => {
const presignedPost = {
url: 'https://storage.example.com/upload',
fields: { key: 'object-key' },
};
const contents = Array.from({ length: 50000 }, (_, i) => `line ${i}\n`).join('');
let tmpDir: string;
let file: string;
beforeAll(async () => {
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'eas-cli-presigned-post-test-'));
file = path.join(tmpDir, 'bundle.js');
await fs.writeFile(file, contents);
});
afterAll(async () => {
await fs.remove(tmpDir);
});
afterEach(() => {
nock.cleanAll();
});
it('sends the whole file and reports activity while it uploads', async () => {
let body = '';
nock('https://storage.example.com')
.post('/upload', requestBody => {
body = requestBody;
return true;
})
.reply(204);
const onAssetUploadBegin = jest.fn();
await uploadWithPresignedPostWithRetryAsync(file, presignedPost, onAssetUploadBegin);
expect(body).toContain('filename="bundle.js"');
expect(body).toContain(contents);
expect(onAssetUploadBegin.mock.calls.length).toBeGreaterThan(2);
expect(onAssetUploadBegin).toHaveBeenCalledWith(
expect.objectContaining({ file, fileSize: contents.length, uploadedBytes: 0 })
);
expect(onAssetUploadBegin).toHaveBeenLastCalledWith(
expect.objectContaining({ file, fileSize: contents.length, uploadedBytes: contents.length })
);
});
});
describe(computeFileMd5Base64Async, () => {
let tmpDir: string;
+1 -1
View File
@@ -33,7 +33,7 @@ const PROJECT_PAGES: Record<string, string> = {
export default class Browse extends EasCommand {
static override description =
'Transition from the terminal to the web browser to view and interact with your project on https://expo.dev';
'transition from the terminal to the web browser to view and interact with your project on https://expo.dev';
static override args = {
page: Args.string({
@@ -13,7 +13,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
import { pollForBackgroundJobReceiptAsync } from '../../utils/pollForBackgroundJobReceiptAsync';
export default class ChannelDelete extends EasCommand {
static override description = 'Delete a channel';
static override description = 'delete a channel';
static override args = {
name: Args.string({
@@ -50,7 +50,7 @@ type ChannelRolloutArgsAndFlags = {
Partial<CreateRolloutNonInteractiveOptions>;
export default class ChannelRollout extends EasCommand {
static override description = 'Roll a new branch out on a channel incrementally.';
static override description = 'roll a new branch out on a channel incrementally';
static override args = {
channel: Args.string({
@@ -7,7 +7,7 @@ import { SetUpBuildCredentialsCommandAction } from '../../credentials/manager/Se
import { selectPlatformAsync } from '../../platform';
export default class InitializeBuildCredentials extends EasCommand {
static override description = 'Set up credentials for building your project.';
static override description = 'set up credentials for building your project';
static override flags = {
platform: Flags.option({
@@ -22,7 +22,7 @@ interface DeployAliasDeleteFlags {
}
export default class WorkerAliasDelete extends EasCommand {
static override description = 'Delete deployment aliases.';
static override description = 'delete deployment aliases';
static override aliases = ['worker:alias:delete'];
static override state = 'preview';
@@ -39,7 +39,7 @@ interface RawDeployAliasFlags {
}
export default class WorkerAlias extends EasCommand {
static override description = 'Assign deployment aliases.';
static override description = 'assign deployment aliases';
static override aliases = ['worker:alias', 'deploy:promote'];
static override state = 'preview';
@@ -23,7 +23,7 @@ interface RawDeployDeleteFlags {
}
export default class WorkerDelete extends EasCommand {
static override description = 'Delete a deployment.';
static override description = 'delete a deployment';
static override aliases = ['worker:delete'];
static override state = 'preview';
+1 -1
View File
@@ -100,7 +100,7 @@ async function withSuppressedOutputAsync<T>(fn: () => Promise<T>): Promise<T> {
/* eslint-enable no-console */
export default class Go extends EasCommand {
static override description = 'Create a custom Expo Go and submit to TestFlight';
static override description = 'create a custom Expo Go and submit to TestFlight';
static override hidden = true;
static override flags = {
@@ -22,7 +22,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
export default class ObserveEvent extends EasCommand {
static override description =
'display a single Observe event (metric, log, or error) by its ID. IDs are included in event data when the `--json` flag is passed to `eas observe:session`, `eas observe:metrics`, or `eas observe:events`.';
'display a single Observe event (metric, log, or error) by its ID. IDs are included in event data when the `--json` flag is passed to `eas observe:session`, `eas observe:metrics`, or `eas observe:events`';
static override args = {
id: Args.string({
@@ -36,7 +36,7 @@ const DEFAULT_EVENTS_LIMIT = 10;
export default class ObserveEvents extends EasCommand {
static override description =
'display individual events emitted by the app via `logEvent`, filtered by the event name in the argument. With no arguments, a list of the available event names and associated event counts is returned.';
'display individual events emitted by the app via `logEvent`, filtered by the event name in the argument. With no arguments, a list of the available event names and associated event counts is returned';
static override args = {
eventName: Args.string({
+1 -1
View File
@@ -134,7 +134,7 @@ export default class New extends EasCommand {
static override aliases = ['new'];
static override description =
'Create a new project configured with Expo Application Services (EAS)';
'create a new project configured with Expo Application Services (EAS)';
static override args = {
path: Args.string({
@@ -35,7 +35,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
export default class SimulatorControl extends EasCommand {
static override aliases = ['sim:control'];
static override description =
'[EXPERIMENTAL] run a serve-sim action on a remote iOS simulator session on EAS. Run without an action to list the actions the session supports. To save a screenshot locally, run `eas simulator:control screenshot.capture --output ./screenshot.png`.';
'[EXPERIMENTAL] run a serve-sim action on a remote iOS simulator session on EAS. Run without an action to list the actions the session supports. To save a screenshot locally, run `eas simulator:control screenshot.capture --output ./screenshot.png`';
static override strict = false;
static override examples = [
@@ -39,7 +39,7 @@ type DeviceRunSessionArtifact = DeviceRunSessionById['artifacts'][number];
export default class SimulatorGet extends EasCommand {
static override aliases = ['sim:get'];
static override description =
"[EXPERIMENTAL] get info about a remote simulator session on EAS by its simulator session ID. Preview links follow the session's saved sharing preference.";
"[EXPERIMENTAL] get info about a remote simulator session on EAS by its simulator session ID. Preview links follow the session's saved sharing preference";
static override flags = {
id: Flags.string({
@@ -83,7 +83,7 @@ export default class Simulator extends EasCommand {
static override aliases = ['simulator:start', 'sim:start'];
static override hiddenAliases = ['sim'];
static override description =
'[EXPERIMENTAL] start a remote simulator session on EAS and get instructions to connect to it. The default browser preview link requires sign-in.';
'[EXPERIMENTAL] start a remote simulator session on EAS and get instructions to connect to it. The default browser preview link requires sign-in';
static override flags = {
platform: Flags.option({
@@ -61,6 +61,7 @@ jest.mock('../../../graphql/queries/UpdateQuery');
jest.mock('../../../prompts');
jest.mock('../../../graphql/queries/EnvironmentVariablesQuery');
jest.mock('../../../ora', () => ({
isSpinnerEnabled: () => false,
ora: () => ({
start: () => ({ succeed: () => {}, fail: () => {}, stop: () => {} }),
}),
+46 -4
View File
@@ -3,6 +3,7 @@ import { EasJson, EasJsonAccessor, EasJsonUtils } from '@expo/eas-json';
import { Errors, Flags } from '@oclif/core';
import chalk from 'chalk';
import nullthrows from 'nullthrows';
import path from 'path';
import { ensureBranchExistsAsync } from '../../branch/queries';
import { ensureRepoIsCleanAsync } from '../../build/utils/repository';
@@ -28,7 +29,7 @@ import {
} from '../../graphql/generated';
import { PublishMutation } from '../../graphql/mutations/PublishMutation';
import Log, { learnMore, link } from '../../log';
import { ora } from '../../ora';
import { isSpinnerEnabled, ora } from '../../ora';
import { RequestedPlatform } from '../../platform';
import { maybeUploadAssetMapAsync } from '../../project/maybeUploadAssetMapAsync';
import {
@@ -56,6 +57,7 @@ import {
resolveInputDirectoryAsync,
uploadAssetsAsync,
} from '../../project/publish';
import { createUploadProgressReporter } from '../../project/uploadProgressReporter';
import { resolveWorkflowPerPlatformAsync } from '../../project/workflow';
import { resolveUpdateGroupsSupersedingActiveRolloutsAsync } from '../../update/active-rollout';
import { ensureEASUpdateIsConfiguredAsync } from '../../update/configure';
@@ -389,13 +391,41 @@ export default class UpdatePublish extends EasCommand {
// - Start with NO_ACTIVITY_TIMEOUT. 180 seconds is chosen because the cloud function that processes
// uploaded assets has a timeout of 60 seconds and uploading can take some time on a slow connection.
// - Each time one or more assets reports as ready, reset the timeout.
// - Each time an asset upload begins, reset the timeout. This includes retries.
// - Each time an asset upload begins or sends more bytes, reset the timeout. This includes retries.
// - Start upload. Internally, uploadAssetsAsync uploads them all first and then checks for successful
// processing every (5 + n) seconds with a linear backoff of n + 1 second.
// - At the same time as upload is started, start timeout checker which checks every 1 second to see
// if timeout has been reached. When timeout expires, send a cancellation signal to currently running
// upload function call to instruct it to stop uploading or checking for successful processing.
const NO_ACTIVITY_TIMEOUT = 180 * 1000; // 180 seconds
let uploadCountText = 'Uploading...';
let activeUploadLines: string[] = [];
let unfinishedAssetPaths: string[] = [];
const fullySentAssetPaths = new Set<string>();
const updateAssetSpinnerText = (): void => {
const processingAssetNames = unfinishedAssetPaths
.filter(assetPath => fullySentAssetPaths.has(assetPath))
.map(assetPath => path.basename(assetPath));
assetSpinner.text = [
uploadCountText,
...activeUploadLines,
...(processingAssetNames.length > 0
? [
`Waiting for the server to process ${processingAssetNames.slice(0, 3).join(', ')}` +
(processingAssetNames.length > 3
? ` and ${processingAssetNames.length - 3} more`
: ''),
]
: []),
].join('\n');
};
const reportUploadProgress = createUploadProgressReporter({
isInteractive: isSpinnerEnabled(),
onActiveUploadsChanged: lines => {
activeUploadLines = lines;
updateAssetSpinnerText();
},
});
let lastUploadedStorageKeys = new Set<string>();
let lastAssetUploadResults: {
asset: RawAsset & { storageKey: string };
@@ -422,11 +452,23 @@ export default class UpdatePublish extends EasCommand {
const totalAssets = assetUploadResults.length;
const missingAssetCount = assetUploadResults.filter(a => !a.finished).length;
assetSpinner.text = `Uploading (${totalAssets - missingAssetCount}/${totalAssets})`;
uploadCountText = `Uploading (${totalAssets - missingAssetCount}/${totalAssets})`;
unfinishedAssetPaths = assetUploadResults
.filter(r => !r.finished)
.map(r => r.asset.path);
updateAssetSpinnerText();
},
() => {
progress => {
// when an upload is retried, reset the timeout as we know this will now need more time
timeAtWhichToTimeout = Date.now() + NO_ACTIVITY_TIMEOUT; // reset timeout to NO_ACTIVITY_TIMEOUT
if (progress) {
if (progress.uploadedBytes >= progress.fileSize) {
fullySentAssetPaths.add(progress.file);
} else {
fullySentAssetPaths.delete(progress.file);
}
reportUploadProgress(progress);
}
}
),
(async () => {
@@ -9,7 +9,7 @@ import { promptAsync } from '../../prompts';
export default class WorkflowRunCancel extends EasCommand {
static override description =
'Cancel one or more workflow runs. If no workflow run IDs are provided, you will be prompted to select IN_PROGRESS runs to cancel.';
'cancel one or more workflow runs. If no workflow run IDs are provided, you will be prompted to select IN_PROGRESS runs to cancel';
static override strict = false;
@@ -9,7 +9,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
export default class WorkflowList extends EasCommand {
static override hidden = true;
static override description = 'List workflows for the current project';
static override description = 'list workflows for the current project';
static override flags = {
...EasJsonOnlyFlag,
@@ -26,7 +26,7 @@ function printLogsForAllSteps(logs: JobLogs): void {
export default class WorkflowLogView extends EasCommand {
static override description =
'view logs for a workflow run, selecting a job and step to view. You can pass in either a workflow run ID or a job ID. If no ID is passed in, you will be prompted to select from recent workflow runs for the current project.';
'view logs for a workflow run, selecting a job and step to view. You can pass in either a workflow run ID or a job ID. If no ID is passed in, you will be prompted to select from recent workflow runs for the current project';
static override flags = {
...EasJsonOnlyFlag,
@@ -93,7 +93,7 @@ export function resolveWorkflowRunSshInput({
export default class WorkflowRun extends EasCommand {
static override description =
'run an EAS workflow. The entire local project directory will be packaged and uploaded to EAS servers for the workflow run, unless the --ref flag is used.';
'run an EAS workflow. The entire local project directory will be packaged and uploaded to EAS servers for the workflow run, unless the --ref flag is used';
static override args = {
file: Args.string({
@@ -32,7 +32,7 @@ import { enableJsonOutput, printJsonOnlyOutput } from '../../utils/json';
export default class WorkflowStatus extends EasCommand {
static override description =
'show the status of an existing workflow run. If no run ID is provided, you will be prompted to select from recent workflow runs for the current project.';
'show the status of an existing workflow run. If no run ID is provided, you will be prompted to select from recent workflow runs for the current project';
static override args = {
WORKFLOW_RUN_ID: Args.string({
@@ -59,7 +59,7 @@ const processedOutputs: (job: WorkflowJobResult) => FormatFieldsItem[] = job =>
export default class WorkflowView extends EasCommand {
static override description =
'view details for a workflow run, including jobs. If no run ID is provided, you will be prompted to select from recent workflow runs for the current project.';
'view details for a workflow run, including jobs. If no run ID is provided, you will be prompted to select from recent workflow runs for the current project';
static override flags = {
...EasJsonOnlyFlag,
@@ -0,0 +1,126 @@
import Log from '../../log';
import { UploadProgress } from '../../uploads';
import { createUploadProgressReporter } from '../uploadProgressReporter';
const MB = 1024 * 1024;
function makeProgress(file: string, uploadedBytes: number, elapsedMs: number): UploadProgress {
return { file, fileSize: 7 * MB, uploadedBytes, elapsedMs };
}
describe(createUploadProgressReporter, () => {
afterEach(() => {
jest.restoreAllMocks();
});
describe('interactive', () => {
it('shows a line for each active upload once it has run for 10 seconds', () => {
const onActiveUploadsChanged = jest.fn();
const report = createUploadProgressReporter({ isInteractive: true, onActiveUploadsChanged });
report(makeProgress('/dist/ios.hbc', MB, 5_000));
expect(onActiveUploadsChanged).toHaveBeenLastCalledWith([]);
report(makeProgress('/dist/ios.hbc', MB, 20_000));
report(makeProgress('/dist/android.hbc', 6 * MB, 10_000));
expect(onActiveUploadsChanged).toHaveBeenLastCalledWith([
'ios.hbc: 1.0 MB of 7.0 MB, 51.2 KB/s, about 2 minutes left',
'android.hbc: 6.0 MB of 7.0 MB, 614 KB/s',
]);
});
it('removes an upload once all of its bytes are sent', () => {
const onActiveUploadsChanged = jest.fn();
const report = createUploadProgressReporter({ isInteractive: true, onActiveUploadsChanged });
report(makeProgress('/dist/ios.hbc', MB, 20_000));
report(makeProgress('/dist/ios.hbc', 7 * MB, 30_000));
expect(onActiveUploadsChanged).toHaveBeenLastCalledWith([]);
});
it('shows at most 5 uploads', () => {
const onActiveUploadsChanged = jest.fn();
const report = createUploadProgressReporter({ isInteractive: true, onActiveUploadsChanged });
for (let i = 0; i < 7; i++) {
report(makeProgress(`/dist/asset-${i}.png`, MB, 20_000));
}
const lines = onActiveUploadsChanged.mock.calls.at(-1)?.[0];
expect(lines).toHaveLength(6);
expect(lines[5]).toBe('and 2 more uploads');
});
it('does not log', () => {
const warnSpy = jest.spyOn(Log, 'warn').mockImplementation(() => {});
const logSpy = jest.spyOn(Log, 'log').mockImplementation(() => {});
const report = createUploadProgressReporter({
isInteractive: true,
onActiveUploadsChanged: () => {},
});
report(makeProgress('/dist/ios.hbc', MB, 20_000));
expect(warnSpy).not.toHaveBeenCalled();
expect(logSpy).not.toHaveBeenCalled();
});
});
describe('non-interactive', () => {
it('warns when an upload is slow and then logs its progress every minute', () => {
let now = 0;
jest.spyOn(Date, 'now').mockImplementation(() => now);
const warnSpy = jest.spyOn(Log, 'warn').mockImplementation(() => {});
const logSpy = jest.spyOn(Log, 'log').mockImplementation(() => {});
const report = createUploadProgressReporter({
isInteractive: false,
onActiveUploadsChanged: () => {},
});
report(makeProgress('/dist/ios.hbc', MB, 20_000));
now = 30_000;
report(makeProgress('/dist/ios.hbc', 2 * MB, 50_000));
now = 60_000;
report(makeProgress('/dist/ios.hbc', 3 * MB, 80_000));
expect(warnSpy).toHaveBeenCalledTimes(1);
expect(warnSpy).toHaveBeenCalledWith(
'Uploading ios.hbc (7.0 MB) is slow (51.2 KB/s). It may take about 2 minutes.'
);
expect(logSpy).toHaveBeenCalledTimes(1);
expect(logSpy).toHaveBeenCalledWith(
'Still uploading ios.hbc: 3.0 MB of 7.0 MB, 38.4 KB/s, about 2 minutes left.'
);
});
it('warns again when a retried upload is still slow', () => {
const warnSpy = jest.spyOn(Log, 'warn').mockImplementation(() => {});
const report = createUploadProgressReporter({
isInteractive: false,
onActiveUploadsChanged: () => {},
});
report(makeProgress('/dist/ios.hbc', MB, 20_000));
report(makeProgress('/dist/ios.hbc', 0, 0));
report(makeProgress('/dist/ios.hbc', MB, 20_000));
expect(warnSpy).toHaveBeenCalledTimes(2);
});
it('does not log when an upload is fast', () => {
const warnSpy = jest.spyOn(Log, 'warn').mockImplementation(() => {});
const logSpy = jest.spyOn(Log, 'log').mockImplementation(() => {});
const report = createUploadProgressReporter({
isInteractive: false,
onActiveUploadsChanged: () => {},
});
report(makeProgress('/dist/ios.hbc', 6 * MB, 10_000));
expect(warnSpy).not.toHaveBeenCalled();
expect(logSpy).not.toHaveBeenCalled();
});
});
});
+3 -3
View File
@@ -44,7 +44,7 @@ import {
truncateString as truncateUpdateMessage,
updatePublishPlatformToAppPlatform,
} from '../update/utils';
import { PresignedPost, uploadWithPresignedPostWithRetryAsync } from '../uploads';
import { PresignedPost, UploadProgress, uploadWithPresignedPostWithRetryAsync } from '../uploads';
import {
expoCommandAsync,
shouldUseVersionedExpoCLI,
@@ -496,7 +496,7 @@ export async function uploadAssetsAsync(
onAssetUploadResultsChanged: (
assetUploadResults: { asset: RawAsset & { storageKey: string }; finished: boolean }[]
) => void,
onAssetUploadBegin: () => void
onAssetUploadBegin: (progress?: UploadProgress) => void
): Promise<AssetUploadResult> {
let assets: RawAsset[] = [];
let platform: keyof FilteredCollectedAssets;
@@ -552,7 +552,7 @@ export async function uploadAssetsAsync(
// Sign URLs in chunks (endpoint caps at 100), then upload all from one pool so it stays saturated.
const uploadBatchAsync = async (
assetsToUpload: AssetWithStorageKey[],
onBegin: () => void
onBegin: (progress?: UploadProgress) => void
): Promise<void> => {
const signedAssets: { asset: AssetWithStorageKey; specification: string }[] = [];
for (const assetChunk of chunk(assetsToUpload, 100)) {
@@ -0,0 +1,91 @@
import path from 'path';
import Log from '../log';
import { UploadProgress } from '../uploads';
import { formatBytes } from '../utils/files';
const MIN_UPLOAD_MS_BEFORE_REPORTING = 10_000;
const SLOW_UPLOAD_SECONDS_LEFT = 60;
const SLOW_UPLOAD_LOG_INTERVAL_MS = 60_000;
const MAX_ACTIVE_UPLOAD_LINES = 5;
function getBytesPerSecond(progress: UploadProgress): number {
return progress.uploadedBytes / (progress.elapsedMs / 1000);
}
function getSecondsLeft(progress: UploadProgress): number {
return (progress.fileSize - progress.uploadedBytes) / getBytesPerSecond(progress);
}
function formatMinutesLeft(progress: UploadProgress): string {
const minutes = Math.ceil(getSecondsLeft(progress) / 60);
return minutes === 1 ? '1 minute' : `${minutes} minutes`;
}
function formatUploadProgress(progress: UploadProgress): string {
const message =
`${path.basename(progress.file)}: ${formatBytes(progress.uploadedBytes)} of ${formatBytes(progress.fileSize)}, ` +
`${formatBytes(getBytesPerSecond(progress))}/s`;
return getSecondsLeft(progress) > SLOW_UPLOAD_SECONDS_LEFT
? `${message}, about ${formatMinutesLeft(progress)} left`
: message;
}
function isSlowUpload(progress: UploadProgress): boolean {
return (
progress.elapsedMs >= MIN_UPLOAD_MS_BEFORE_REPORTING &&
getSecondsLeft(progress) > SLOW_UPLOAD_SECONDS_LEFT
);
}
export function createUploadProgressReporter({
isInteractive,
onActiveUploadsChanged,
}: {
isInteractive: boolean;
onActiveUploadsChanged: (lines: string[]) => void;
}): (progress: UploadProgress) => void {
const activeUploads = new Map<string, UploadProgress>();
const lastSlowUploadLogTimes = new Map<string, number>();
return progress => {
if (progress.uploadedBytes === 0) {
lastSlowUploadLogTimes.delete(progress.file);
}
if (progress.uploadedBytes >= progress.fileSize) {
activeUploads.delete(progress.file);
} else {
activeUploads.set(progress.file, progress);
}
if (isInteractive) {
const lines = [...activeUploads.values()]
.filter(upload => upload.elapsedMs >= MIN_UPLOAD_MS_BEFORE_REPORTING)
.map(formatUploadProgress);
onActiveUploadsChanged(
lines.length > MAX_ACTIVE_UPLOAD_LINES
? [
...lines.slice(0, MAX_ACTIVE_UPLOAD_LINES),
`and ${lines.length - MAX_ACTIVE_UPLOAD_LINES} more uploads`,
]
: lines
);
return;
}
if (!isSlowUpload(progress)) {
return;
}
const lastLogTime = lastSlowUploadLogTimes.get(progress.file);
if (lastLogTime === undefined) {
Log.warn(
`Uploading ${path.basename(progress.file)} (${formatBytes(progress.fileSize)}) is slow ` +
`(${formatBytes(getBytesPerSecond(progress))}/s). It may take about ${formatMinutesLeft(progress)}.`
);
lastSlowUploadLogTimes.set(progress.file, Date.now());
} else if (Date.now() - lastLogTime >= SLOW_UPLOAD_LOG_INTERVAL_MS) {
Log.log(`Still uploading ${formatUploadProgress(progress)}.`);
lastSlowUploadLogTimes.set(progress.file, Date.now());
}
};
}
+20 -4
View File
@@ -20,6 +20,13 @@ export interface PresignedPost {
fields: Record<string, string>;
}
export interface UploadProgress {
file: string;
fileSize: number;
uploadedBytes: number;
elapsedMs: number;
}
export async function uploadFileAtPathToGCSAsync(
graphqlClient: ExpoGraphqlClient,
type: UploadSessionType,
@@ -88,7 +95,7 @@ export async function uploadAppScopedFileAtPathToGCSAsync(
export async function uploadWithPresignedPostWithRetryAsync(
file: string,
presignedPost: PresignedPost,
onAssetUploadBegin: () => void
onAssetUploadBegin: (progress?: UploadProgress) => void
): Promise<Response> {
return await promiseRetry(
async retry => {
@@ -96,7 +103,7 @@ export async function uploadWithPresignedPostWithRetryAsync(
let response: Response;
try {
onAssetUploadBegin();
response = await uploadWithPresignedPostAsync(file, presignedPost);
response = await uploadWithPresignedPostAsync(file, presignedPost, onAssetUploadBegin);
} catch (e: any) {
return retry(e);
}
@@ -127,7 +134,8 @@ export async function uploadWithPresignedPostWithRetryAsync(
async function uploadWithPresignedPostAsync(
file: string,
presignedPost: PresignedPost
presignedPost: PresignedPost,
onProgress: (progress: UploadProgress) => void
): Promise<Response> {
const fileStat = await fs.stat(file);
const fileSize = fileStat.size;
@@ -135,7 +143,15 @@ async function uploadWithPresignedPostAsync(
for (const [fieldKey, fieldValue] of Object.entries(presignedPost.fields)) {
form.append(fieldKey, fieldValue);
}
form.append('file', fs.createReadStream(file), { knownLength: fileSize });
const readStream = fs.createReadStream(file);
form.append('file', readStream, { knownLength: fileSize });
const startTime = Date.now();
let uploadedBytes = 0;
onProgress({ file, fileSize, uploadedBytes, elapsedMs: 0 });
readStream.addListener('data', (chunk: Buffer | string) => {
uploadedBytes += Buffer.byteLength(chunk);
onProgress({ file, fileSize, uploadedBytes, elapsedMs: Date.now() - startTime });
});
const formHeaders = form.getHeaders();
return await fetch(presignedPost.url, {
method: 'POST',