Compare commits

...
Author SHA1 Message Date
Cedric van Putten 149195ddc3 refactor(eas-cli): replace node-fetch with undici 2025-01-21 14:57:33 +01:00
Cedric van Putten ebe6848918 chore: bump minimum required Node version to 18.14.0
This version includes the `File` primitive that might be required for FormData and large file uploads.

See: https://nodejs.org/en/blog/release/v18.13.0\#introduce-file
See: https://github.com/expo/eas-cli/pull/2414\#discussion_r1634640527
2025-01-17 14:16:34 +01:00
13 changed files with 294 additions and 132 deletions

No files matched your search

+2
View File
@@ -6,6 +6,8 @@ This is the log of notable changes to EAS CLI and related packages.
### 🛠 Breaking changes
- Bump minimum required Node version to 18.14.0 to support `FormData` file streaming withi `undici`. ([#2420](https://github.com/expo/eas-cli/pull/2420) by [@byCedric](https://github.com/byCedric))
### 🎉 New features
- Allow filtering by `--fingerprint_hash` in `eas build:list` command. ([#2818](https://github.com/expo/eas-cli/pull/2818) by [@szdziedzic](https://github.com/szdziedzic))
+2 -4
View File
@@ -54,7 +54,6 @@
"gradle-to-js": "2.0.1",
"graphql": "16.8.1",
"graphql-tag": "2.12.6",
"https-proxy-agent": "5.0.1",
"ignore": "5.3.0",
"indent-string": "4.0.0",
"jks-js": "1.1.0",
@@ -66,7 +65,6 @@
"minimatch": "5.1.2",
"minizlib": "3.0.1",
"nanoid": "3.3.8",
"node-fetch": "2.6.7",
"node-forge": "1.3.1",
"nullthrows": "1.1.1",
"ora": "5.1.0",
@@ -84,6 +82,7 @@
"terminal-link": "2.1.1",
"tslib": "2.6.2",
"turndown": "7.1.2",
"undici": "^6.18.2",
"untildify": "4.0.0",
"uuid": "9.0.1",
"wrap-ansi": "7.0.0",
@@ -105,7 +104,6 @@
"@types/getenv": "^1.0.0",
"@types/jsonwebtoken": "8.5.1",
"@types/mime": "3.0.1",
"@types/node-fetch": "2.6.2",
"@types/node-forge": "1.3.1",
"@types/pngjs": "6.0.4",
"@types/promise-retry": "1.1.3",
@@ -129,7 +127,7 @@
"typescript": "5.3.3"
},
"engines": {
"node": ">=18.0.0"
"node": ">=18.14.0"
},
"files": [
"/bin",
@@ -11,10 +11,10 @@ import {
} from '@urql/core';
import { retryExchange } from '@urql/exchange-retry';
import { DocumentNode } from 'graphql';
import fetch from 'node-fetch';
import { fetch } from 'undici';
import { getExpoApiBaseUrl } from '../../../api';
import { httpsProxyAgent } from '../../../fetch';
import { sharedProxyAgent } from '../../../fetch';
export interface ExpoGraphqlClient extends Client {
query<Data = any, Variables extends AnyVariables = AnyVariables>(
@@ -54,7 +54,7 @@ export function createGraphqlClient(authInfo: {
headers['expo-session'] = authInfo.sessionSecret;
}
return {
...(httpsProxyAgent ? { agent: httpsProxyAgent } : {}),
...(sharedProxyAgent ? { agent: sharedProxyAgent } : {}),
headers,
};
},
@@ -175,7 +175,7 @@ export default class WorkerDeploy extends EasCommand {
} else if (!response.ok) {
throw new Error(`Upload failed! (${response.statusText})`);
} else {
const json = await response.json();
const json: any = await response.json(); // TODO: remove any
if (!json.success || !json.result || typeof json.result !== 'object') {
throw new Error(json.message ? `Upload failed: ${json.message}` : 'Upload failed!');
}
+51 -25
View File
@@ -1,8 +1,55 @@
import { Agent } from 'https';
import createHttpsProxyAgent from 'https-proxy-agent';
import fetch, { RequestInfo, RequestInit, Response } from 'node-fetch';
import { env } from 'node:process';
import { EnvHttpProxyAgent, fetch } from 'undici';
export * from 'node-fetch';
export {
Agent,
Dispatcher,
FormData,
Headers,
HeadersInit,
Request,
RequestInit,
Response,
ResponseInit,
} from 'undici';
/**
* Create a proxy agent when `http_proxy`, `https_proxy`, or uppercase equivalents environment variables are defined.
* This agent is built-in with Undici and can be used as global or per-fetch dispatchers.
*/
function createProxyAgent(): EnvHttpProxyAgent | null {
// See: https://github.com/nodejs/undici/blob/v6.21.1/docs/docs/api/EnvHttpProxyAgent.md
const httpProxy = env.http_proxy ?? env.HTTP_PROXY;
const httpsProxy = env.https_proxy ?? env.HTTPS_PROXY;
return httpProxy || httpsProxy ? new EnvHttpProxyAgent({ httpProxy, httpsProxy }) : null;
}
export const sharedProxyAgent = createProxyAgent();
/**
* Wrap the `fetch` method and throw a `RequestError` whenever response status `>=400` are encountered.
* Normally, fetch does not throw on response errors.
*/
function wrapFetchWithRequestError(fetchRef: typeof fetch): typeof fetch {
return async (url, init = {}) => {
if (sharedProxyAgent && !init.dispatcher) {
init.dispatcher = sharedProxyAgent;
}
const response = await fetchRef(url, init);
if (response.status >= 400) {
throw new RequestError(
`Request failed: ${response.status} (${response.statusText})`,
response
);
}
return response;
};
}
export default wrapFetchWithRequestError(fetch);
export class RequestError extends Error {
constructor(
@@ -12,24 +59,3 @@ export class RequestError extends Error {
super(message);
}
}
function createHttpsAgent(): Agent | null {
const httpsProxyUrl = process.env.https_proxy;
if (!httpsProxyUrl) {
return null;
}
return createHttpsProxyAgent(httpsProxyUrl);
}
export const httpsProxyAgent: Agent | null = createHttpsAgent();
export default async function (url: RequestInfo, init?: RequestInit): Promise<Response> {
const response = await fetch(url, {
...init,
...(httpsProxyAgent ? { agent: httpsProxyAgent } : {}),
});
if (response.status >= 400) {
throw new RequestError(`Request failed: ${response.status} (${response.statusText})`, response);
}
return response;
}
+1 -2
View File
@@ -1,10 +1,9 @@
import FormData from 'form-data';
import fs from 'fs-extra';
import { Response } from 'node-fetch';
import promiseRetry from 'promise-retry';
import { ExpoGraphqlClient } from './commandUtils/context/contextUtils/createGraphqlClient';
import fetch from './fetch';
import fetch, { Response } from './fetch';
import { AccountUploadSessionType, UploadSessionType } from './graphql/generated';
import { SignedUrl, UploadSessionMutation } from './graphql/mutations/UploadSessionMutation';
import { ProgressHandler } from './utils/progress';
@@ -0,0 +1,104 @@
import nock from 'nock';
import { getExpoApiBaseUrl } from '../../api';
import { RequestError } from '../../fetch';
import { wrapFetchWithProgress } from '../download';
describe(wrapFetchWithProgress, () => {
const url = getExpoApiBaseUrl();
it('returns response with body', async () => {
nock(url)
.get('/test-body')
.reply(200, 'success', { 'Content-Length': String(Buffer.byteLength('success')) });
const response = await wrapFetchWithProgress()(`${url}/test-body`, {}, jest.fn());
expect(await response.text()).toBe('success');
});
it('calls progress handler when loading body', async () => {
const testSize = 1024 * 1024; // 1MB
nock(url)
.get('/test-progress')
.reply(200, Buffer.alloc(testSize), { 'Content-Length': String(testSize) });
const progressTracker = jest.fn();
const fetchWithProgress = wrapFetchWithProgress();
const response = await fetchWithProgress(`${url}/test-progress`, {}, progressTracker);
// Response should be successful
expect(response).toMatchObject({ ok: true });
// Load the the response body to trigger the progress events
expect(await response.blob()).not.toBeUndefined();
// Progress tracker should start at 0%
expect(progressTracker).toHaveBeenCalledWith({
isComplete: false,
progress: {
total: testSize,
percent: 0,
transferred: 0,
},
});
// Progress tracker should end at 100%
expect(progressTracker).toHaveBeenCalledWith({
isComplete: true,
progress: {
total: testSize,
percent: 1,
transferred: testSize,
},
});
});
it('skips progress events when request fails', async () => {
nock(url)
.get('/test-fail')
.reply(404, 'Not Found', { 'Content-Length': String(Buffer.byteLength('Not Found')) });
const progressTracker = jest.fn();
const response = await wrapFetchWithProgress()(`${url}/test-fail`, {}, progressTracker).catch(
(requestError: RequestError) => requestError.response
);
// Response should not be successful
expect(response).toMatchObject({ ok: false });
// Repsonse should contain our error message
expect(await response.text()).toBe('Not Found');
// No progression events should be called
expect(progressTracker).not.toHaveBeenCalled();
});
it('skips progress events when response is empty', async () => {
nock(url).get('/test-empty').reply(204, undefined, { 'Content-Length': '0' });
const progressTracker = jest.fn();
const response = await wrapFetchWithProgress()(`${url}/test-empty`, {}, progressTracker);
// Response should be successful
expect(response).toMatchObject({ ok: true });
// Body should be empty
expect(await response.text()).toBe('');
// No progression events should be called
expect(progressTracker).not.toHaveBeenCalled();
});
it('skips progress events when no content-length header is available', async () => {
nock(url).get('/test-missing-content-length').reply(200, 'success');
const progressTracker = jest.fn();
const response = await wrapFetchWithProgress()(
`${url}/test-missing-content-length`,
{},
progressTracker
);
// Response should be successful
expect(response).toMatchObject({ ok: true });
// Body should be empty
expect(await response.text()).toBe('success');
// No progression events should be called
expect(progressTracker).not.toHaveBeenCalled();
});
});
+70 -43
View File
@@ -1,6 +1,8 @@
import spawnAsync from '@expo/spawn-async';
import glob from 'fast-glob';
import fs from 'fs-extra';
import { Readable } from 'node:stream';
import { ReadableStream } from 'node:stream/web';
import path from 'path';
import { Stream } from 'stream';
import { extract } from 'tar';
@@ -10,14 +12,15 @@ import { v4 as uuidv4 } from 'uuid';
import { formatBytes } from './files';
import { getTmpDirectory } from './paths';
import { ProgressHandler, createProgressTracker } from './progress';
import fetch, { RequestInit, Response } from '../fetch';
import fetch, { type RequestInit, Response } from '../fetch';
import { AppPlatform } from '../graphql/generated';
import Log from '../log';
import { promptAsync } from '../prompts';
const pipeline = promisify(Stream.pipeline);
function wrapFetchWithProgress(): (
/** @internal - Exposed for testing only */
export function wrapFetchWithProgress(): (
url: string,
init: RequestInit,
progressHandler: ProgressHandler
@@ -26,47 +29,68 @@ function wrapFetchWithProgress(): (
return async (url: string, init: RequestInit, progressHandler: ProgressHandler) => {
const response = await fetch(url, init);
if (response.ok) {
const totalDownloadSize = response.headers.get('Content-Length');
const total = Number(totalDownloadSize);
if (!totalDownloadSize || isNaN(total) || total < 0) {
Log.warn(
'Progress callback not supported for network request because "Content-Length" header missing or invalid in response from URL:',
url.toString()
);
return response;
}
let length = 0;
const onProgress = (chunkLength?: number): void => {
if (chunkLength) {
length += chunkLength;
}
const progress = length / total;
if (!didProgressBarFinish) {
progressHandler({
progress: { total, percent: progress, transferred: length },
isComplete: total === length,
});
if (total === length) {
didProgressBarFinish = true;
}
}
};
response.body.on('data', chunk => {
onProgress(chunk.length);
});
response.body.on('end', () => {
onProgress();
});
// Abort if the request failed, or if no response body is available
if (!response.ok || !response.body) {
return response;
}
return response;
// Calculate total progress size
const contentLength = response.headers.get('Content-Length');
const progressTotal = contentLength ? parseInt(contentLength, 10) : undefined;
if (!progressTotal || isNaN(progressTotal) || progressTotal < 0) {
// TODO: add debug logging explaining it bailed out due to faulty server response
return response;
}
// Prepare progress coutner and handler
let progressLength = 0;
const onProgress = (chunkSize = 0): void => {
progressLength += chunkSize;
if (!didProgressBarFinish) {
if (progressLength === progressTotal) {
didProgressBarFinish = true;
}
progressHandler({
isComplete: progressLength === progressTotal,
progress: {
total: progressTotal,
percent: progressLength / progressTotal,
transferred: progressLength,
},
});
}
};
// Create a new body-wrapping stream that monitors progression
const bodyReader = response.body.getReader();
const bodyWithProgress = new ReadableStream({
start(controller) {
// Notify the stream is starting, and read the first chunk
onProgress();
next();
function next(): void {
void bodyReader.read().then(({ done, value }) => {
// Close the controller when the stream is done
if (done) {
controller.close();
return;
}
// Update progression
onProgress(Buffer.byteLength(value));
// Continue the stream and read the next chunk
controller.enqueue(value);
next();
});
}
},
});
return new Response(bodyWithProgress, response);
};
}
@@ -82,7 +106,7 @@ async function downloadFileWithProgressTrackerAsync(
const response = await wrapFetchWithProgress()(
url,
{
timeout: 1000 * 60 * 5, // 5 minutes
signal: AbortSignal.timeout(1000 * 60 * 5), // 5 minutes
},
createProgressTracker({
message: progressTrackerMessage,
@@ -93,8 +117,11 @@ async function downloadFileWithProgressTrackerAsync(
if (!response.ok) {
throw new Error(`Failed to download file from ${url}`);
}
if (!response.body) {
throw new Error(`Failed to download file, no response body returned ${url}`);
}
await pipeline(response.body, fs.createWriteStream(outputPath));
await pipeline(Readable.fromWeb(response.body), fs.createWriteStream(outputPath));
} catch (error: any) {
if (await fs.pathExists(outputPath)) {
await fs.remove(outputPath);
+6 -1
View File
@@ -1,4 +1,5 @@
import fs from 'fs-extra';
import { Readable } from 'node:stream';
import { Metadata, PNG } from 'pngjs';
import fetch from '../fetch';
@@ -72,7 +73,11 @@ export async function isPNGAsync(imagePathOrURL: string): Promise<boolean> {
async function getImageStreamAsync(imagePathOrURL: string): Promise<NodeJS.ReadableStream> {
if (isURL(imagePathOrURL)) {
const response = await fetch(imagePathOrURL);
return response.body;
if (!response.body) {
throw new Error(`Image request did not return a response body`);
}
return Readable.fromWeb(response.body);
} else {
return fs.createReadStream(imagePathOrURL);
}
+47 -26
View File
@@ -1,11 +1,20 @@
import * as https from 'https';
import createHttpsProxyAgent from 'https-proxy-agent';
import mime from 'mime';
import { Gzip } from 'minizlib';
import fetch, { Headers, HeadersInit, RequestInit, Response } from 'node-fetch';
import fs, { createReadStream } from 'node:fs';
import path from 'node:path';
import { Readable } from 'node:stream';
import promiseRetry from 'promise-retry';
import {
Agent,
type Dispatcher,
Headers,
type HeadersInit,
type RequestInit,
type Response,
fetch,
} from 'undici';
import { sharedProxyAgent } from '../fetch';
const MAX_RETRIES = 4;
const MAX_CONCURRENCY = 10;
@@ -43,21 +52,33 @@ export interface UploadResult {
response: Response;
}
let sharedAgent: https.Agent | undefined;
const getAgent = (): https.Agent => {
if (sharedAgent) {
return sharedAgent;
} else if (process.env.https_proxy) {
return (sharedAgent = createHttpsProxyAgent(process.env.https_proxy));
let sharedDispatcher: Dispatcher | null = null;
const getDispatcher = (): Dispatcher => {
if (sharedDispatcher) {
return sharedDispatcher;
} else if (sharedProxyAgent) {
return (sharedDispatcher = sharedProxyAgent);
} else {
return (sharedAgent = new https.Agent({
keepAlive: true,
maxSockets: MAX_CONCURRENCY,
maxTotalSockets: MAX_CONCURRENCY,
scheduling: 'lifo',
timeout: 4_000,
}));
return (sharedDispatcher = new Agent({ pipelining: 0 }));
}
// if (sharedAgent) return sharedAgent;
// if (sharedProxyAgent) return (sharedAgent = sharedProxyAgent);
// return (sharedAgent = )
// if (sharedAgent) {
// return sharedAgent;
// } else if (process.env.https_proxy) {
// return (sharedAgent = createHttpsProxyAgent(process.env.https_proxy));
// } else {
// return (sharedAgent = new https.Agent({
// keepAlive: true,
// maxSockets: MAX_CONCURRENCY,
// maxTotalSockets: MAX_CONCURRENCY,
// scheduling: 'lifo',
// timeout: 4_000,
// }));
// }
};
export async function uploadAsync(params: UploadParams): Promise<UploadResult> {
@@ -85,12 +106,12 @@ export async function uploadAsync(params: UploadParams): Promise<UploadResult> {
headers.set('content-type', contentType);
}
let bodyStream: NodeJS.ReadableStream = createReadStream(filePath);
let bodyStream = createReadStream(filePath);
if (compress && isCompressible(contentType, stat.size)) {
const gzip = new Gzip({ portable: true });
bodyStream.on('error', error => gzip.emit('error', error));
// @ts-expect-error: Gzip implements a Readable-like interface
bodyStream = bodyStream.pipe(gzip) as NodeJS.ReadableStream;
// @ts-expect-error
bodyStream = bodyStream.pipe(gzip); // TODO: validate this works!
headers.set('content-encoding', 'gzip');
}
@@ -99,18 +120,18 @@ export async function uploadAsync(params: UploadParams): Promise<UploadResult> {
response = await fetch(params.url, {
...requestInit,
method,
body: bodyStream,
body: Readable.toWeb(bodyStream),
headers,
agent: getAgent(),
// @ts-expect-error: Internal types don't match
dispatcher: getDispatcher(),
signal,
duplex: 'half',
});
} catch (error) {
return retry(error);
}
const getErrorMessageAsync = async (): Promise<string> => {
const body = await response.json().catch(() => null);
const body: any = await response.json().catch(() => null); // TODO: remove any
return body?.error ?? `Upload of "${filePath}" failed: ${response.statusText}`;
};
@@ -148,18 +169,18 @@ export async function callUploadApiAsync(url: string | URL, init?: RequestInit):
try {
response = await fetch(url, {
...init,
agent: getAgent(),
dispatcher: getDispatcher(),
});
} catch (error) {
return retry(error);
}
if (response.status >= 500 && response.status <= 599) {
retry(new Error(`Deployment failed: ${response.statusText}`));
return retry(new Error(`Deployment failed: ${response.statusText}`));
}
try {
return await response.json();
} catch (error) {
retry(error);
return retry(error);
}
});
}
+1 -1
View File
@@ -26,7 +26,7 @@
"typescript": "5.3.3"
},
"engines": {
"node": ">=18.0.0"
"node": ">=18.14.0"
},
"homepage": "https://github.com/expo/eas-cli",
"license": "MIT",
+1 -1
View File
@@ -15,7 +15,7 @@
"author": "Expo <[email protected]>",
"license": "MIT",
"engines": {
"node": ">=18.0.0"
"node": ">=18.14.0"
},
"dependencies": {
"chalk": "5.2.0",
Generated
+5 -25
View File
@@ -3821,14 +3821,6 @@
resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.0.tgz#69a23a3ad29caf0097f06eda59b361ee2f0639f6"
integrity sha1-aaI6OtKcrwCX8G7aWbNh7i8GOfY=
"@types/[email protected]":
version "2.6.2"
resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.2.tgz#d1a9c5fd049d9415dce61571557104dec3ec81da"
integrity sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==
dependencies:
"@types/node" "*"
form-data "^3.0.0"
"@types/[email protected]":
version "1.3.1"
resolved "https://registry.yarnpkg.com/@types/node-forge/-/node-forge-1.3.1.tgz#49e44432c306970b4e900c3b214157c480af19fa"
@@ -7340,15 +7332,6 @@ [email protected], form-data@^4.0.0:
combined-stream "^1.0.8"
mime-types "^2.1.12"
form-data@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682"
integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==
dependencies:
asynckit "^0.4.0"
combined-stream "^1.0.8"
mime-types "^2.1.12"
[email protected]:
version "0.2.0"
resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811"
@@ -8224,14 +8207,6 @@ http2-wrapper@^1.0.0-beta.5.2:
quick-lru "^5.1.1"
resolve-alpn "^1.0.0"
[email protected]:
version "5.0.1"
resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6"
integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==
dependencies:
agent-base "6"
debug "4"
https-proxy-agent@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2"
@@ -14165,6 +14140,11 @@ undici-types@~5.26.4:
resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
undici@^6.18.2:
version "6.21.1"
resolved "https://registry.yarnpkg.com/undici/-/undici-6.21.1.tgz#336025a14162e6837e44ad7b819b35b6c6af0e05"
integrity sha512-q/1rj5D0/zayJB2FraXdaWxbhWiNKDvu8naDT2dl1yTlvJp4BLtOcp2a5BvgGNQpYYJzau7tf1WgKv3b+7mqpQ==
unique-filename@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230"