Compare commits

...
11 changed files with 436 additions and 9 deletions

No files matched your search

+2
View File
@@ -8,6 +8,8 @@ This is the log of notable changes to EAS CLI and related packages.
### 🎉 New features
- Add support for EAS Update asset excludes. ([#1526](https://github.com/expo/eas-cli/pull/1526) by [@wschurman](https://github.com/wschurman))
### 🐛 Bug fixes
### 🧹 Chores
+20
View File
@@ -23267,6 +23267,26 @@
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "excludedAssets",
"description": "",
"type": {
"kind": "LIST",
"name": null,
"ofType": {
"kind": "NON_NULL",
"name": null,
"ofType": {
"kind": "INPUT_OBJECT",
"name": "PartialManifestAsset",
"ofType": null
}
}
},
"defaultValue": null,
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "gitCommitHash",
"description": "",
+39 -4
View File
@@ -15,6 +15,7 @@ import { EasNonInteractiveAndJsonFlags } from '../../commandUtils/flags';
import { getPaginatedQueryOptions } from '../../commandUtils/pagination';
import fetch from '../../fetch';
import {
PartialManifestAsset,
PublishUpdateGroupInput,
StatuspageServiceName,
Update,
@@ -32,6 +33,7 @@ import {
buildBundlesAsync,
buildUnsortedUpdateInfoGroupAsync,
collectAssetsAsync,
convertAssetToUpdateInfoGroupFormatAsync,
filterExportedPlatformsByFlag,
isUploadedAssetCountAboveWarningThreshold,
resolveInputDirectoryAsync,
@@ -246,6 +248,7 @@ export default class UpdatePublish extends EasCommand {
let unsortedUpdateInfoGroups: UpdateInfoGroup = {};
let oldMessage: string, oldRuntimeVersion: string;
let uploadedAssetCount = 0;
let excludedAssets: PartialManifestAsset[] = [];
let assetLimitPerUpdateGroup = 0;
if (republish) {
@@ -376,6 +379,7 @@ export default class UpdatePublish extends EasCommand {
const uploadResults = await uploadAssetsAsync(
graphqlClient,
assets,
projectDir,
projectId,
(totalAssets, missingAssets) => {
assetSpinner.text = `Uploading (${totalAssets - missingAssets}/${totalAssets})`;
@@ -383,12 +387,42 @@ export default class UpdatePublish extends EasCommand {
);
uploadedAssetCount = uploadResults.uniqueUploadedAssetCount;
excludedAssets = await Promise.all(
uploadResults.excludedAssets.map(convertAssetToUpdateInfoGroupFormatAsync)
);
assetLimitPerUpdateGroup = uploadResults.assetLimitPerUpdateGroup;
unsortedUpdateInfoGroups = await buildUnsortedUpdateInfoGroupAsync(assets, exp);
const uploadAssetSuccessMessage = uploadedAssetCount
? `Uploaded ${uploadedAssetCount} ${uploadedAssetCount === 1 ? 'platform' : 'platforms'}`
: `Uploaded: No changes detected`;
assetSpinner.succeed(uploadAssetSuccessMessage);
// NOTE(cedric): we assume that bundles are always uploaded, and always are part of
// `uploadedAssetCount`, perferably we don't assume. For that, we need to refactor the
// `uploadAssetsAsync` and be able to determine asset type from the uploaded assets.
const uploadedBundleCount = uploadResults.launchAssetCount;
const uploadedNormalAssetCount = Math.max(0, uploadedAssetCount - uploadedBundleCount);
const reusedNormalAssetCount = uploadResults.uniqueAssetCount - uploadedNormalAssetCount;
assetSpinner.stop();
Log.withTick(
`Uploaded ${uploadedBundleCount} app ${uploadedBundleCount === 1 ? 'bundle' : 'bundles'}`
);
if (excludedAssets.length > 0) {
Log.withTick(`Excluded ${excludedAssets.length} assets from upload`);
}
if (uploadedNormalAssetCount === 0) {
Log.withTick(`Uploading assets skipped - no new assets found`);
} else {
let message = `Uploaded ${uploadedNormalAssetCount} ${
uploadedNormalAssetCount === 1 ? 'asset' : 'assets'
}`;
if (reusedNormalAssetCount > 0) {
message += ` (reused ${reusedNormalAssetCount} ${
reusedNormalAssetCount === 1 ? 'asset' : 'assets'
})`;
}
Log.withTick(message);
}
for (const uploadedAssetPath of uploadResults.uniqueUploadedAssetPaths) {
Log.debug(chalk.dim(`- ${uploadedAssetPath}`));
}
} catch (e) {
assetSpinner.fail('Failed to upload');
throw e;
@@ -450,6 +484,7 @@ export default class UpdatePublish extends EasCommand {
message: truncatedMessage,
gitCommitHash,
awaitingCodeSigningInfo: !!codeSigningInfo,
excludedAssets,
};
}
);
+1
View File
@@ -3347,6 +3347,7 @@ export type PublicArtifacts = {
export type PublishUpdateGroupInput = {
awaitingCodeSigningInfo?: InputMaybe<Scalars['Boolean']>;
branchId: Scalars['String'];
excludedAssets?: InputMaybe<Array<PartialManifestAsset>>;
gitCommitHash?: InputMaybe<Scalars['String']>;
message?: InputMaybe<Scalars['String']>;
runtimeVersion: Scalars['String'];
@@ -1,3 +1,4 @@
import { EasJsonUtils } from '@expo/eas-json';
import fs from 'fs-extra';
import mockdate from 'mockdate';
import path from 'path';
@@ -16,7 +17,9 @@ import {
convertAssetToUpdateInfoGroupFormatAsync,
filterExportedPlatformsByFlag,
filterOutAssetsThatAlreadyExistAsync,
getAssetHashFromPath,
getBase64URLEncoding,
getOriginalPathFromAssetMap,
getStorageKey,
getStorageKeyForAssetAsync,
guessContentTypeFromExtension,
@@ -26,6 +29,17 @@ import {
jest.mock('../../uploads');
jest.mock('fs');
jest.mock('@expo/eas-json', () => {
const actual = jest.requireActual('@expo/eas-json');
const EasJsonUtilsMock = {
getUpdatesConfigAsync: jest.fn(),
};
return {
...actual,
EasJsonUtils: EasJsonUtilsMock,
};
});
const dummyFileBuffer = Buffer.from('dummy-file');
@@ -279,9 +293,55 @@ describe(resolveInputDirectoryAsync, () => {
});
});
describe(getAssetHashFromPath, () => {
it('returns asset hash from path', () => {
expect(getAssetHashFromPath('assets/5b2a819c71d035ca45d223e4c47ed4f9')).toBe(
'5b2a819c71d035ca45d223e4c47ed4f9'
);
});
it('returns null for incorrect path', () => {
expect(getAssetHashFromPath('assets/this/is/not/a/hash.jpg')).toBeNull();
});
});
describe(getOriginalPathFromAssetMap, () => {
// Partial assetmap.json, with only fields we need
const fakeAssetMap = {
'5b2a819c71d035ca45d223e4c47ed4f9': {
httpServerLocation: '/assets/src/assets',
name: 'asset-420',
type: 'jpg',
},
};
it('returns null path when asset map is null', () => {
expect(
getOriginalPathFromAssetMap(null, {
path: 'assets/5b2a819c71d035ca45d223e4c47ed4f9',
ext: 'jpg',
})
).toBeNull();
});
it('returns null when asset is not found in asset map', () => {
expect(
getOriginalPathFromAssetMap(fakeAssetMap, {
path: 'assets/fb64d3b2fb71b3d739ad5c13a93e12c5',
ext: 'jpg',
})
).toBeNull();
});
it('returns reconstructed original path from existing asset in asset map', () => {
expect(
getOriginalPathFromAssetMap(fakeAssetMap, {
path: 'assets/5b2a819c71d035ca45d223e4c47ed4f9',
ext: 'jpg',
})
).toBe('/src/assets/asset-420.jpg');
});
});
describe(collectAssetsAsync, () => {
it('builds an update info group', async () => {
const fakeHash = 'md5-hash-of-jpg';
const fakeHash = 'hdahukw8adhawi8fawhfa8';
const bundles = {
android: 'android-bundle-code',
ios: 'ios-bundle-code',
@@ -294,6 +354,7 @@ describe(collectAssetsAsync, () => {
fileExtension: '.jpg',
contentType: 'image/jpeg',
path: `${inputDir}/assets/${fakeHash}`,
originalPath: `assets/wat.jpg`,
},
];
@@ -326,6 +387,24 @@ describe(collectAssetsAsync, () => {
},
})
);
await fs.writeFile(
path.resolve(inputDir, 'assetmap.json'),
JSON.stringify({
[fakeHash]: {
__packager_asset: true,
fileSystemLocation: '/Users/blah/temp/assets',
httpServerLocation: 'assets/assets',
width: 2339,
height: 1560,
scales: [1],
files: ['/Users/blah/temp/assets/wat.jpg'],
hash: fakeHash,
name: 'wat',
type: 'jpg',
fileHashes: [fakeHash],
},
})
);
expect(await collectAssetsAsync(inputDir)).toEqual({
android: {
@@ -410,6 +489,7 @@ describe(uploadAssetsAsync, () => {
const androidBundlePath = uuidv4();
const iosBundlePath = uuidv4();
const dummyFilePath = uuidv4();
const dummyOriginalFilePath = uuidv4();
const userDefinedPath = uuidv4();
const testProjectId = uuidv4();
const expectedAssetLimit = 1400;
@@ -427,7 +507,15 @@ describe(uploadAssetsAsync, () => {
contentType: 'application/javascript',
path: androidBundlePath,
},
assets: [userDefinedAsset, { type: 'jpg', contentType: 'image/jpeg', path: dummyFilePath }],
assets: [
userDefinedAsset,
{
type: 'jpg',
contentType: 'image/jpeg',
path: dummyFilePath,
originalPath: dummyOriginalFilePath,
},
],
},
ios: {
launchAsset: {
@@ -435,7 +523,15 @@ describe(uploadAssetsAsync, () => {
contentType: 'application/javascript',
path: androidBundlePath,
},
assets: [userDefinedAsset, { type: 'jpg', contentType: 'image/jpeg', path: dummyFilePath }],
assets: [
userDefinedAsset,
{
type: 'jpg',
contentType: 'image/jpeg',
path: dummyFilePath,
originalPath: dummyOriginalFilePath,
},
],
},
};
@@ -486,12 +582,16 @@ describe(uploadAssetsAsync, () => {
});
mockdate.set(0);
const projectDir = path.resolve();
await expect(
uploadAssetsAsync(graphqlClient, assetsForUpdateInfoGroup, testProjectId)
uploadAssetsAsync(graphqlClient, assetsForUpdateInfoGroup, projectDir, testProjectId)
).resolves.toEqual({
assetCount: 6,
launchAssetCount: 2,
uniqueAssetCount: 3,
excludedAssets: [],
uniqueUploadedAssetCount: 0,
uniqueUploadedAssetPaths: [],
assetLimitPerUpdateGroup: expectedAssetLimit,
});
});
@@ -522,12 +622,16 @@ describe(uploadAssetsAsync, () => {
});
mockdate.set(0);
const projectDir = path.resolve();
await expect(
uploadAssetsAsync(graphqlClient, assetsForUpdateInfoGroup, testProjectId)
uploadAssetsAsync(graphqlClient, assetsForUpdateInfoGroup, projectDir, testProjectId)
).resolves.toEqual({
assetCount: 6,
launchAssetCount: 2,
uniqueAssetCount: 3,
excludedAssets: [],
uniqueUploadedAssetCount: 2,
uniqueUploadedAssetPaths: [],
assetLimitPerUpdateGroup: expectedAssetLimit,
});
});
@@ -560,9 +664,11 @@ describe(uploadAssetsAsync, () => {
const updateSpinnerFn = jest.fn((_totalAssets, _missingAssets) => {});
mockdate.set(0);
const projectDir = path.resolve();
await uploadAssetsAsync(
graphqlClient,
assetsForUpdateInfoGroup,
projectDir,
testProjectId,
updateSpinnerFn
);
@@ -574,3 +680,131 @@ describe(uploadAssetsAsync, () => {
]);
});
});
describe(uploadAssetsAsync, () => {
jest.spyOn(PublishMutation, 'getUploadURLsAsync').mockImplementation(async () => {
return { specifications: ['{}', '{}', '{}'] };
});
jest
.spyOn(PublishQuery, 'getAssetLimitPerUpdateGroupAsync')
.mockImplementation(async () => expectedAssetLimit);
const publishBundles = {
android: {
code: 'android bundle code',
assets: [{ files: ['md5-hash-of-file'], type: 'jpg' }],
map: 'dummy-string',
},
ios: {
code: 'ios bundle code',
assets: [{ files: ['md5-hash-of-file'], type: 'jpg' }],
map: 'dummy-string',
},
};
const androidBundlePath = uuidv4();
const iosBundlePath = uuidv4();
const dummyFilePath = `${uuidv4()}.jpg`; // this one should be excluded
const userDefinedPath = `${uuidv4()}.bundle`;
const testProjectId = uuidv4();
const expectedAssetLimit = 1400;
const userDefinedAsset = {
type: 'bundle',
contentType: 'application/octet-stream',
path: userDefinedPath,
originalPath: 'random.bundle',
};
const assetsForUpdateInfoGroup = {
android: {
launchAsset: {
type: 'bundle',
contentType: 'application/javascript',
path: androidBundlePath,
},
assets: [
userDefinedAsset,
{ type: 'jpg', contentType: 'image/jpeg', path: dummyFilePath, originalPath: 'test.jpg' },
],
},
ios: {
launchAsset: {
type: 'bundle',
contentType: 'application/javascript',
path: androidBundlePath,
},
assets: [
userDefinedAsset,
{ type: 'jpg', contentType: 'image/jpeg', path: dummyFilePath, originalPath: 'test.jpg' },
],
},
};
beforeAll(async () => {
await fs.writeFile(androidBundlePath, publishBundles.android.code);
await fs.writeFile(iosBundlePath, publishBundles.ios.code);
await fs.writeFile(dummyFilePath, dummyFileBuffer);
await fs.writeFile(userDefinedPath, 'I am an octet stream');
});
afterAll(async () => {
await fs.remove(androidBundlePath);
await fs.remove(iosBundlePath);
await fs.remove(dummyFilePath);
await fs.remove(userDefinedPath);
});
it('supports asset exclusion', async () => {
const graphqlClient = instance(mock<ExpoGraphqlClient>());
jest.spyOn(PublishQuery, 'getAssetMetadataAsync').mockImplementation(async () => {
const status = AssetMetadataStatus.Exists;
jest.runAllTimers();
return [
{
storageKey: 'qbgckgkgfdjnNuf9dQd7FDTWUmlEEzg7l1m1sKzQaq0',
status,
__typename: 'AssetMetadataResult',
}, // userDefinedAsset
{
storageKey: 'bbjgXFSIXtjviGwkaPFY0HG4dVVIGiXHAboRFQEqVa4',
status,
__typename: 'AssetMetadataResult',
}, // android.code
{
storageKey: 'dP-nC8EJXKz42XKh_Rc9tYxiGAT-ilpkRltEi6HIKeQ',
status,
__typename: 'AssetMetadataResult',
}, // ios.code
];
});
mockdate.set(0);
const projectDir = path.resolve();
jest.mocked(EasJsonUtils.getUpdatesConfigAsync).mockResolvedValue({
assetExcludePatterns: ['**/*.jpg'],
});
await expect(
uploadAssetsAsync(graphqlClient, assetsForUpdateInfoGroup, projectDir, testProjectId)
).resolves.toMatchObject({
assetCount: 4,
excludedAssets: [
{
contentType: 'image/jpeg',
originalPath: 'test.jpg',
type: 'jpg',
},
{
contentType: 'image/jpeg',
originalPath: 'test.jpg',
type: 'jpg',
},
],
launchAssetCount: 2,
uniqueAssetCount: 2,
assetLimitPerUpdateGroup: expectedAssetLimit,
});
});
});
+88
View File
@@ -1,9 +1,11 @@
import { ExpoConfig, Platform } from '@expo/config';
import { EasJsonAccessor, EasJsonUtils } from '@expo/eas-json';
import JsonFile from '@expo/json-file';
import crypto from 'crypto';
import fs from 'fs-extra';
import Joi from 'joi';
import mime from 'mime';
import minimatch from 'minimatch';
import path from 'path';
import promiseLimit from 'promise-limit';
@@ -15,6 +17,8 @@ import Log from '../log';
import { PresignedPost, uploadWithPresignedPostWithRetryAsync } from '../uploads';
import { expoCommandAsync, shouldUseVersionedExpoCLI } from '../utils/expoCli';
import chunk from '../utils/expodash/chunk';
import { truthy } from '../utils/expodash/filter';
import partition from '../utils/expodash/partition';
import uniqBy from '../utils/expodash/uniqBy';
export type ExpoCLIExportPlatformFlag = Platform | 'all';
@@ -30,6 +34,8 @@ export type RawAsset = {
fileExtension?: string;
contentType: string;
path: string;
/** Original asset path derrived from asset map, or exported folder */
originalPath?: string;
};
type CollectedAssets = {
[platform in Platform]?: {
@@ -51,6 +57,16 @@ type UpdateInfoGroup = {
[key in Platform]: ManifestFragment;
};
// Partial copy of `@expo/dev-server` `BundleAssetWithFileHashes`
type AssetMap = Record<
string,
{
httpServerLocation: string;
name: string;
type: string;
}
>;
const fileMetadataJoi = Joi.object({
assets: Joi.array()
.required()
@@ -169,6 +185,7 @@ export async function buildBundlesAsync({
'--output-dir',
inputDir,
'--dump-sourcemap',
'--dump-assetmap',
'--platform',
platformFlag,
]);
@@ -181,6 +198,7 @@ export async function buildBundlesAsync({
'--experimental-bundle',
'--non-interactive',
'--dump-sourcemap',
'--dump-assetmap',
'--platform',
platformFlag,
]);
@@ -245,9 +263,48 @@ export function filterExportedPlatformsByFlag<T extends Partial<Record<Platform,
return { [platform]: record[platform] } as T;
}
/** Try to load the asset map for logging the names of assets published */
export async function loadAssetMapAsync(distRoot: string): Promise<AssetMap | null> {
const assetMapPath = path.join(distRoot, 'assetmap.json');
if (!(await fs.pathExists(assetMapPath))) {
return null;
}
const assetMap: AssetMap = JsonFile.read(path.join(distRoot, 'assetmap.json'));
// TODO: basic validation?
return assetMap;
}
// exposed for testing
export function getAssetHashFromPath(assetPath: string): string | null {
const [, hash] = assetPath.match(new RegExp(/assets\/([a-z0-9]+)$/, 'i')) ?? [];
return hash ?? null;
}
// exposed for testing
export function getOriginalPathFromAssetMap(
assetMap: AssetMap | null,
asset: { path: string; ext: string }
): string | null {
if (!assetMap) {
return null;
}
const assetHash = getAssetHashFromPath(asset.path);
const assetMapEntry = assetHash && assetMap[assetHash];
if (!assetMapEntry) {
return null;
}
const pathPrefix = assetMapEntry.httpServerLocation.substring('/assets'.length);
return `${pathPrefix}/${assetMapEntry.name}.${assetMapEntry.type}`;
}
/** Given a directory, load the metadata.json and collect the assets for each platform. */
export async function collectAssetsAsync(dir: string): Promise<CollectedAssets> {
const metadata = loadMetadata(dir);
const assetmap = await loadAssetMapAsync(dir);
const collectedAssets: CollectedAssets = {};
@@ -260,6 +317,7 @@ export async function collectAssetsAsync(dir: string): Promise<CollectedAssets>
},
assets: metadata.fileMetadata[platform].assets.map(asset => ({
fileExtension: asset.ext ? ensureLeadingPeriod(asset.ext) : undefined,
originalPath: getOriginalPathFromAssetMap(assetmap, asset) ?? undefined,
contentType: guessContentTypeFromExtension(asset.ext),
path: path.join(dir, asset.path),
})),
@@ -293,21 +351,34 @@ export async function filterOutAssetsThatAlreadyExistAsync(
}
type AssetUploadResult = {
/** All found assets within the exported folder per platform */
assetCount: number;
/** The uploaded JS bundles, per platform */
launchAssetCount: number;
/** All unique assets within the exported folder with platforms combined */
uniqueAssetCount: number;
/** All unique assets uploaded */
uniqueUploadedAssetCount: number;
/** All (non-launch) asset original paths, used for logging */
uniqueUploadedAssetPaths: string[];
/** The asset limit received from the server */
assetLimitPerUpdateGroup: number;
/** The assets excluded from upload */
excludedAssets: RawAsset[];
};
export async function uploadAssetsAsync(
graphqlClient: ExpoGraphqlClient,
assetsForUpdateInfoGroup: CollectedAssets,
projectDir: string,
projectId: string,
updateSpinnerText?: (totalAssets: number, missingAssets: number) => void
): Promise<AssetUploadResult> {
let assets: RawAsset[] = [];
let platform: keyof CollectedAssets;
const launchAssets: RawAsset[] = [];
for (platform in assetsForUpdateInfoGroup) {
launchAssets.push(assetsForUpdateInfoGroup[platform]!.launchAsset);
assets = [
...assets,
assetsForUpdateInfoGroup[platform]!.launchAsset,
@@ -315,6 +386,19 @@ export async function uploadAssetsAsync(
];
}
const easJsonAccessor = new EasJsonAccessor(projectDir);
const updatesConfig = await EasJsonUtils.getUpdatesConfigAsync(easJsonAccessor);
const assetExcludePatterns = updatesConfig?.assetExcludePatterns;
let excludedAssets: RawAsset[] = [];
if (assetExcludePatterns) {
// some assets (like bundles) don't have original path so include those by default
[assets, excludedAssets] = partition(
assets,
({ originalPath }) =>
!originalPath || !assetExcludePatterns.some(pattern => minimatch(originalPath, pattern))
);
}
const assetsWithStorageKey = await Promise.all(
assets.map(async asset => {
return {
@@ -334,6 +418,7 @@ export async function uploadAssetsAsync(
updateSpinnerText?.(totalAssets, totalAssets);
let missingAssets = await filterOutAssetsThatAlreadyExistAsync(graphqlClient, uniqueAssets);
const uniqueUploadedAssetCount = missingAssets.length;
const uniqueUploadedAssetPaths = missingAssets.map(asset => asset.originalPath).filter(truthy);
const missingAssetChunks = chunk(missingAssets, 100);
const specifications: string[] = [];
@@ -371,8 +456,11 @@ export async function uploadAssetsAsync(
}
return {
assetCount: assets.length,
launchAssetCount: launchAssets.length,
uniqueAssetCount: uniqueAssets.length,
excludedAssets,
uniqueUploadedAssetCount,
uniqueUploadedAssetPaths,
assetLimitPerUpdateGroup,
};
}
@@ -0,0 +1,13 @@
import { truthy } from '../filter';
import partition from '../partition';
describe(partition, () => {
it('partitions', () => {
expect(partition([true, false], truthy)).toMatchObject([[true], [false]]);
expect(partition([1, 2, 3, 4], e => e <= 2)).toMatchObject([
[1, 2],
[3, 4],
]);
});
});
@@ -0,0 +1,14 @@
export default function partition<T>(arr: T[], predicate: (item: T) => boolean): [T[], T[]] {
const trueItems: T[] = [];
const falseItems: T[] = [];
for (const item of arr) {
if (predicate(item)) {
trueItems.push(item);
} else {
falseItems.push(item);
}
}
return [trueItems, falseItems];
}
+3
View File
@@ -13,4 +13,7 @@ export const EasJsonSchema = Joi.object({
}),
build: Joi.object().pattern(Joi.string(), BuildProfileSchema),
submit: Joi.object().pattern(Joi.string(), SubmitProfileSchema),
update: Joi.object({
assetExcludePatterns: Joi.array().items(Joi.string()),
}),
});
+3
View File
@@ -25,4 +25,7 @@ export interface EasJson {
};
build?: { [profileName: string]: EasJsonBuildProfile };
submit?: { [profileName: string]: EasJsonSubmitProfile };
update?: {
assetExcludePatterns?: string[];
};
}
+14
View File
@@ -48,4 +48,18 @@ export class EasJsonUtils {
const easJson = await accessor.readAsync();
return resolveSubmitProfile({ easJson, platform, profileName });
}
public static async getUpdatesConfigAsync(
accessor: EasJsonAccessor
): Promise<EasJson['update'] | null> {
try {
const easJson = await accessor.readAsync();
return easJson.update ?? null;
} catch (err: any) {
if (err instanceof MissingEasJsonError) {
return null;
}
throw err;
}
}
}