Compare commits

...
Author SHA1 Message Date
Will Schurman a3655c7407 [eas-cli] Add support for asset excludes 2023-02-23 11:24:37 -08:00
12 changed files with 435 additions and 136 deletions

No files matched your search

+2
View File
@@ -8,6 +8,8 @@ This is the log of notable changes to EAS CLI and related packages.
### 🎉 New features
- Add support for EAS Update asset excludes. ([#1526](https://github.com/expo/eas-cli/pull/1526) by [@wschurman](https://github.com/wschurman))
### 🐛 Bug fixes
### 🧹 Chores
@@ -225,8 +225,8 @@ function mockTestExport({
);
}
jest.mocked(collectAssetsAsync).mockResolvedValue(
Object.fromEntries(
jest.mocked(collectAssetsAsync).mockResolvedValue({
collectedAssets: new Map(
platforms.map(platform => [
platform,
{
@@ -237,8 +237,9 @@ function mockTestExport({
},
},
])
)
);
),
excludedAssets: [],
});
jest.mocked(uploadAssetsAsync).mockResolvedValue({
// platforms are mocked all containing only the launch asset
+10 -2
View File
@@ -302,9 +302,9 @@ export default class UpdatePublish extends EasCommand {
let assetLimitPerUpdateGroup = 0;
try {
const collectedAssets = await collectAssetsAsync(distRoot);
const { collectedAssets, excludedAssets } = await collectAssetsAsync(projectDir, distRoot);
const assets = filterExportedPlatformsByFlag(collectedAssets, platformFlag);
realizedPlatforms = Object.keys(assets) as PublishPlatform[];
realizedPlatforms = Array.from(assets.keys());
const uploadResults = await uploadAssetsAsync(
graphqlClient,
@@ -330,6 +330,13 @@ export default class UpdatePublish extends EasCommand {
Log.withTick(
`Uploaded ${uploadedBundleCount} app ${uploadedBundleCount === 1 ? 'bundle' : 'bundles'}`
);
if (excludedAssets.length > 0) {
Log.withTick(
`Excluded ${excludedAssets.length} ${
excludedAssets.length === 1 ? 'asset' : 'assets'
} from upload`
);
}
if (uploadedNormalAssetCount === 0) {
Log.withTick(`Uploading assets skipped - no new assets found`);
} else {
@@ -408,6 +415,7 @@ export default class UpdatePublish extends EasCommand {
};
}
);
let newUpdates: UpdatePublishMutation['updateBranch']['publishUpdateGroups'];
const publishSpinner = ora('Publishing...').start();
try {
@@ -1,3 +1,4 @@
import { EasJsonUtils } from '@expo/eas-json';
import fs from 'fs-extra';
import mockdate from 'mockdate';
import path from 'path';
@@ -10,6 +11,7 @@ import { AssetMetadataStatus } from '../../graphql/generated';
import { PublishMutation } from '../../graphql/mutations/PublishMutation';
import { PublishQuery } from '../../graphql/queries/PublishQuery';
import {
CollectedAssets,
MetadataJoi,
buildUnsortedUpdateInfoGroupAsync,
collectAssetsAsync,
@@ -28,6 +30,17 @@ import {
jest.mock('../../uploads');
jest.mock('fs');
jest.mock('@expo/eas-json', () => {
const actual = jest.requireActual('@expo/eas-json');
const EasJsonUtilsMock = {
getUpdatesConfigAsync: jest.fn(),
};
return {
...actual,
EasJsonUtils: EasJsonUtilsMock,
};
});
const dummyFileBuffer = Buffer.from('dummy-file');
@@ -96,20 +109,38 @@ describe('MetadataJoi', () => {
describe(filterExportedPlatformsByFlag, () => {
it(`returns all`, () => {
expect(filterExportedPlatformsByFlag({ web: true, ios: true, android: true }, 'all')).toEqual({
web: true,
ios: true,
android: true,
});
expect(
filterExportedPlatformsByFlag(
new Map([
['web', {}],
['ios', {}],
['android', {}],
]),
'all'
)
).toEqual(
new Map([
['web', {}],
['ios', {}],
['android', {}],
])
);
});
it(`selects a platform`, () => {
expect(filterExportedPlatformsByFlag({ web: true, ios: true, android: true }, 'ios')).toEqual({
ios: true,
});
expect(
filterExportedPlatformsByFlag(
new Map([
['web', {}],
['ios', {}],
['android', {}],
]),
'ios'
)
).toEqual(new Map([['ios', {}]]));
});
it(`asserts selected platform missing`, () => {
expect(() =>
filterExportedPlatformsByFlag({ web: true }, 'ios')
filterExportedPlatformsByFlag(new Map([['web', {}]]), 'ios')
).toThrowErrorMatchingInlineSnapshot(
`"--platform="ios" not found in metadata.json. Available platform(s): web"`
);
@@ -207,22 +238,25 @@ describe(buildUnsortedUpdateInfoGroupAsync, () => {
it('returns the correct value', async () => {
await expect(
buildUnsortedUpdateInfoGroupAsync(
{
android: {
launchAsset: {
fileExtension: '.bundle',
contentType: 'bundle/javascript',
path: androidBundlePath,
},
assets: [
{
fileExtension: '.jpg',
contentType: 'image/jpeg',
path: assetPath,
new Map([
[
'android',
{
launchAsset: {
fileExtension: '.bundle',
contentType: 'bundle/javascript',
path: androidBundlePath,
},
],
},
},
assets: [
{
fileExtension: '.jpg',
contentType: 'image/jpeg',
path: assetPath,
},
],
},
],
]),
{
slug: 'hello',
name: 'hello',
@@ -394,31 +428,155 @@ describe(collectAssetsAsync, () => {
})
);
expect(await collectAssetsAsync(inputDir)).toEqual({
android: {
launchAsset: {
fileExtension: '.bundle',
contentType: 'application/javascript',
path: `${inputDir}/bundles/android.js`,
},
assets: userDefinedAssets,
const projectDir = path.resolve();
expect(await collectAssetsAsync(projectDir, inputDir)).toEqual({
excludedAssets: [],
collectedAssets: new Map([
[
'android',
{
launchAsset: {
fileExtension: '.bundle',
contentType: 'application/javascript',
path: `${inputDir}/bundles/android.js`,
},
assets: userDefinedAssets,
},
],
[
'ios',
{
launchAsset: {
fileExtension: '.bundle',
contentType: 'application/javascript',
path: `${inputDir}/bundles/ios.js`,
},
assets: userDefinedAssets,
},
],
[
'web',
{
launchAsset: {
fileExtension: '.bundle',
contentType: 'application/javascript',
path: `${inputDir}/bundles/web.js`,
},
assets: userDefinedAssets,
},
],
]),
});
});
it('supports asset exclusion', async () => {
const fakeHash = 'hdahukw8adhawi8fawhfa8';
const bundles = {
android: 'android-bundle-code',
ios: 'ios-bundle-code',
web: 'web-bundle-code',
};
const inputDir = path.resolve(uuidv4());
const userDefinedAssets = [
{
fileExtension: '.jpg',
contentType: 'image/jpeg',
path: `${inputDir}/assets/${fakeHash}`,
originalPath: `assets/wat.jpg`,
},
ios: {
launchAsset: {
fileExtension: '.bundle',
contentType: 'application/javascript',
path: `${inputDir}/bundles/ios.js`,
];
const bundleDir = path.resolve(`${inputDir}/bundles`);
const assetDir = path.resolve(`${inputDir}/assets`);
await fs.mkdir(bundleDir, { recursive: true });
await fs.mkdir(assetDir, { recursive: true });
for (const platform of defaultPublishPlatforms) {
await fs.writeFile(path.resolve(inputDir, `bundles/${platform}.js`), bundles[platform]);
}
await fs.writeFile(path.resolve(`${inputDir}/assets/${fakeHash}`), dummyFileBuffer);
await fs.writeFile(
path.resolve(inputDir, 'metadata.json'),
JSON.stringify({
version: 0,
bundler: 'metro',
fileMetadata: {
android: {
assets: [{ path: `assets/${fakeHash}`, ext: 'jpg' }],
bundle: 'bundles/android.js',
},
ios: {
assets: [{ path: `assets/${fakeHash}`, ext: 'jpg' }],
bundle: 'bundles/ios.js',
},
web: {
assets: [{ path: `assets/${fakeHash}`, ext: 'jpg' }],
bundle: 'bundles/web.js',
},
},
assets: userDefinedAssets,
},
web: {
launchAsset: {
fileExtension: '.bundle',
contentType: 'application/javascript',
path: `${inputDir}/bundles/web.js`,
})
);
await fs.writeFile(
path.resolve(inputDir, 'assetmap.json'),
JSON.stringify({
[fakeHash]: {
__packager_asset: true,
fileSystemLocation: '/Users/blah/temp/assets',
httpServerLocation: 'assets/assets',
width: 2339,
height: 1560,
scales: [1],
files: ['/Users/blah/temp/assets/wat.jpg'],
hash: fakeHash,
name: 'wat',
type: 'jpg',
fileHashes: [fakeHash],
},
assets: userDefinedAssets,
},
})
);
jest.mocked(EasJsonUtils.getUpdatesConfigAsync).mockResolvedValue({
assetExcludePatterns: ['**/*.jpg'],
});
const projectDir = path.resolve();
expect(await collectAssetsAsync(projectDir, inputDir)).toEqual({
excludedAssets: userDefinedAssets,
collectedAssets: new Map([
[
'android',
{
launchAsset: {
fileExtension: '.bundle',
contentType: 'application/javascript',
path: `${inputDir}/bundles/android.js`,
},
assets: [],
},
],
[
'ios',
{
launchAsset: {
fileExtension: '.bundle',
contentType: 'application/javascript',
path: `${inputDir}/bundles/ios.js`,
},
assets: [],
},
],
[
'web',
{
launchAsset: {
fileExtension: '.bundle',
contentType: 'application/javascript',
path: `${inputDir}/bundles/web.js`,
},
assets: [],
},
],
]),
});
});
});
@@ -488,40 +646,46 @@ describe(uploadAssetsAsync, () => {
path: userDefinedPath,
};
const assetsForUpdateInfoGroup = {
android: {
launchAsset: {
type: 'bundle',
contentType: 'application/javascript',
path: androidBundlePath,
},
assets: [
userDefinedAsset,
{
type: 'jpg',
contentType: 'image/jpeg',
path: dummyFilePath,
originalPath: dummyOriginalFilePath,
const assetsForUpdateInfoGroup: CollectedAssets = new Map([
[
'android',
{
launchAsset: {
type: 'bundle',
contentType: 'application/javascript',
path: androidBundlePath,
},
],
},
ios: {
launchAsset: {
type: 'bundle',
contentType: 'application/javascript',
path: androidBundlePath,
assets: [
userDefinedAsset,
{
type: 'jpg',
contentType: 'image/jpeg',
path: dummyFilePath,
originalPath: dummyOriginalFilePath,
},
],
},
assets: [
userDefinedAsset,
{
type: 'jpg',
contentType: 'image/jpeg',
path: dummyFilePath,
originalPath: dummyOriginalFilePath,
],
[
'ios',
{
launchAsset: {
type: 'bundle',
contentType: 'application/javascript',
path: androidBundlePath,
},
],
},
};
assets: [
userDefinedAsset,
{
type: 'jpg',
contentType: 'image/jpeg',
path: dummyFilePath,
originalPath: dummyOriginalFilePath,
},
],
},
],
]);
beforeAll(async () => {
await fs.writeFile(androidBundlePath, publishBundles.android.code);
+106 -53
View File
@@ -1,9 +1,12 @@
import { ExpoConfig, Platform } from '@expo/config';
import { EasJsonAccessor, EasJsonUtils } from '@expo/eas-json';
import JsonFile from '@expo/json-file';
import crypto from 'crypto';
import fs from 'fs-extra';
import Joi from 'joi';
import mime from 'mime';
import minimatch from 'minimatch';
import nullthrows from 'nullthrows';
import path from 'path';
import promiseLimit from 'promise-limit';
@@ -16,6 +19,9 @@ import { PresignedPost, uploadWithPresignedPostWithRetryAsync } from '../uploads
import { expoCommandAsync, shouldUseVersionedExpoCLI } from '../utils/expoCli';
import chunk from '../utils/expodash/chunk';
import { truthy } from '../utils/expodash/filter';
import mapMap from '../utils/expodash/mapMap';
import mapMapAsync from '../utils/expodash/mapMapAsync';
import partition from '../utils/expodash/partition';
import uniqBy from '../utils/expodash/uniqBy';
export type ExpoCLIExportPlatformFlag = Platform | 'all';
@@ -35,12 +41,15 @@ export type RawAsset = {
originalPath?: string;
};
type CollectedAssets = {
[platform in Platform]?: {
export type CollectedAssets = Map<
Platform,
{
launchAsset: RawAsset;
assets: RawAsset[];
};
};
}
>;
type ExcludedAssets = RawAsset[];
type ManifestExtra = {
expoClient?: { [key: string]: any };
@@ -145,20 +154,21 @@ export async function buildUnsortedUpdateInfoGroupAsync(
assets: CollectedAssets,
exp: ExpoConfig
): Promise<UpdateInfoGroup> {
let platform: Platform;
const updateInfoGroup: Partial<UpdateInfoGroup> = {};
for (platform in assets) {
updateInfoGroup[platform] = {
launchAsset: await convertAssetToUpdateInfoGroupFormatAsync(assets[platform]?.launchAsset!),
assets: await Promise.all(
(assets[platform]?.assets ?? []).map(convertAssetToUpdateInfoGroupFormatAsync)
),
extra: {
expoClient: exp,
},
};
}
return updateInfoGroup as UpdateInfoGroup;
return Object.fromEntries(
(
await mapMapAsync(assets, async (collectedAssets): Promise<ManifestFragment> => {
return {
launchAsset: await convertAssetToUpdateInfoGroupFormatAsync(collectedAssets.launchAsset),
assets: await Promise.all(
collectedAssets.assets.map(convertAssetToUpdateInfoGroupFormatAsync)
),
extra: {
expoClient: exp,
},
};
})
).entries()
) as UpdateInfoGroup;
}
export async function buildBundlesAsync({
@@ -240,25 +250,25 @@ export function loadMetadata(distRoot: string): Metadata {
return metadata;
}
export function filterExportedPlatformsByFlag<T extends Partial<Record<Platform, any>>>(
record: T,
export function filterExportedPlatformsByFlag<V>(
record: Map<Platform, V>,
platformFlag: ExpoCLIExportPlatformFlag
): T {
): Map<Platform, V> {
if (platformFlag === 'all') {
return record;
}
const platform = platformFlag as Platform;
if (!record[platform]) {
if (!record.has(platform)) {
throw new Error(
`--platform="${platform}" not found in metadata.json. Available platform(s): ${Object.keys(
record
`--platform="${platform}" not found in metadata.json. Available platform(s): ${Array.from(
record.keys()
).join(', ')}`
);
}
return { [platform]: record[platform] } as T;
return new Map([[platform, nullthrows(record.get(platform))]]);
}
/** Try to load the asset map for logging the names of assets published */
@@ -300,29 +310,77 @@ export function getOriginalPathFromAssetMap(
}
/** Given a directory, load the metadata.json and collect the assets for each platform. */
export async function collectAssetsAsync(dir: string): Promise<CollectedAssets> {
const metadata = loadMetadata(dir);
const assetmap = await loadAssetMapAsync(dir);
export async function collectAssetsAsync(
projectDir: string,
distRoot: string
): Promise<{ collectedAssets: CollectedAssets; excludedAssets: ExcludedAssets }> {
const metadata = loadMetadata(distRoot);
const assetmap = await loadAssetMapAsync(distRoot);
const collectedAssets: CollectedAssets = {};
const easJsonAccessor = new EasJsonAccessor(projectDir);
const updatesConfig = await EasJsonUtils.getUpdatesConfigAsync(easJsonAccessor);
const assetExcludePatterns = updatesConfig?.assetExcludePatterns;
for (const platform of Object.keys(metadata.fileMetadata) as Platform[]) {
collectedAssets[platform] = {
launchAsset: {
fileExtension: '.bundle',
contentType: 'application/javascript',
path: path.resolve(dir, metadata.fileMetadata[platform].bundle),
},
assets: metadata.fileMetadata[platform].assets.map(asset => ({
fileExtension: asset.ext ? ensureLeadingPeriod(asset.ext) : undefined,
originalPath: getOriginalPathFromAssetMap(assetmap, asset) ?? undefined,
contentType: guessContentTypeFromExtension(asset.ext),
path: path.join(dir, asset.path),
})),
const platformToAssets = new Map(
(Object.keys(metadata.fileMetadata) as Platform[]).map(platform => [
platform,
metadata.fileMetadata[platform].assets,
])
);
const platformToIncludedAndExcludedAssets = mapMap(platformToAssets, assets => {
if (!assetExcludePatterns) {
return {
includedAssets: assets,
excludedAssets: [],
};
}
const [includedAssets, excludedAssets] = partition(assets, asset => {
const originalPath = getOriginalPathFromAssetMap(assetmap, asset) ?? undefined;
return (
!originalPath || !assetExcludePatterns.some(pattern => minimatch(originalPath, pattern))
);
});
return {
includedAssets,
excludedAssets,
};
}
});
return collectedAssets;
const transformAsset = (asset: { path: string; ext: string }): RawAsset => {
return {
fileExtension: asset.ext ? ensureLeadingPeriod(asset.ext) : undefined,
originalPath: getOriginalPathFromAssetMap(assetmap, asset) ?? undefined,
contentType: guessContentTypeFromExtension(asset.ext),
path: path.join(distRoot, asset.path),
};
};
const collectedAssets = mapMap(
platformToIncludedAndExcludedAssets,
({ includedAssets }, platform) => {
return {
launchAsset: {
fileExtension: '.bundle',
contentType: 'application/javascript',
path: path.resolve(distRoot, metadata.fileMetadata[platform].bundle),
},
assets: includedAssets.map(transformAsset),
};
}
);
const excludedAssets = uniqBy(
Array.from(
mapMap(platformToIncludedAndExcludedAssets, ({ excludedAssets }) =>
excludedAssets.map(transformAsset)
).values()
).flat(),
rawAsset => rawAsset.path
);
return { collectedAssets, excludedAssets };
}
// ensure the file extension has a '.' prefix
@@ -370,16 +428,11 @@ export async function uploadAssetsAsync(
updateSpinnerText?: (totalAssets: number, missingAssets: number) => void
): Promise<AssetUploadResult> {
let assets: RawAsset[] = [];
let platform: keyof CollectedAssets;
const launchAssets: RawAsset[] = [];
for (platform in assetsForUpdateInfoGroup) {
launchAssets.push(assetsForUpdateInfoGroup[platform]!.launchAsset);
assets = [
...assets,
assetsForUpdateInfoGroup[platform]!.launchAsset,
...assetsForUpdateInfoGroup[platform]!.assets,
];
}
mapMap(assetsForUpdateInfoGroup, collectedAssets => {
launchAssets.push(collectedAssets.launchAsset);
assets = [...assets, collectedAssets.launchAsset, ...collectedAssets.assets];
});
const assetsWithStorageKey = await Promise.all(
assets.map(async asset => {
@@ -0,0 +1,13 @@
import { truthy } from '../filter';
import partition from '../partition';
describe(partition, () => {
it('partitions', () => {
expect(partition([true, false], truthy)).toMatchObject([[true], [false]]);
expect(partition([1, 2, 3, 4], e => e <= 2)).toMatchObject([
[1, 2],
[3, 4],
]);
});
});
@@ -0,0 +1,10 @@
export default function mapMap<K, V, M>(
map: ReadonlyMap<K, V>,
mapper: (value: V, key: K) => M
): Map<K, M> {
const resultingMap = new Map();
for (const [k, v] of map) {
resultingMap.set(k, mapper(v, k));
}
return resultingMap;
}
@@ -0,0 +1,14 @@
export default async function mapMapAsync<K, V, M>(
map: ReadonlyMap<K, V>,
mapper: (value: V, key: K) => Promise<M>
): Promise<Map<K, M>> {
const resultingMap: Map<K, M> = new Map();
await Promise.all(
Array.from(map.keys()).map(async k => {
const initialValue = map.get(k) as V;
const result = await mapper(initialValue, k);
resultingMap.set(k, result);
})
);
return resultingMap;
}
@@ -0,0 +1,14 @@
export default function partition<T>(arr: T[], predicate: (item: T) => boolean): [T[], T[]] {
const trueItems: T[] = [];
const falseItems: T[] = [];
for (const item of arr) {
if (predicate(item)) {
trueItems.push(item);
} else {
falseItems.push(item);
}
}
return [trueItems, falseItems];
}
+3
View File
@@ -13,4 +13,7 @@ export const EasJsonSchema = Joi.object({
}),
build: Joi.object().pattern(Joi.string(), BuildProfileSchema),
submit: Joi.object().pattern(Joi.string(), SubmitProfileSchema),
update: Joi.object({
assetExcludePatterns: Joi.array().items(Joi.string()),
}),
});
+3
View File
@@ -25,4 +25,7 @@ export interface EasJson {
};
build?: { [profileName: string]: EasJsonBuildProfile };
submit?: { [profileName: string]: EasJsonSubmitProfile };
update?: {
assetExcludePatterns?: string[];
};
}
+14
View File
@@ -48,4 +48,18 @@ export class EasJsonUtils {
const easJson = await accessor.readAsync();
return resolveSubmitProfile({ easJson, platform, profileName });
}
public static async getUpdatesConfigAsync(
accessor: EasJsonAccessor
): Promise<EasJson['update'] | null> {
try {
const easJson = await accessor.readAsync();
return easJson.update ?? null;
} catch (err: any) {
if (err instanceof MissingEasJsonError) {
return null;
}
throw err;
}
}
}