update changelog

* update xdl (fix support for binary plist)

* Update CHANGELOG.md

* Apply suggestions from code review

Co-Authored-By: Dominik Sokal <[email protected]>

CHANGELOG.md

@@ -5,10 +5,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Adhoc Builds: use provisioningProfileId if specified.
+- Registering supported SDK versions in Redis.
+
+## [0.8.6] - 2019-06-25
+### Added
+- Added support for Android build modes: debug or release.
+### Fixed
+- Upgrading the shell app for a given SDK version when the previous version is already downloaded (bug in `turtle-cli`).
+- Updated Android shell app for SDK33 to include `[email protected]` which fixes crashes on Android 9.0 when showing some Lottie animations.
+- Schema for `buildModes`.
+- Updated @expo/xdl to 55.0.7 (fixed reading from binary plist).
 
 ## [0.8.4] - 2019-06-13
 ### Changed
 - Upgraded `sharp` from `0.21.3` to `0.22.1`.
+- `gulp-cli` is no longer required to build an app with `turtle-cli`.
 
 ## [0.8.3] - 2019-06-13
 ### Added
@@ -116,7 +129,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.5.10] - 2019-02-19
 ### Added
-- Register Turtle version in Redis.
+- Registering Turtle version in Redis.
 ### Changed
 - Update Android shell app for SDK 32 to fix [update](https://github.com/expo/expo/issues/3504) and [back button](https://github.com/expo/expo/issues/1786) issues.
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "turtle-cli",
-  "version": "0.8.4",
+  "version": "0.8.6",
   "description": "The command-line tool for building standalone Expo apps",
   "preferGlobal": true,
   "scripts": {
@@ -13,6 +13,7 @@
     "prepare": "rimraf ./build && yarn build",
     "build": "ttsc",
     "build:production": "./scripts/build.sh",
+    "watch": "ttsc --watch",
     "secrets:init-private": "./scripts/initPrivateSecrets.sh",
     "set-workingdir:remote": "./scripts/setWorkingdirMode.sh remote",
     "set-workingdir:local": "./scripts/setWorkingdirMode.sh local",
@@ -110,7 +111,7 @@
   },
   "dependencies": {
     "@expo/spawn-async": "^1.4.2",
-    "@expo/xdl": "54.1.4",
+    "@expo/xdl": "55.0.7",
     "@google-cloud/logging-bunyan": "^0.9.5",
     "async-retry": "^1.2.1",
     "aws-sdk": "^2.226.1",

shellTarballs/android/sdk33

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-e244e865ed8cb8e20d29ea00b6373845d1c61068.tar.gz
+s3://exp-artifacts/android-shell-builder-5090aab941f92c912c6643a1e25cc69f9f67d8e6.tar.gz

src/bin/commands/build/android.ts

@@ -23,9 +23,15 @@ export default (program: any, setCommonCommandOptions: any) => {
       /^(app-bundle|apk)$/i,
       'app-bundle',
     )
+    .option(
+      '-m --mode <build>',
+      'type of build: debug|release',
+      /^(debug|release)$/i,
+      'release',
+    )
     .description(
-      'Build a standalone APK or App Bundle for your project, signed and ready for submission to the Google Play'
-      + ' Store.',
+      'Build a standalone APK or App Bundle for your project, either signed and ready for submission to'
+      + ' the Google Play Store or in debug mode.',
     )
     .asyncAction(
       createBuilderAction({

src/bin/setup/android/index.ts

@@ -20,10 +20,6 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
     command: 'bash',
     missingDescription: 'Please install bash',
   },
-  {
-    command: 'gulp',
-    missingDescription: 'Run `npm install -g gulp-cli` (or `yarn global add gulp-cli`) to install gulp',
-  },
   {
     command: 'javac',
     missingDescription: 'Please install JDK (version 8 or newer) - check https://jdk.java.net/',

src/bin/setup/android/ndk.ts

@@ -4,7 +4,7 @@ import path from 'path';
 import decompress from 'decompress';
 import fs from 'fs-extra';
 
-import { formatArtifactDownloadPath } from 'turtle/bin/setup/utils/common';
+import * as utils from 'turtle/bin/setup/utils/common';
 import download from 'turtle/bin/setup/utils/downloader';
 import config from 'turtle/config';
 import logger from 'turtle/logger';
@@ -16,9 +16,10 @@ const l = logger.child(LOGGER_FIELDS);
 
 export default async function ensureAndroidNDKIsPresent() {
   const androidNdkDir = path.join(config.directories.androidDependenciesDir, 'ndk');
+  await utils.removeDirectoryUnlessReady(androidNdkDir);
   if (!(await fs.pathExists(androidNdkDir))) {
     await fs.ensureDir(androidNdkDir);
-    const androidNdkDownloadPath = formatArtifactDownloadPath(ANDROID_NDK_URL);
+    const androidNdkDownloadPath = utils.formatArtifactDownloadPath(ANDROID_NDK_URL);
 
     try {
       l.info('Downloading Android NDK');
@@ -26,6 +27,7 @@ export default async function ensureAndroidNDKIsPresent() {
       await download(ANDROID_NDK_URL, androidNdkDownloadPath);
       l.info('Decompressing Android NDK');
       await decompress(androidNdkDownloadPath, androidNdkDir, { strip: 1 });
+      await utils.markDirectoryAsReady(androidNdkDir);
       l.info('Android NDK installed successfully');
     } catch (err) {
       await fs.remove(androidNdkDir);

src/bin/setup/android/sdk.ts

@@ -5,7 +5,7 @@ import { ExponentTools } from '@expo/xdl';
 import decompress from 'decompress';
 import fs from 'fs-extra';
 
-import { formatArtifactDownloadPath } from 'turtle/bin/setup/utils/common';
+import * as utils from 'turtle/bin/setup/utils/common';
 import download from 'turtle/bin/setup/utils/downloader';
 import config from 'turtle/config';
 import logger from 'turtle/logger';
@@ -17,9 +17,10 @@ const l = logger.child(LOGGER_FIELDS);
 
 export default async function ensureAndroidSDKIsPresent() {
   const androidSdkDir = path.join(config.directories.androidDependenciesDir, 'sdk');
+  await utils.removeDirectoryUnlessReady(androidSdkDir);
   if (!(await fs.pathExists(androidSdkDir))) {
     await fs.ensureDir(androidSdkDir);
-    const androidSdkDownloadPath = formatArtifactDownloadPath(ANDROID_SDK_URL);
+    const androidSdkDownloadPath = utils.formatArtifactDownloadPath(ANDROID_SDK_URL);
 
     try {
       l.info('Downloading Android SDK');
@@ -31,6 +32,7 @@ export default async function ensureAndroidSDKIsPresent() {
       await fs.remove(androidSdkDownloadPath);
       l.info('Configuring Android SDK, this may take a while');
       await _configureSdk(androidSdkDir);
+      await utils.markDirectoryAsReady(androidSdkDir);
       l.info('Android SDK installed and configured successfully');
     } catch (err) {
       await fs.remove(androidSdkDir);

src/bin/setup/utils/common.ts

@@ -29,6 +29,8 @@ export async function ensureShellAppIsPresent(
   postDownloadAction?: PostDownloadAction,
 ) {
   const workingdir = formatters.formatShellAppDirectory({ sdkVersion });
+  const shellAppMetadata = await _readShellAppTarballS3Uri(sdkVersion, formatters);
+  await removeDirectoryUnlessReady(workingdir, shellAppMetadata);
   if (await fs.pathExists(workingdir)) {
     return;
   }
@@ -37,6 +39,7 @@ export async function ensureShellAppIsPresent(
   if (postDownloadAction) {
     await postDownloadAction(sdkVersion, workingdir);
   }
+  await markDirectoryAsReady(workingdir, shellAppMetadata);
 }
 
 async function _downloadShellApp(sdkVersion: string, targetDirectory: string, formatters: IShellAppFormaters) {
@@ -65,3 +68,31 @@ async function _readShellAppTarballS3Uri(sdkVersion: string, formatters: IShellA
   const data = await fs.readFile(filePath, 'utf8');
   return data.trim();
 }
+
+export async function removeDirectoryUnlessReady(dir: string, metadata?: string) {
+  const readyFilePath = path.join(dir, '.ready');
+  const readyFileExists = await fs.pathExists(readyFilePath);
+  let shouldRemove = false;
+  if (readyFileExists) {
+    if (metadata !== undefined) {
+      const readyFileContents = (await fs.readFile(readyFilePath, 'utf-8')).trim();
+      if (readyFileContents !== metadata) {
+        shouldRemove = true;
+       }
+    }
+  } else {
+    shouldRemove = true;
+  }
+  if (shouldRemove) {
+    await fs.remove(dir);
+  }
+}
+
+export async function markDirectoryAsReady(dir: string, metadata?: string) {
+  const readyFilePath = path.join(dir, '.ready');
+  if (metadata === undefined) {
+    await fs.open(readyFilePath, 'w');
+  } else {
+    await fs.writeFile(readyFilePath, metadata);
+  }
+}

src/bin/utils/builder.ts

@@ -57,6 +57,7 @@ export function createBuilderAction({
       const args = {
         releaseChannel: cmd.releaseChannel || 'default',
         buildType: cmd.type,
+        buildMode: cmd.mode,
         username: userData.username || 'anonymous',
         projectDir: ProjectUtils.resolveAbsoluteDir(projectDirArg),
         publicUrl: cmd.publicUrl,
@@ -88,13 +89,14 @@ export function createBuilderAction({
 const buildJobObject = async (
   platform: 'android' | 'ios',
   appJSON: any,
-  { releaseChannel, buildType, username, publicUrl, projectDir }: any,
+  { releaseChannel, buildType, buildMode, username, publicUrl, projectDir }: any,
   credentials: any,
 ) => {
   const job = {
     config: {
       ..._.get(appJSON, 'expo.ios.config', {}),
       buildType,
+      ...(platform === 'android' ? { buildMode } : {}),
       releaseChannel,
       ...(platform === 'ios' ? { bundleIdentifier: _.get(appJSON, 'expo.ios.bundleIdentifier') } : {}),
       ...(platform === 'android' ? { androidPackage: _.get(appJSON, 'expo.android.package') } : {}),

src/builders/android.ts

@@ -93,6 +93,7 @@ async function runShellAppBuilder(
       outputFile: outputFilePath,
       modules: enabledModules,
       buildType: jobData.config.buildType,
+      buildMode: jobData.config.buildMode,
     });
   } catch (err) {
     commonUtils.logErrorOnce(err);

src/builders/utils/ios/adhocBuild.ts

@@ -3,7 +3,6 @@ import path from 'path';
 import spawnAsync from '@expo/spawn-async';
 import { IosCodeSigning } from '@expo/xdl';
 import fs from 'fs-extra';
-import isEmpty from 'lodash/isEmpty';
 
 import * as sqs from 'turtle/aws/sqs';
 import BuildError, { BuildErrorReason } from 'turtle/builders/BuildError';
@@ -21,10 +20,18 @@ async function prepareAdHocBuildCredentials(job: IJob) {
   }
 
   const { bundleIdentifier } = job.config;
-  const { certP12, certPassword, teamId, appleSession, udids } = job.credentials;
+  const {
+    certP12,
+    certPassword,
+    teamId,
+    appleSession,
+    udids,
+    provisioningProfileId,
+  } = job.credentials;
 
   const certSerialNumber = IosCodeSigning.findP12CertSerialNumber(certP12, certPassword);
   const args = [
+    ...(provisioningProfileId ? ['--profile-id', provisioningProfileId] : []),
     teamId,
     udids!.join(','),
     bundleIdentifier,
@@ -37,20 +44,15 @@ async function prepareAdHocBuildCredentials(job: IJob) {
       args,
       { env: { FASTLANE_SESSION: appleSession } },
     );
-    if (isEmpty(credentials)) {
-      if (!job.credentials.provisioningProfile) {
-        // a user has generated the provisioning profile for his app, but he probably deleted it from Expo servers, ...
-        throw new BuildError(
-          BuildErrorReason.PROVISIONING_PROFILE_MISSING,
-          `Provisioning profile is missing on Expo servers.`,
-        );
-      }
-      return;
-    }
 
     logger.info('New ad hoc provisioning profile successfully created');
     job.credentials.provisioningProfile = credentials.provisioningProfile;
 
+    if (!credentials.provisioningProfileUpdateTimestamp) {
+      //  dont need to persist profile because nothing changed
+      return;
+    }
+
     if (isOffline()) {
       const provisioningProfilePath = path.join(job.projectDir, `adhoc.mobileprovision`);
       await fs.writeFile(provisioningProfilePath, Buffer.from(credentials.provisioningProfile, 'base64'));

src/builders/utils/version.ts

@@ -5,26 +5,32 @@ import path from 'path';
 import config from 'turtle/config';
 import { IJob } from 'turtle/job';
 
-let SUPPORTED_SDK_VERSIONS: number[];
+interface IVersionsMap {
+  [platform: string]: number[];
+}
+
+const versions: IVersionsMap = {};
 
 export async function ensureCanBuildSdkVersion(job: IJob) {
   if (config.builder.useLocalWorkingDir) {
     return;
   }
 
-  if (!SUPPORTED_SDK_VERSIONS) {
-    SUPPORTED_SDK_VERSIONS = await findSupportedSdkVersions(job.platform);
-  }
-
+  const platformVersions = await findSupportedSdkVersions(job.platform);
   const targetMajorSdkVersion = ExponentTools.parseSdkMajorVersion(job.sdkVersion || job.manifest.sdkVersion);
-  if (!SUPPORTED_SDK_VERSIONS.includes(targetMajorSdkVersion)) {
+  if (!platformVersions.includes(targetMajorSdkVersion)) {
     throw new Error(`Unsupported SDK Version!`);
   }
 }
 
-async function findSupportedSdkVersions(platform: 'android' | 'ios'): Promise<number[]> {
-  const SDK_DIR_PREFIX = 'sdk';
-  const files = await fs.readdir(path.join(config.directories.workingDir, platform));
-  const sdks = files.filter((file) => file.startsWith(SDK_DIR_PREFIX));
-  return sdks.map((sdk) => parseInt(sdk.substr(SDK_DIR_PREFIX.length), 10));
+export async function findSupportedSdkVersions(platform: 'android' | 'ios'): Promise<number[]> {
+  if (versions[platform]) {
+    return versions[platform];
+  } else {
+    const SDK_DIR_PREFIX = 'sdk';
+    const files = await fs.readdir(path.join(config.directories.workingDir, platform));
+    const sdks = files.filter((file) => file.startsWith(SDK_DIR_PREFIX));
+    versions[platform] = sdks.map((sdk) => parseInt(sdk.substr(SDK_DIR_PREFIX.length), 10));
+    return versions[platform];
+  }
 }

src/constants/index.ts

@@ -16,4 +16,9 @@ export enum ANDROID_BUILD_TYPES {
   APP_BUNDLE = 'app-bundle',
 }
 
+export enum ANDROID_BUILD_MODES {
+  DEBUG = 'debug',
+  RELEASE = 'release',
+}
+
 export { BUILD };

src/job.ts

@@ -1,4 +1,4 @@
-import { ANDROID_BUILD_TYPES, IOS_BUILD_TYPES } from 'turtle/constants';
+import { ANDROID_BUILD_MODES, ANDROID_BUILD_TYPES, IOS_BUILD_TYPES } from 'turtle/constants';
 
 export interface IJob {
   platform: 'android' | 'ios';
@@ -14,6 +14,7 @@ export interface IJob {
     bundleIdentifier?: string;
     // android
     androidPackage?: string;
+    buildMode?: ANDROID_BUILD_MODES;
   };
   credentials: {
     // android
@@ -25,6 +26,7 @@ export interface IJob {
     certP12?: string;
     certPassword?: string;
     provisioningProfile?: string;
+    provisioningProfileId?: string;
     teamId?: string;
     appleSession?: string;
     udids?: string[];

src/jobsSchemas/android.ts

@@ -1,6 +1,6 @@
 import Joi from 'joi';
 
-import { PLATFORMS } from 'turtle/constants';
+import { ANDROID_BUILD_MODES, ANDROID_BUILD_TYPES, PLATFORMS } from 'turtle/constants';
 import baseJobSchema from 'turtle/jobsSchemas/base';
 
 export default baseJobSchema.concat(
@@ -12,5 +12,9 @@ export default baseJobSchema.concat(
       keystoreAlias: Joi.string(),
       keystore: Joi.string(),
     }),
+    config: Joi.object().keys({
+      buildMode: Joi.string().valid(Object.values(ANDROID_BUILD_MODES)).default(ANDROID_BUILD_MODES.RELEASE),
+      buildType: Joi.string().valid(Object.values(ANDROID_BUILD_TYPES)).default(ANDROID_BUILD_TYPES.APK),
+    }),
   }),
 );

src/logger/index.ts

@@ -17,6 +17,7 @@ class Logger {
       name: 'turtle',
       level: config.logger.level,
       platform: config.platform,
+      serializers: bunyan.stdSerializers,
       ...config.deploymentEnv && { environment: config.deploymentEnv },
       streams: Object.values(streams),
     });

src/server.ts

@@ -1,13 +1,15 @@
 import { LoggerDetach } from '@expo/xdl';
 
+import { findSupportedSdkVersions } from 'turtle/builders/utils/version';
 import config from 'turtle/config';
 import { doJob } from 'turtle/jobManager';
 import logger from 'turtle/logger';
 import setup from 'turtle/setup';
 import { checkShouldExit, setShouldExit, turtleVersion } from 'turtle/turtleContext';
-import { getRedisClient, RedisClient } from 'turtle/utils/redis';
+import { getRedisClient } from 'turtle/utils/redis';
 
 const REDIS_TURTLE_VERSION_KEY = 'turtle:version';
+const REDIS_TURTLE_SUPPORTED_SDK_VERSIONS_KEY = `turtle:${config.platform}:sdkVersions`;
 
 process.on('unhandledRejection', (err) => logger.error({ err }, 'Unhandled promise rejection'));
 
@@ -35,11 +37,14 @@ async function main() {
   }
 
   try {
-    const redis = await getRedisClient(RedisClient.Configuration);
+    const redis = await getRedisClient();
     await redis.set(REDIS_TURTLE_VERSION_KEY, turtleVersion);
-    logger.info(`Register Turtle version ${turtleVersion} in Redis`);
+    logger.info(`Registered Turtle version (${turtleVersion}) in Redis`);
+    const sdkVersions = (await findSupportedSdkVersions(config.platform)).join(',');
+    await redis.set(REDIS_TURTLE_SUPPORTED_SDK_VERSIONS_KEY, sdkVersions);
+    logger.info(`Registered supported SDK versions (${sdkVersions}) in Redis`);
   } catch (err) {
-    logger.error({ err }, 'Failed to register Turtle version in Redis');
+    logger.error({ err }, 'Failed to set versions in Redis');
   }
 
   while (true) {

supervisord.conf

@@ -7,6 +7,10 @@ command=node ./build/server.js
 autorestart=true
 startretries=5
 startsecs=600
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
 
 [eventlistener:turtle_failure]
 events=PROCESS_STATE_FATAL