Release 0.8.3

* add supervisord

- restart pod only if there is at least 5 failures in 10 minutes

* Update CHANGELOG.md

Co-Authored-By: Dominik Sokal <[email protected]>

* Update scripts/android/supervisordEventListener.sh

Co-Authored-By: Dominik Sokal <[email protected]>

CHANGELOG.md

@@ -5,10 +5,30 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- `turtle-cli` is now fetching the app manifest before running the build.
+### Added
+- Running Android Turtle process with `supervisor` (pod is restarted if there were 5 failures in the last 10 minutes).
+
+## [0.8.2] - 2019-06-11
+### Changed
+- Updated Android shell app for SDK33 to remove `expo-face-detector` and `expo-payments-stripe` from standalone builds.
+- Updated Android shell app for SDK33 to get rid of `node_modules` for `tools-public`.
+- From now on, `turtle-cli` installs dependencies for `tools-public` in shell apps for SDK versions < 33.
+### Fixed
+- Android builds with turtle-cli by installing all unmodules if `dependencies` is empty.
+
+## [0.8.1] - 2019-06-10
+### Changed
+- Add expo dependencies when resolving unimodules.
+### Fixed
+- Support for Android tarballs without `universe-package.json`.
+
+## [0.8.0] - 2019-06-07
 ### Added
 - Turtle CLI logo to the project (created by [@zularizal](https://github.com/zularizal))!
-- Added shell apps for SDK 33.
-- Support for Android App Bundles
+- Shell apps for SDK 33.
+- Support for Android App Bundles (App bundles are built by default, use `-t apk` to build apk).
 ### Changed
 - Updated Android shell app for SDK 32 to fix AppAuth issue (https://github.com/expo/expo/pull/4115) and to fix a newer than expected JSC version.
 - Default value for build timeout to 15 minutes.
@@ -18,7 +38,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Generating Android keystore in turtle when one is not passed to turtle-cli (https://github.com/expo/turtle/pull/65).
 - Upload logs to s3 if build has been canceled or timed out.
 - Unhandled exception on asset download (xdl).
-- Fixed support for Android tarballs without `universe-package.json`.
 
 ## [0.7.2] - 2019-05-09
 ### Added

Dockerfile

@@ -21,10 +21,10 @@ RUN dpkg --add-architecture i386 && \
   libncurses5:i386\
   libstdc++6:i386\
   rsync\
-  time\
   unzip\
   yarn\
   zlib1g:i386\
+  supervisor\
   --no-install-recommends && \
   apt-get clean
 
@@ -52,20 +52,8 @@ ENV PATH ${ANDROID_HOME}/tools:${PATH}
 ENV PATH ${ANDROID_HOME}/tools/bin:${PATH}
 ENV PATH ${ANDROID_HOME}/build-tools/28.0.3/:${PATH}
 
-RUN yes | sdkmanager --licenses > /dev/null
-RUN yes | sdkmanager \
-  "platforms;android-23" \
-  "platforms;android-24" \
-  "platforms;android-25" \
-  "platforms;android-26" \
-  "platforms;android-27" \
-  "platforms;android-28" | grep -v '=' || echo 'no output'
-RUN yes | sdkmanager "platform-tools" | grep -v '=' || echo 'no output'
-RUN yes | sdkmanager "build-tools;28.0.3" | grep -v '=' || echo 'no output'
-RUN yes | sdkmanager \
-  "extras;android;m2repository" \
-  "extras;google;m2repository" \
-  "extras;google;google_play_services" | grep -v '=' || echo 'no output'
+ADD scripts/android/configureAndroidSdk.sh /tmp/configureAndroidSdk.sh
+RUN /tmp/configureAndroidSdk.sh --install-all-platforms
 
 # Install Android NDK
 ENV ANDROID_NDK_VERSION android-ndk-r10e
@@ -117,11 +105,13 @@ RUN mv /app/turtle/workingdir /app && \
       yarn install; \
       fi \
     ; done && \
-    time mv /app/workingdir /app/turtle
+    mv /app/workingdir /app/turtle
 
 ENV NODE_ENV "production"
 ENV TURTLE_WORKING_DIR_PATH /app/turtle/workingdir/
 
 WORKDIR /app/turtle
 
-CMD ["node", "./build/server.js"]
+ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+CMD ["/usr/bin/supervisord"]

package.json

@@ -1,6 +1,6 @@
 {
   "name": "turtle-cli",
-  "version": "0.8.0",
+  "version": "0.8.3",
   "description": "The command-line tool for building standalone Expo apps",
   "preferGlobal": true,
   "scripts": {
@@ -59,7 +59,7 @@
   "files": [
     "bin",
     "build",
-    "scripts/cli",
+    "scripts/android/configureAndroidSdk.sh",
     "shellTarballs"
   ],
   "keywords": [

scripts/android/configureAndroidSdk.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+SDKMANAGER=$ANDROID_HOME/tools/bin/sdkmanager
+
+yes | $SDKMANAGER --licenses > /dev/null
+if [[ "$1" = "--install-all-platforms" ]]; then
+  yes | $SDKMANAGER \
+    "platforms;android-23" \
+    "platforms;android-24" \
+    "platforms;android-25" \
+    "platforms;android-26" \
+    "platforms;android-27" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]
+fi
+yes | $SDKMANAGER "platforms;android-28" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]
+yes | $SDKMANAGER "platform-tools" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]
+yes | $SDKMANAGER "build-tools;28.0.3" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]
+yes | $SDKMANAGER \
+  "extras;android;m2repository" \
+  "extras;google;m2repository" \
+  "extras;google;google_play_services" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]

scripts/android/supervisordEventListener.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+while read line; do
+  echo "[supervisord:eventlistener] Processing Event: $line" >&2;
+  kill -SIGQUIT $PPID
+done < /dev/stdin

scripts/cli/configureAndroidSdk.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-SDKMANAGER=$ANDROID_HOME/tools/bin/sdkmanager
-
-yes | $SDKMANAGER --licenses > /dev/null
-yes | $SDKMANAGER "platforms;android-28"
-yes | $SDKMANAGER "platform-tools"
-yes | $SDKMANAGER "build-tools;28.0.3"
-yes | $SDKMANAGER \
-  "extras;android;m2repository" \
-  "extras;google;m2repository" \
-  "extras;google;google_play_services"

shellTarballs/android/sdk33

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-86974c7df1fc9db46f02b59f4a663b45e2f7a2af.tar.gz
+s3://exp-artifacts/android-shell-builder-e244e865ed8cb8e20d29ea00b6373845d1c61068.tar.gz

src/bin/commands/build/android.ts

@@ -1,6 +1,5 @@
 import fs from 'fs-extra';
 import _ from 'lodash';
-import uuid from 'uuid';
 
 import { ErrorWithCommandHelp } from 'turtle/bin/commands/ErrorWithCommandHelp';
 import { createBuilderAction } from 'turtle/bin/utils/builder';
@@ -33,7 +32,6 @@ export default (program: any, setCommonCommandOptions: any) => {
         program,
         command,
         prepareCredentials,
-        buildJobObject,
         builder,
         platform: PLATFORMS.ANDROID,
         os: ['darwin', 'linux'],
@@ -41,26 +39,6 @@ export default (program: any, setCommonCommandOptions: any) => {
     );
 };
 
-const buildJobObject = (
-  appJSON: any,
-  { releaseChannel, username, projectDir, publicUrl, buildType }: any,
-  credentials: any,
-) => ({
-  config: {
-    ..._.get(appJSON, 'expo.android.config', {}),
-    releaseChannel,
-    androidPackage: _.get(appJSON, 'expo.android.package'),
-    publicUrl,
-    buildType,
-  },
-  id: uuid.v4(),
-  platform: PLATFORMS.ANDROID,
-  sdkVersion: _.get(appJSON, 'expo.sdkVersion'),
-  projectDir,
-  experienceName: `@${username}/${_.get(appJSON, 'expo.slug')}`,
-  ...(credentials && { credentials }),
-});
-
 const prepareCredentials = async (cmd: any) => {
   const { keystorePath, keystoreAlias } = cmd;
   const keystorePassword = process.env.EXPO_ANDROID_KEYSTORE_PASSWORD;

src/bin/commands/build/ios.ts

@@ -1,6 +1,5 @@
 import fs from 'fs-extra';
 import _ from 'lodash';
-import uuid from 'uuid';
 
 import { ErrorWithCommandHelp } from 'turtle/bin/commands/ErrorWithCommandHelp';
 import { createBuilderAction } from 'turtle/bin/utils/builder';
@@ -32,7 +31,6 @@ export default (program: any, setCommonCommandOptions: any) => {
         program,
         command,
         prepareCredentials,
-        buildJobObject,
         builder,
         platform: PLATFORMS.IOS,
         os: 'darwin',
@@ -40,21 +38,6 @@ export default (program: any, setCommonCommandOptions: any) => {
     );
 };
 
-const buildJobObject = (appJSON: any, { releaseChannel, buildType, username, publicUrl }: any, credentials: any) => ({
-  config: {
-    ..._.get(appJSON, 'expo.ios.config', {}),
-    buildType,
-    releaseChannel,
-    bundleIdentifier: _.get(appJSON, 'expo.ios.bundleIdentifier'),
-    publicUrl,
-  },
-  id: uuid.v4(),
-  platform: PLATFORMS.IOS,
-  sdkVersion: _.get(appJSON, 'expo.sdkVersion'),
-  experienceName: `@${username}/${_.get(appJSON, 'expo.slug')}`,
-  ...(credentials && { credentials }),
-});
-
 const prepareCredentials = async (cmd: any) => {
   if (cmd.type !== IOS_BUILD_TYPES.ARCHIVE) {
     return null;

src/bin/setup/android/index.ts

@@ -67,7 +67,7 @@ function formatShellAppTarballUriPath(sdkMajorVersion: string) {
   return path.join(config.directories.shellTarballsDir, PLATFORMS.ANDROID, `sdk${sdkMajorVersion}`);
 }
 
-async function _shellAppPostDownloadAction(workingdir: string) {
+async function _shellAppPostDownloadAction(sdkVersion: string, workingdir: string) {
   const inWorkingdir = (filename: string) => path.join(workingdir, filename);
 
   if (await fs.pathExists(inWorkingdir('universe-package.json'))) {
@@ -82,9 +82,11 @@ async function _shellAppPostDownloadAction(workingdir: string) {
     await _installNodeModules(workingdir);
   }
 
-  // TODO: remove following lines after upgrading android shell app
-  const toolsPublicDir = path.join(workingdir, 'tools-public');
-  await _installNodeModules(toolsPublicDir);
+  // TODO: remove this after making sure that we don't need node_modules in tools-public for older sdks
+  if (ExponentTools.parseSdkMajorVersion(sdkVersion) < 33) {
+    const toolsPublicDir = path.join(workingdir, 'tools-public');
+    await _installNodeModules(toolsPublicDir);
+  }
 }
 
 async function _installNodeModules(cwd: string) {

src/bin/setup/android/sdk.ts

@@ -52,7 +52,7 @@ async function _configureSdk(androidSdkDir: string) {
   await ExponentTools.spawnAsyncThrowError('./configureAndroidSdk.sh', [], {
     pipeToLogger: true,
     loggerFields: LOGGER_FIELDS,
-    cwd: path.join(config.directories.rootDir, 'scripts/cli'),
+    cwd: path.join(config.directories.rootDir, 'scripts/android'),
     env: {
       ...process.env,
       ANDROID_HOME: androidSdkDir,

src/bin/setup/utils/common.ts

@@ -17,7 +17,7 @@ interface IShellAppFormaters {
   formatShellAppTarballUriPath: (sdkMajorVersion: string) => string;
 }
 
-type PostDownloadAction = (workingdir: string) => void;
+type PostDownloadAction = (sdkVersion: string, workingdir: string) => void;
 
 export async function checkSystem(requiredTools: IToolDefinition[]) {
   await ensureToolsAreInstalled(requiredTools);
@@ -35,7 +35,7 @@ export async function ensureShellAppIsPresent(
   l.info(`shell app for SDK ${sdkVersion} doesn't exist, downloading...`);
   await _downloadShellApp(sdkVersion, workingdir, formatters);
   if (postDownloadAction) {
-    await postDownloadAction(workingdir);
+    await postDownloadAction(sdkVersion, workingdir);
   }
 }
 

src/bin/utils/builder.ts

@@ -1,10 +1,13 @@
+import { ExponentTools } from '@expo/xdl';
 import _ from 'lodash';
+import uuid from 'uuid';
 
 import { ErrorWithCommandHelp } from 'turtle/bin/commands/ErrorWithCommandHelp';
 import { ErrorWithProgramHelp } from 'turtle/bin/commands/ErrorWithProgramHelp';
 import setup from 'turtle/bin/setup/setup';
 import * as ProjectUtils from 'turtle/bin/utils/project';
 import * as UserUtils from 'turtle/bin/utils/user';
+import { getExperienceUrl } from 'turtle/builders/utils/common';
 import logger from 'turtle/logger';
 import { sanitizeJob } from 'turtle/validator';
 
@@ -12,7 +15,6 @@ export function createBuilderAction({
   program,
   command,
   prepareCredentials,
-  buildJobObject,
   builder,
   platform,
   os,
@@ -65,7 +67,7 @@ export function createBuilderAction({
       await setup(platform, sdkVersion);
       const credentials = await prepareCredentials(cmd);
       const rawJob = {
-        ...buildJobObject(appJSON, args, credentials),
+        ...await buildJobObject(platform, appJSON, args, credentials),
         ...cmd.buildDir && { fakeUploadDir: ProjectUtils.resolveAbsoluteDir(cmd.buildDir) },
         ...cmd.output && { fakeUploadBuildPath: ProjectUtils.resolveAbsoluteDir(cmd.output) },
       };
@@ -82,3 +84,37 @@ export function createBuilderAction({
     }
   };
 }
+
+const buildJobObject = async (
+  platform: 'android' | 'ios',
+  appJSON: any,
+  { releaseChannel, buildType, username, publicUrl, projectDir }: any,
+  credentials: any,
+) => {
+  const job = {
+    config: {
+      ..._.get(appJSON, 'expo.ios.config', {}),
+      buildType,
+      releaseChannel,
+      ...(platform === 'ios' ? { bundleIdentifier: _.get(appJSON, 'expo.ios.bundleIdentifier') } : {}),
+      ...(platform === 'android' ? { androidPackage: _.get(appJSON, 'expo.android.package') } : {}),
+      publicUrl,
+    },
+    id: uuid.v4(),
+    platform,
+    projectDir,
+    sdkVersion: _.get(appJSON, 'expo.sdkVersion'),
+    experienceName: `@${username}/${_.get(appJSON, 'expo.slug')}`,
+    ...(credentials && { credentials }),
+  };
+  const url = getExperienceUrl(job.experienceName, job.config.publicUrl);
+
+  const manifest = await ExponentTools.getManifestAsync(url, {
+    'Exponent-SDK-Version': _.get(appJSON, 'expo.sdkVersion'),
+    'Exponent-Platform': platform,
+    'Expo-Release-Channel': releaseChannel,
+    'Accept': 'application/expo+json,application/json',
+  });
+
+  return { ...job, manifest };
+};

src/builders/android.ts

@@ -75,12 +75,12 @@ async function runShellAppBuilder(
 
   logger.info({ buildPhase: 'resolve native modules' }, 'Resolving universal modules dependencies');
   const enabledModules = semver.satisfies(sdkVersion, '>= 33.0.0')
-    ? await resolveNativeModules(workingDir, manifest.dependencies)
+    ? await resolveNativeModules(workingDir, _.get(manifest, 'dependencies'))
     : null;
 
   try {
     await AndroidShellApp.createAndroidShellAppAsync({
-      url: commonUtils.getExperienceUrl(jobData),
+      url: commonUtils.getExperienceUrl(jobData.experienceName, jobData.config.publicUrl),
       sdkVersion,
       keystore: tempKeystorePath,
       manifest,

src/builders/utils/common.ts

@@ -1,13 +1,9 @@
 import * as url from 'url';
 
 import config from 'turtle/config';
-import { IJob } from 'turtle/job';
 import logger from 'turtle/logger';
 
-export function getExperienceUrl(job: IJob) {
-  const { experienceName, config: jobConfig } = job;
-  const { publicUrl } = jobConfig;
-
+export function getExperienceUrl(experienceName: string, publicUrl?: string) {
   // publicUrl is passed in if user wants to build an externally hosted app
   if (publicUrl) {
     return publicUrl;

src/builders/utils/ios/shellAppBuilder.ts

@@ -43,7 +43,7 @@ export default async function runShellAppBuilder(ctx: IContext, job: IJob): Prom
     });
   } else {
     Object.assign(shellAppParams, {
-      url: commonUtils.getExperienceUrl(job),
+      url: commonUtils.getExperienceUrl(job.experienceName, job.config.publicUrl),
       releaseChannel,
       sdkVersion,
     });

src/builders/utils/unimodules.ts

@@ -44,9 +44,11 @@ export async function resolveNativeModules(workingdir: string, packages?: string
 
 async function readCorePackages(workingdir: string) {
   const pkgJsonPath = path.join(workingdir, 'node_modules', 'react-native-unimodules', 'package.json');
-  if (await fs.pathExists(pkgJsonPath)) {
+  const expoPkgJsonPath = path.join(workingdir, 'packages', 'expo', 'package.json');
+  if (await fs.pathExists(pkgJsonPath) && await fs.pathExists(expoPkgJsonPath)) {
     const packageJson = JSON.parse(await fs.readFile(pkgJsonPath, 'utf-8'));
-    return keys(packageJson.dependencies);
+    const expoPackageJson = JSON.parse(await fs.readFile(expoPkgJsonPath, 'utf-8'));
+    return keys(packageJson.dependencies).concat(keys(expoPackageJson.dependencies));
   } else {
     logger.warn('No core packages detected');
     return [];

src/scripts/setConfigurations.ts

@@ -43,13 +43,13 @@ const AVAILABLE_CONFIGURATIONS_ANDROID = [
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
+  HIGH_CONFIGURATION,
+  HIGH_CONFIGURATION,
+  HIGH_CONFIGURATION,
+  HIGH_CONFIGURATION,
+  HIGH_CONFIGURATION,
+  HIGH_CONFIGURATION,
+  HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,

supervisord.conf

@@ -0,0 +1,13 @@
+[supervisord]
+nodaemon=true
+
+[program:turtle]
+directory=/app/turtle
+command=node ./build/server.js
+autorestart=true
+startretries=5
+startsecs=600
+
+[eventlistener:turtle_failure]
+events=PROCESS_STATE_FATAL
+command=/app/turtle/scripts/android/supervisordEventListener.sh