[turtle-cli] don't require gulp-cli installed globally

* add supervisord

- restart pod only if there is at least 5 failures in 10 minutes

* Update CHANGELOG.md

Co-Authored-By: Dominik Sokal <[email protected]>

* Update scripts/android/supervisordEventListener.sh

Co-Authored-By: Dominik Sokal <[email protected]>

CHANGELOG.md

@@ -6,6 +6,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.8.4] - 2019-06-13
+### Changed
+- Upgraded `sharp` from `0.21.3` to `0.22.1`.
+
+## [0.8.3] - 2019-06-13
+### Added
+- Running Android Turtle process with `supervisor` (pod is restarted if there were 5 failures in the last 10 minutes).
+### Changed
+- `turtle-cli` is now fetching the app manifest before running the build.
+
 ## [0.8.2] - 2019-06-11
 ### Changed
 - Updated Android shell app for SDK33 to remove `expo-face-detector` and `expo-payments-stripe` from standalone builds.
@@ -24,7 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Turtle CLI logo to the project (created by [@zularizal](https://github.com/zularizal))!
 - Shell apps for SDK 33.
-- Support for Android App Bundles.
+- Support for Android App Bundles (App bundles are built by default, use `-t apk` to build apk).
 ### Changed
 - Updated Android shell app for SDK 32 to fix AppAuth issue (https://github.com/expo/expo/pull/4115) and to fix a newer than expected JSC version.
 - Default value for build timeout to 15 minutes.

Dockerfile

@@ -24,6 +24,7 @@ RUN dpkg --add-architecture i386 && \
   unzip\
   yarn\
   zlib1g:i386\
+  supervisor\
   --no-install-recommends && \
   apt-get clean
 
@@ -51,8 +52,8 @@ ENV PATH ${ANDROID_HOME}/tools:${PATH}
 ENV PATH ${ANDROID_HOME}/tools/bin:${PATH}
 ENV PATH ${ANDROID_HOME}/build-tools/28.0.3/:${PATH}
 
-ADD scripts/cli/configureAndroidSdk.sh /tmp/configureAndroidSdk.sh
-RUN /tmp/configureAndroidSdk.sh
+ADD scripts/android/configureAndroidSdk.sh /tmp/configureAndroidSdk.sh
+RUN /tmp/configureAndroidSdk.sh --install-all-platforms
 
 # Install Android NDK
 ENV ANDROID_NDK_VERSION android-ndk-r10e
@@ -111,4 +112,6 @@ ENV TURTLE_WORKING_DIR_PATH /app/turtle/workingdir/
 
 WORKDIR /app/turtle
 
-CMD ["node", "./build/server.js"]
+ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+CMD ["/usr/bin/supervisord"]

package.json

@@ -1,6 +1,6 @@
 {
   "name": "turtle-cli",
-  "version": "0.8.2",
+  "version": "0.8.4",
   "description": "The command-line tool for building standalone Expo apps",
   "preferGlobal": true,
   "scripts": {
@@ -59,7 +59,7 @@
   "files": [
     "bin",
     "build",
-    "scripts/cli",
+    "scripts/android/configureAndroidSdk.sh",
     "shellTarballs"
   ],
   "keywords": [
@@ -131,7 +131,7 @@
     "request": "^2.88.0",
     "request-progress": "^3.0.0",
     "semver": "^6.0.0",
-    "sharp": "^0.21.0",
+    "sharp": "0.22.1",
     "source-map-support": "^0.5.6",
     "tar": "^4.4.6",
     "uuid": "^3.2.1",

scripts/android/configureAndroidSdk.sh

@@ -3,6 +3,14 @@
 SDKMANAGER=$ANDROID_HOME/tools/bin/sdkmanager
 
 yes | $SDKMANAGER --licenses > /dev/null
+if [[ "$1" = "--install-all-platforms" ]]; then
+  yes | $SDKMANAGER \
+    "platforms;android-23" \
+    "platforms;android-24" \
+    "platforms;android-25" \
+    "platforms;android-26" \
+    "platforms;android-27" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]
+fi
 yes | $SDKMANAGER "platforms;android-28" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]
 yes | $SDKMANAGER "platform-tools" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]
 yes | $SDKMANAGER "build-tools;28.0.3" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]

scripts/android/supervisordEventListener.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+printf "READY\n"; # signal to superivsord that listener is ready
+
+while read line; do
+  echo "[supervisord:eventlistener] Processing Event: $line" >&2;
+  kill -SIGQUIT $PPID
+done < /dev/stdin

src/bin/commands/build/android.ts

@@ -1,6 +1,5 @@
 import fs from 'fs-extra';
 import _ from 'lodash';
-import uuid from 'uuid';
 
 import { ErrorWithCommandHelp } from 'turtle/bin/commands/ErrorWithCommandHelp';
 import { createBuilderAction } from 'turtle/bin/utils/builder';
@@ -33,7 +32,6 @@ export default (program: any, setCommonCommandOptions: any) => {
         program,
         command,
         prepareCredentials,
-        buildJobObject,
         builder,
         platform: PLATFORMS.ANDROID,
         os: ['darwin', 'linux'],
@@ -41,26 +39,6 @@ export default (program: any, setCommonCommandOptions: any) => {
     );
 };
 
-const buildJobObject = (
-  appJSON: any,
-  { releaseChannel, username, projectDir, publicUrl, buildType }: any,
-  credentials: any,
-) => ({
-  config: {
-    ..._.get(appJSON, 'expo.android.config', {}),
-    releaseChannel,
-    androidPackage: _.get(appJSON, 'expo.android.package'),
-    publicUrl,
-    buildType,
-  },
-  id: uuid.v4(),
-  platform: PLATFORMS.ANDROID,
-  sdkVersion: _.get(appJSON, 'expo.sdkVersion'),
-  projectDir,
-  experienceName: `@${username}/${_.get(appJSON, 'expo.slug')}`,
-  ...(credentials && { credentials }),
-});
-
 const prepareCredentials = async (cmd: any) => {
   const { keystorePath, keystoreAlias } = cmd;
   const keystorePassword = process.env.EXPO_ANDROID_KEYSTORE_PASSWORD;

src/bin/commands/build/ios.ts

@@ -1,6 +1,5 @@
 import fs from 'fs-extra';
 import _ from 'lodash';
-import uuid from 'uuid';
 
 import { ErrorWithCommandHelp } from 'turtle/bin/commands/ErrorWithCommandHelp';
 import { createBuilderAction } from 'turtle/bin/utils/builder';
@@ -32,7 +31,6 @@ export default (program: any, setCommonCommandOptions: any) => {
         program,
         command,
         prepareCredentials,
-        buildJobObject,
         builder,
         platform: PLATFORMS.IOS,
         os: 'darwin',
@@ -40,21 +38,6 @@ export default (program: any, setCommonCommandOptions: any) => {
     );
 };
 
-const buildJobObject = (appJSON: any, { releaseChannel, buildType, username, publicUrl }: any, credentials: any) => ({
-  config: {
-    ..._.get(appJSON, 'expo.ios.config', {}),
-    buildType,
-    releaseChannel,
-    bundleIdentifier: _.get(appJSON, 'expo.ios.bundleIdentifier'),
-    publicUrl,
-  },
-  id: uuid.v4(),
-  platform: PLATFORMS.IOS,
-  sdkVersion: _.get(appJSON, 'expo.sdkVersion'),
-  experienceName: `@${username}/${_.get(appJSON, 'expo.slug')}`,
-  ...(credentials && { credentials }),
-});
-
 const prepareCredentials = async (cmd: any) => {
   if (cmd.type !== IOS_BUILD_TYPES.ARCHIVE) {
     return null;

src/bin/setup/android/index.ts

@@ -20,10 +20,6 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
     command: 'bash',
     missingDescription: 'Please install bash',
   },
-  {
-    command: 'gulp',
-    missingDescription: 'Run `npm install -g gulp-cli` (or `yarn global add gulp-cli`) to install gulp',
-  },
   {
     command: 'javac',
     missingDescription: 'Please install JDK (version 8 or newer) - check https://jdk.java.net/',

src/bin/setup/android/sdk.ts

@@ -52,7 +52,7 @@ async function _configureSdk(androidSdkDir: string) {
   await ExponentTools.spawnAsyncThrowError('./configureAndroidSdk.sh', [], {
     pipeToLogger: true,
     loggerFields: LOGGER_FIELDS,
-    cwd: path.join(config.directories.rootDir, 'scripts/cli'),
+    cwd: path.join(config.directories.rootDir, 'scripts/android'),
     env: {
       ...process.env,
       ANDROID_HOME: androidSdkDir,

src/bin/utils/builder.ts

@@ -1,10 +1,13 @@
+import { ExponentTools } from '@expo/xdl';
 import _ from 'lodash';
+import uuid from 'uuid';
 
 import { ErrorWithCommandHelp } from 'turtle/bin/commands/ErrorWithCommandHelp';
 import { ErrorWithProgramHelp } from 'turtle/bin/commands/ErrorWithProgramHelp';
 import setup from 'turtle/bin/setup/setup';
 import * as ProjectUtils from 'turtle/bin/utils/project';
 import * as UserUtils from 'turtle/bin/utils/user';
+import { getExperienceUrl } from 'turtle/builders/utils/common';
 import logger from 'turtle/logger';
 import { sanitizeJob } from 'turtle/validator';
 
@@ -12,7 +15,6 @@ export function createBuilderAction({
   program,
   command,
   prepareCredentials,
-  buildJobObject,
   builder,
   platform,
   os,
@@ -65,7 +67,7 @@ export function createBuilderAction({
       await setup(platform, sdkVersion);
       const credentials = await prepareCredentials(cmd);
       const rawJob = {
-        ...buildJobObject(appJSON, args, credentials),
+        ...await buildJobObject(platform, appJSON, args, credentials),
         ...cmd.buildDir && { fakeUploadDir: ProjectUtils.resolveAbsoluteDir(cmd.buildDir) },
         ...cmd.output && { fakeUploadBuildPath: ProjectUtils.resolveAbsoluteDir(cmd.output) },
       };
@@ -82,3 +84,37 @@ export function createBuilderAction({
     }
   };
 }
+
+const buildJobObject = async (
+  platform: 'android' | 'ios',
+  appJSON: any,
+  { releaseChannel, buildType, username, publicUrl, projectDir }: any,
+  credentials: any,
+) => {
+  const job = {
+    config: {
+      ..._.get(appJSON, 'expo.ios.config', {}),
+      buildType,
+      releaseChannel,
+      ...(platform === 'ios' ? { bundleIdentifier: _.get(appJSON, 'expo.ios.bundleIdentifier') } : {}),
+      ...(platform === 'android' ? { androidPackage: _.get(appJSON, 'expo.android.package') } : {}),
+      publicUrl,
+    },
+    id: uuid.v4(),
+    platform,
+    projectDir,
+    sdkVersion: _.get(appJSON, 'expo.sdkVersion'),
+    experienceName: `@${username}/${_.get(appJSON, 'expo.slug')}`,
+    ...(credentials && { credentials }),
+  };
+  const url = getExperienceUrl(job.experienceName, job.config.publicUrl);
+
+  const manifest = await ExponentTools.getManifestAsync(url, {
+    'Exponent-SDK-Version': _.get(appJSON, 'expo.sdkVersion'),
+    'Exponent-Platform': platform,
+    'Expo-Release-Channel': releaseChannel,
+    'Accept': 'application/expo+json,application/json',
+  });
+
+  return { ...job, manifest };
+};

src/builders/android.ts

@@ -75,12 +75,12 @@ async function runShellAppBuilder(
 
   logger.info({ buildPhase: 'resolve native modules' }, 'Resolving universal modules dependencies');
   const enabledModules = semver.satisfies(sdkVersion, '>= 33.0.0')
-    ? await resolveNativeModules(workingDir, manifest && manifest.dependencies)
+    ? await resolveNativeModules(workingDir, _.get(manifest, 'dependencies'))
     : null;
 
   try {
     await AndroidShellApp.createAndroidShellAppAsync({
-      url: commonUtils.getExperienceUrl(jobData),
+      url: commonUtils.getExperienceUrl(jobData.experienceName, jobData.config.publicUrl),
       sdkVersion,
       keystore: tempKeystorePath,
       manifest,

src/builders/utils/common.ts

@@ -1,13 +1,9 @@
 import * as url from 'url';
 
 import config from 'turtle/config';
-import { IJob } from 'turtle/job';
 import logger from 'turtle/logger';
 
-export function getExperienceUrl(job: IJob) {
-  const { experienceName, config: jobConfig } = job;
-  const { publicUrl } = jobConfig;
-
+export function getExperienceUrl(experienceName: string, publicUrl?: string) {
   // publicUrl is passed in if user wants to build an externally hosted app
   if (publicUrl) {
     return publicUrl;

src/builders/utils/ios/shellAppBuilder.ts

@@ -43,7 +43,7 @@ export default async function runShellAppBuilder(ctx: IContext, job: IJob): Prom
     });
   } else {
     Object.assign(shellAppParams, {
-      url: commonUtils.getExperienceUrl(job),
+      url: commonUtils.getExperienceUrl(job.experienceName, job.config.publicUrl),
       releaseChannel,
       sdkVersion,
     });

src/jobManager.ts

@@ -64,7 +64,10 @@ async function processJob(jobData: any) {
     setCurrentJobId(job.id);
     const pingerHandle = sqs.changeMessageVisibilityRecurring(priority, jobData.ReceiptHandle, job.id);
 
-    logger.info(`Doing job MessageId=${jobData.MessageId} BuildId=${job.id} ${Date.now()}`);
+    logger.info(
+      `Doing job MessageId=${jobData.MessageId} BuildId=${job.id} `
+      + `Priority=${priority.slice(0, priority.length - 8)} ${Date.now()}`,
+    );
 
     const cancelled = await redis.checkIfCancelled(job.id);
     if (cancelled) {

supervisord.conf

@@ -0,0 +1,13 @@
+[supervisord]
+nodaemon=true
+
+[program:turtle]
+directory=/app/turtle
+command=node ./build/server.js
+autorestart=true
+startretries=5
+startsecs=600
+
+[eventlistener:turtle_failure]
+events=PROCESS_STATE_FATAL
+command=/app/turtle/scripts/android/supervisordEventListener.sh

yarn.lock

@@ -2803,13 +2803,6 @@ binary-extensions@^1.0.0:
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.13.0.tgz#9523e001306a32444b907423f1de2164222f6ab1"
   integrity sha512-EgmjVLMn22z7eGGv3kcnHwSnJXmFHjISTY9E/S5lIcTD3Oxw05QTcBLNkJFzcb3cNueUdF/IN4U+d78V0zO8Hw==
 
-bindings@^1.3.1:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.4.0.tgz#909efa49f2ebe07ecd3cb136778f665052040127"
-  integrity sha512-7znEVX22Djn+nYjxCWKDne0RRloa9XfYa84yk3s+HkE3LpDYZmhArYr9O9huBoHY3/oXispx5LorIX7Sl2CgSQ==
-  dependencies:
-    file-uri-to-path "1.0.0"
-
 bl@^1.0.0:
   version "1.2.2"
   resolved "https://registry.yarnpkg.com/bl/-/bl-1.2.2.tgz#a160911717103c07410cef63ef51b397c025af9c"
@@ -3663,7 +3656,7 @@ color-string@^1.5.2:
     color-name "^1.0.0"
     simple-swizzle "^0.2.2"
 
-color@^3.0.0, color@^3.1.0:
+color@^3.0.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/color/-/color-3.1.0.tgz#d8e9fb096732875774c84bf922815df0308d0ffc"
   integrity sha512-CwyopLkuRYO5ei2EpzpIh6LqJMt6Mt+jZhO5VI5f/wJLZriXQE32/SSqzmrh+QB+AZT81Cj8yv+7zwToW8ahZg==
@@ -5629,11 +5622,6 @@ file-type@^9.0.0:
   resolved "https://registry.yarnpkg.com/file-type/-/file-type-9.0.0.tgz#a68d5ad07f486414dfb2c8866f73161946714a18"
   integrity sha512-Qe/5NJrgIOlwijpq3B7BEpzPFcgzggOTagZmkXQY4LA6bsXKTUstK7Wp12lEJ/mLKTpvIZxmIuRcLYWT6ov9lw==
 
-[email protected]:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd"
-  integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==
-
 filename-regex@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26"
@@ -10784,28 +10772,6 @@ postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.14, postcss@^7.0.5, postcss@^7.0.6:
     source-map "^0.6.1"
     supports-color "^6.1.0"
 
-prebuild-install@^5.2.2:
-  version "5.2.4"
-  resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-5.2.4.tgz#8cc41a217ef778a31d3a876fe6668d05406db750"
-  integrity sha512-CG3JnpTZXdmr92GW4zbcba4jkDha6uHraJ7hW4Fn8j0mExxwOKK20hqho8ZuBDCKYCHYIkFM1P2jhtG+KpP4fg==
-  dependencies:
-    detect-libc "^1.0.3"
-    expand-template "^2.0.3"
-    github-from-package "0.0.0"
-    minimist "^1.2.0"
-    mkdirp "^0.5.1"
-    napi-build-utils "^1.0.1"
-    node-abi "^2.7.0"
-    noop-logger "^0.1.1"
-    npmlog "^4.0.1"
-    os-homedir "^1.0.1"
-    pump "^2.0.1"
-    rc "^1.2.7"
-    simple-get "^2.7.0"
-    tar-fs "^1.13.0"
-    tunnel-agent "^0.6.0"
-    which-pm-runs "^1.0.0"
-
 prebuild-install@^5.2.4:
   version "5.2.5"
   resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-5.2.5.tgz#c7485911fe98950b7f7cd15bb9daee11b875cc44"
@@ -12082,24 +12048,7 @@ sha.js@^2.4.0, sha.js@^2.4.8:
     inherits "^2.0.1"
     safe-buffer "^5.0.1"
 
-sharp@^0.21.0:
-  version "0.21.3"
-  resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.21.3.tgz#381937de66c123687f2ac7186f85921c6bb19cdd"
-  integrity sha512-5qZk8r+YgfyztLEKkNez20Wynq/Uh1oNyP5T/3gTYwt2lBYGs9iDs5m0yVsZEPm8eVBbAJhS08J1wp/g+Ai1Qw==
-  dependencies:
-    bindings "^1.3.1"
-    color "^3.1.0"
-    detect-libc "^1.0.3"
-    fs-copy-file-sync "^1.1.1"
-    nan "^2.12.1"
-    npmlog "^4.1.2"
-    prebuild-install "^5.2.2"
-    semver "^5.6.0"
-    simple-get "^3.0.3"
-    tar "^4.4.8"
-    tunnel-agent "^0.6.0"
-
-sharp@^0.22.1:
+[email protected]2.1, sharp@^0.22.1:
   version "0.22.1"
   resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.22.1.tgz#a67c0e75567f03dd5a7861b901fec04072c5b0f4"
   integrity sha512-lXzSk/FL5b/MpWrT1pQZneKe25stVjEbl6uhhJcTULm7PhmJgKKRbTDM/vtjyUuC/RLqL2PRyC4rpKwbv3soEw==