Release 0.8.8

* add circleci jobs for turtle deployment

* add checkout step

* remove redundant npm scripts

* remove docker:* and deploy:android npm scripts

* improve docs and remove duplicated scripts

.circleci/config.yml

@@ -137,6 +137,23 @@ commands:
           paths:
             - "<< parameters.workingdir >>/<< parameters.platform >>/<< parameters.sdk_version >>"
 
+  deploy_turtle:
+    parameters:
+      platform:
+        type: string
+      env:
+        type: string
+    steps:
+      - checkout
+      - run:
+          name: Install curl & jq
+          command: nix-env -iA nixpkgs.curl nixpkgs.jq
+      - run:
+          name: Schedule a Turtle deploy
+          command: |
+            echo "We're going to deploy << parameters.platform >> Turtles to << parameters.env >>"
+            ./scripts/deploy.sh "<< parameters.platform >>" "<< parameters.env >>"
+
 jobs:
   build:
     executor: nix
@@ -203,6 +220,34 @@ jobs:
           sdk_version: sdk32
           platform: android
 
+  android_staging_deploy:
+    executor: nix
+    steps:
+      - deploy_turtle:
+          platform: android
+          env: staging
+
+  android_production_deploy:
+    executor: nix
+    steps:
+      - deploy_turtle:
+          platform: android
+          env: production
+
+  ios_staging_deploy:
+    executor: nix
+    steps:
+      - deploy_turtle:
+          platform: ios
+          env: staging
+
+  ios_production_deploy:
+    executor: nix
+    steps:
+      - deploy_turtle:
+          platform: ios
+          env: production
+
 workflows:
   checks:
     jobs:
@@ -212,3 +257,25 @@ workflows:
       - unit_tests
       - ios_smoke_tests
       - android_smoke_tests
+  deploy:
+    jobs:
+      - android_staging_approve_deploy:
+          type: approval
+      - android_staging_deploy:
+          requires:
+            - android_staging_approve_deploy
+      - android_production_approve_deploy:
+          type: approval
+      - android_production_deploy:
+          requires:
+            - android_production_approve_deploy
+      - ios_staging_approve_deploy:
+          type: approval
+      - ios_staging_deploy:
+          requires:
+            - ios_staging_approve_deploy
+      - ios_production_approve_deploy:
+          type: approval
+      - ios_production_deploy:
+          requires:
+            - ios_production_approve_deploy

.envrc

@@ -8,16 +8,13 @@ source_local() {
   watch_file $file
 }
 
-source_up || true
 source_local
 
 export NODE_ENV=development
 
 export EXPO_TURTLE_TOOLS_DIR="$(pwd)/tools"
 
-if [ -z "$EXPO_UNIVERSE_DIR" ]; then
-  PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
-  if [ "$CI" = "true" ]; then
-    PATH_add $EXPO_TURTLE_TOOLS_DIR/ci/bin
-  fi
+PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
+if [ "$CI" = "true" ]; then
+  PATH_add $EXPO_TURTLE_TOOLS_DIR/ci/bin
 fi

CHANGELOG.md

@@ -5,10 +5,56 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+## [0.8.8] - 2019-07-11
+### Fixed
+- LocalAuthentication module on iOS (fixes [expo/expo#663](https://github.com/expo/expo/issues/663)).
+
+## [0.8.7] - 2019-07-05
+### Added
+- Adhoc Builds: use provisioningProfileId if specified.
+- Registering supported SDK versions in Redis.
+- Support for 64 bit builds in SDK33.
+
+## [0.8.6] - 2019-06-25
+### Added
+- Added support for Android build modes: debug or release.
+### Fixed
+- Upgrading the shell app for a given SDK version when the previous version is already downloaded (bug in `turtle-cli`).
+- Updated Android shell app for SDK33 to include `[email protected]` which fixes crashes on Android 9.0 when showing some Lottie animations.
+- Schema for `buildModes`.
+- Updated @expo/xdl to 55.0.7 (fixed reading from binary plist).
+
+## [0.8.4] - 2019-06-13
+### Changed
+- Upgraded `sharp` from `0.21.3` to `0.22.1`.
+- `gulp-cli` is no longer required to build an app with `turtle-cli`.
+
+## [0.8.3] - 2019-06-13
+### Added
+- Running Android Turtle process with `supervisor` (pod is restarted if there were 5 failures in the last 10 minutes).
+### Changed
+- `turtle-cli` is now fetching the app manifest before running the build.
+
+## [0.8.2] - 2019-06-11
+### Changed
+- Updated Android shell app for SDK33 to remove `expo-face-detector` and `expo-payments-stripe` from standalone builds.
+- Updated Android shell app for SDK33 to get rid of `node_modules` for `tools-public`.
+- From now on, `turtle-cli` installs dependencies for `tools-public` in shell apps for SDK versions < 33.
+### Fixed
+- Android builds with turtle-cli by installing all unmodules if `dependencies` is empty.
+
+## [0.8.1] - 2019-06-10
+### Changed
+- Add expo dependencies when resolving unimodules.
+### Fixed
+- Support for Android tarballs without `universe-package.json`.
+
+## [0.8.0] - 2019-06-07
 ### Added
 - Turtle CLI logo to the project (created by [@zularizal](https://github.com/zularizal))!
-- Added shell apps for SDK 33.
-- Support for Android App Bundles
+- Shell apps for SDK 33.
+- Support for Android App Bundles (App bundles are built by default, use `-t apk` to build apk).
 ### Changed
 - Updated Android shell app for SDK 32 to fix AppAuth issue (https://github.com/expo/expo/pull/4115) and to fix a newer than expected JSC version.
 - Default value for build timeout to 15 minutes.
@@ -18,7 +64,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Generating Android keystore in turtle when one is not passed to turtle-cli (https://github.com/expo/turtle/pull/65).
 - Upload logs to s3 if build has been canceled or timed out.
 - Unhandled exception on asset download (xdl).
-- Fixed support for Android tarballs without `universe-package.json`.
 
 ## [0.7.2] - 2019-05-09
 ### Added
@@ -91,7 +136,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.5.10] - 2019-02-19
 ### Added
-- Register Turtle version in Redis.
+- Registering Turtle version in Redis.
 ### Changed
 - Update Android shell app for SDK 32 to fix [update](https://github.com/expo/expo/issues/3504) and [back button](https://github.com/expo/expo/issues/1786) issues.
 

CONTRIBUTING.md

@@ -16,9 +16,6 @@ This repository contains the codebase for the `turtle-cli` NPM package. The same
 ### Branches
 The `master` branch of the repository should be kept releasable at any time. This way we can continuously release fixes and improvements without costly managing of different branches and issues will be noticed and fixed quickly. This also ensures other contributors can check out the latest version from GitHub and work on it with minimal disruption from other features in progress.
 
-## Setting up the repository for development
-*TODO*
-
 ## Submitting a pull request
 To submit a pull request:
 1. Fork the [repository](https://github.com/expo/turtle) and create a feature branch. (Existing contributors can create feature branches without forking. Prefix the branch name with `@your-github-username/`.)

DEVELOPMENT.md

@@ -0,0 +1,35 @@
+# Turtle Development
+
+## Prerequisites
+
+- Working `www` service, see [this README](https://github.com/expo/universe/tree/master/server/www/README.md) for more details.
+- Own AWS SQS queues. See [this README](https://github.com/expo/universe/tree/master/terraform/turtle-queues/README.md) to learn more.
+
+## Setup
+
+- Run `update-local-secrets` to populate the `.secrets*` files. You'll need to run this any time any secret changes (it shouldn't be very often).
+- Run `EXPO_UNIVERSE_DIR=/path/to/universe/dir yarn secrets:init-private YOUR_SQS_NICKNAME`. Alternatively, you can create `.secrets.local-queue` file with the SQS queues' urls yourself (`AWS_SQS_OUT_QUEUE_URL`, `AWS_SQS_ANDROID_QUEUE_URL`, `AWS_SQS_ANDROID_PRIORITY_QUEUE_URL`, `AWS_SQS_IOS_QUEUE_URL`, `AWS_SQS_IOS_PRIORITY_QUEUE_URL`)
+- Run `yarn set-workingdir:local` or `yarn set-workingdir:remote`:
+  - `:local` creates symlinks to the shell apps in `expo` repository (_unfortunately this is currently broken_)
+  - `:remote` downloads precompiled shell apps
+- Run `yarn` to install node modules.
+- Run `yarn start:ios` or `yarn start:android` to run Turtle.
+
+## Updating shell apps
+
+_TODO_
+
+## Generating recording files for smoke tests
+
+Smoke tests are powered by [Nock](https://www.npmjs.com/package/nock) which uses JSON files containing responses to the HTTP requests fired by the build process.
+
+To record new responses remove old `recording.json` and run tests and then manually schedule a build with `expo-cli` tool.
+
+As a precaution, the file with HTTP requests/responses contains, among other things, iOS certificates used for the test build, is
+encrypted with `git-crypt`. Remember not to revoke them, because then the tests will fail.
+
+## Other notes
+
+Turtle uses Redis to get notifications (from www) about builds cancellations. To make this feature working, you have to use the same Redis instance as www. Therefore, you need to remember about starting www before starting Turtle. If you forget about it, don't worry, Turtle will remind you to run `yarn start-docker` in `server/www` directory.
+
+Turtle uploads build artifacts to S3. This could take some time if you have a poor internet connection. To facilitate local development, there is a feature which allows Turtle to fake artifacts upload. `secrets:init-private` script, which you should have run while setting up Turtle, sets fake upload directory to your user's default `Downloads` directory (we're assuming you're using Mac). If you need to test uploading artifacts to S3, you have to edit `.secrets.local` file and comment or remove `TURTLE_FAKE_UPLOAD` and `TURTLE_FAKE_UPLOAD_DIR` environment variables.

Dockerfile

@@ -24,6 +24,7 @@ RUN dpkg --add-architecture i386 && \
   unzip\
   yarn\
   zlib1g:i386\
+  supervisor\
   --no-install-recommends && \
   apt-get clean
 
@@ -51,20 +52,8 @@ ENV PATH ${ANDROID_HOME}/tools:${PATH}
 ENV PATH ${ANDROID_HOME}/tools/bin:${PATH}
 ENV PATH ${ANDROID_HOME}/build-tools/28.0.3/:${PATH}
 
-RUN yes | sdkmanager --licenses > /dev/null
-RUN yes | sdkmanager \
-  "platforms;android-23" \
-  "platforms;android-24" \
-  "platforms;android-25" \
-  "platforms;android-26" \
-  "platforms;android-27" \
-  "platforms;android-28" | grep -v '=' || echo 'no output'
-RUN yes | sdkmanager "platform-tools" | grep -v '=' || echo 'no output'
-RUN yes | sdkmanager "build-tools;28.0.3" | grep -v '=' || echo 'no output'
-RUN yes | sdkmanager \
-  "extras;android;m2repository" \
-  "extras;google;m2repository" \
-  "extras;google;google_play_services" | grep -v '=' || echo 'no output'
+ADD scripts/android/configureAndroidSdk.sh /tmp/configureAndroidSdk.sh
+RUN /tmp/configureAndroidSdk.sh --install-all-platforms
 
 # Install Android NDK
 ENV ANDROID_NDK_VERSION android-ndk-r10e
@@ -123,4 +112,6 @@ ENV TURTLE_WORKING_DIR_PATH /app/turtle/workingdir/
 
 WORKDIR /app/turtle
 
-CMD ["node", "./build/server.js"]
+ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+CMD ["/usr/bin/supervisord"]

README.md

@@ -15,21 +15,20 @@ Turtle CLI is a command line interface for building Expo standalone apps.
 You can use it both on your CI and your private computer.
 
 - [Changelog](CHANGELOG.md)
-- [Contributing to Turtle CLI](CONTRIBUTING.md)
+- [Contributing](CONTRIBUTING.md)
+- [Development](DEVELOPMENT.md)
 - [Code of Conduct](CODE_OF_CONDUCT.md)
 
-## Glossary
-- **Turtle** - script which takes a shell app, fills it with user's code and produces a standalone app
-- **Shell app** — precompiled (in case of iOS builds) Expo app which doesn't have user's code
-- **Standalone app** - .ipa/.apk file, a user uploads standalone apps to the Google Play Store/Apple App Store
-
 ## Installation
+
 [Installation instructions and documentation here.](https://docs.expo.io/versions/latest/distribution/turtle-cli)
 
 ## Usage example
+
 See the [expo/turtle-cli-example](https://github.com/expo/turtle-cli-example) repository to learn how to use Turtle CLI in order to build Expo standalone apps on your CI.
 
 ## Do you need help?
+
 - Make sure you've read [the Expo Docs](https://docs.expo.io/), especially [Building Standalone Apps on Your CI
 ](https://docs.expo.io/versions/latest/distribution/turtle-cli) and [Building Standalone Apps](https://docs.expo.io/versions/latest/distribution/building-standalone-apps) guides.
 - Search for existing GitHub issues [in this repository](https://github.com/expo/turtle/issues) and in [expo/expo](https://github.com/expo/expo/issues) repository.
@@ -39,5 +38,6 @@ See the [expo/turtle-cli-example](https://github.com/expo/turtle-cli-example) re
   * ask a question on [Expo Developers Slack](http://slack.expo.io).
 
 ## License
+
 The Turtle CLI source code is made available under the [MIT license](LICENSE).
 Some of the dependencies are licensed differently, with the BSD license, for example.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "turtle-cli",
-  "version": "0.8.0",
+  "version": "0.8.8",
   "description": "The command-line tool for building standalone Expo apps",
   "preferGlobal": true,
   "scripts": {
@@ -13,14 +13,10 @@
     "prepare": "rimraf ./build && yarn build",
     "build": "ttsc",
     "build:production": "./scripts/build.sh",
+    "watch": "ttsc --watch",
     "secrets:init-private": "./scripts/initPrivateSecrets.sh",
     "set-workingdir:remote": "./scripts/setWorkingdirMode.sh remote",
     "set-workingdir:local": "./scripts/setWorkingdirMode.sh local",
-    "docker:pull": "retry5 gcloud docker -- pull gcr.io/exponentjs/turtle-android:latest",
-    "docker:pull:tag": "retry5 gcloud docker -- pull gcr.io/exponentjs/turtle-android:${TAG}",
-    "docker:build": "yarn build:production android && gcloud docker -- build -t gcr.io/exponentjs/turtle-android:latest -t gcr.io/exponentjs/turtle-android:${TAG} .",
-    "docker:push": "retry5 gcloud docker -- push gcr.io/exponentjs/turtle-android:${TAG} && retry5 gcloud docker -- push gcr.io/exponentjs/turtle-android:latest",
-    "deploy:android": "./scripts/android/deploy.sh",
     "test": "yarn jest",
     "test:integration": "yarn test --config jest/integration-config.json",
     "test:unit": "yarn test --config jest/unit-config.json",
@@ -28,19 +24,7 @@
     "test:smoke:android": "PLATFORM=android yarn test:smoke --testNamePattern=android",
     "test:smoke:ios": "PLATFORM=ios yarn test:smoke --testNamePattern=iOS",
     "release": "./scripts/release.sh",
-    "release:beta": "./scripts/releaseBeta.sh",
-    "ssh:android:pods-names": "./scripts/ssh/findAndroidPods.sh",
-    "ssh:android:connect": "./scripts/ssh/connectToAndroidPod.sh",
-    "ssh:ios:ips": "./scripts/ssh/findIosIps.sh",
-    "set-priorities": "yarn set-priorities:local",
-    "set-priorities:default": "expenv ts-node -r tsconfig-paths/register --files src/scripts/setConfigurations.ts",
-    "set-priorities:local": "EXPENV_OVERRIDES=local-common,local,local-queue yarn set-priorities:default",
-    "set-priorities:staging": "yarn set-priorities:default",
-    "set-priorities:production": "EXPENV_OVERRIDES=prod yarn set-priorities:default",
-    "set-replica-count:default": "expenv ts-node -r tsconfig-paths/register --files src/scripts/setReplicaCount.ts | bunyan -o short -l debug",
-    "set-replica-count:local": "EXPENV_OVERRIDES=local-common,local,local-queue yarn set-replica-count:default",
-    "set-replica-count:staging": "yarn set-replica-count:default",
-    "set-replica-count:production": "EXPENV_OVERRIDES=prod yarn set-replica-count:default"
+    "release:beta": "./scripts/releaseBeta.sh"
   },
   "husky": {
     "hooks": {
@@ -59,7 +43,7 @@
   "files": [
     "bin",
     "build",
-    "scripts/cli",
+    "scripts/android/configureAndroidSdk.sh",
     "shellTarballs"
   ],
   "keywords": [
@@ -79,11 +63,13 @@
     "registry": "https://registry.npmjs.org/"
   },
   "devDependencies": {
+    "@expo/babel-preset-cli": "^0.1.3",
     "@types/decompress": "^4.2.3",
     "@types/fs-extra": "^5.0.4",
     "@types/jest": "^23.3.9",
     "@types/joi": "^13.4.3",
     "@types/lodash": "^4.14.129",
+    "@types/npm-package-arg": "^6.1.0",
     "@types/progress": "^2.0.1",
     "@types/raven": "^2.5.1",
     "@types/request": "^2.47.1",
@@ -110,7 +96,7 @@
   },
   "dependencies": {
     "@expo/spawn-async": "^1.4.2",
-    "@expo/xdl": "54.1.4",
+    "@expo/xdl": "56.0.1-rc.0",
     "@google-cloud/logging-bunyan": "^0.9.5",
     "async-retry": "^1.2.1",
     "aws-sdk": "^2.226.1",
@@ -131,13 +117,13 @@
     "request": "^2.88.0",
     "request-progress": "^3.0.0",
     "semver": "^6.0.0",
-    "sharp": "^0.21.0",
+    "sharp": "0.22.1",
     "source-map-support": "^0.5.6",
     "tar": "^4.4.6",
     "uuid": "^3.2.1",
     "which": "^1.3.1"
   },
   "optionalDependencies": {
-    "@expo/traveling-fastlane-darwin": "1.9.4"
+    "@expo/traveling-fastlane-darwin": "1.9.9"
   }
 }

scripts/android/configureAndroidSdk.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+SDKMANAGER=$ANDROID_HOME/tools/bin/sdkmanager
+
+yes | $SDKMANAGER --licenses > /dev/null
+if [[ "$1" = "--install-all-platforms" ]]; then
+  yes | $SDKMANAGER \
+    "platforms;android-23" \
+    "platforms;android-24" \
+    "platforms;android-25" \
+    "platforms;android-26" \
+    "platforms;android-27" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]
+fi
+yes | $SDKMANAGER "platforms;android-28" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]
+yes | $SDKMANAGER "platform-tools" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]
+yes | $SDKMANAGER "build-tools;28.0.3" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]
+yes | $SDKMANAGER \
+  "extras;android;m2repository" \
+  "extras;google;m2repository" \
+  "extras;google;google_play_services" | grep -v '='; [[ ${PIPESTATUS[1]} = 0 ]]

scripts/android/supervisordEventListener.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+printf "READY\n"; # signal to superivsord that listener is ready
+
+while read line; do
+  echo "[supervisord:eventlistener] Processing Event: $line" >&2;
+  kill -SIGQUIT $PPID
+done < /dev/stdin

scripts/cli/configureAndroidSdk.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-SDKMANAGER=$ANDROID_HOME/tools/bin/sdkmanager
-
-yes | $SDKMANAGER --licenses > /dev/null
-yes | $SDKMANAGER "platforms;android-28"
-yes | $SDKMANAGER "platform-tools"
-yes | $SDKMANAGER "build-tools;28.0.3"
-yes | $SDKMANAGER \
-  "extras;android;m2repository" \
-  "extras;google;m2repository" \
-  "extras;google;google_play_services"

scripts/deploy.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+SCRIPT_NAME=`basename "$0"`
+DEPLOY_ENDPOINT_URL="https://circleci.com/api/v1.1/project/github/expo/turtle-deploy/tree/master"
+
+if [[ -z "$1" || -z "$2" ]]; then
+  echo "usage: ${bold}$SCRIPT_NAME PLATFORM ENVIRONMENT [FORCE]${normal}"
+  exit 1
+fi
+
+PLATFORM="$1"
+ENVIRONMENT="$2"
+
+if [[ "$3" = "--force" ]]; then
+  FORCE_DEPLOY="true"
+fi
+
+if [[ "$PLATFORM" != "android" && "$PLATFORM" != "ios" ]]; then
+  echo "unsupported PLATFORM passed: ${bold}$PLATFORM${normal}"
+  echo "valid values are: ${bold}android${normal}, ${bold}ios${normal}"
+  exit 1
+fi
+
+if [[ "$ENVIRONMENT" != "staging" && "$ENVIRONMENT" != "production" ]]; then
+  echo "unsupported ENVIRONMENT passed: ${bold}$ENVIRONMENT${normal}"
+  echo "valid values are: ${bold}staging${normal}, ${bold}production${normal}"
+  exit 1
+fi
+
+if [[ ! -z "$CIRCLE_SHA1" ]]; then
+  TURTLE_COMMIT="$CIRCLE_SHA1"
+elif [[ -z "$FORCE_DEPLOY" ]]; then
+  read -p "You're about to schedule a deploy, are you sure you want to deploy the current commit? (y/n) " -n 1 -r
+  echo    # (optional) move to a new line
+  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "exiting..."
+    exit 0
+  else
+    TURTLE_COMMIT=`git log --pretty=format:%H -1`
+  fi
+fi
+
+if [[ -z "$TURTLE_BOT_CIRCLE_API_USER_TOKEN" ]]; then
+  echo "Please set the TURTLE_BOT_CIRCLE_API_USER_TOKEN env variable."
+  exit 1
+fi
+
+if [[ $FORCE_DEPLOY = "true" ]]; then
+  curl -X POST \
+    --header "Content-Type: application/json" \
+    -u $TURTLE_BOT_CIRCLE_API_USER_TOKEN: \
+    -d '
+      {
+        "build_parameters": {
+          "CIRCLE_JOB": "deploy_'$PLATFORM'",
+          "TURTLE_ENV": "'$ENVIRONMENT'",
+          "TURTLE_FORCE_DEPLOY": "true"
+        }
+      }' \
+    $DEPLOY_ENDPOINT_URL | jq -r .build_url
+else
+  curl -X POST \
+    --header "Content-Type: application/json" \
+    -u $TURTLE_BOT_CIRCLE_API_USER_TOKEN: \
+    -d '
+      {
+        "build_parameters": {
+          "CIRCLE_JOB": "deploy_'$PLATFORM'",
+          "TURTLE_ENV": "'$ENVIRONMENT'",
+          "TURTLE_COMMIT": "'$TURTLE_COMMIT'"
+        }
+      }' \
+    $DEPLOY_ENDPOINT_URL | jq -r .build_url
+fi

scripts/initPrivateSecrets.sh

@@ -13,13 +13,17 @@ fi
 TURTLE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
 TERRAFORM_DIR="${EXPO_UNIVERSE_DIR}/terraform"
 
-ANDROID=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_android_in | head -1 | cut -d"=" -f2 | xargs`
-IOS=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_ios_in | head -1 | cut -d"=" -f2 | xargs`
+ANDROID_NORMAL=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_android_in | head -1 | cut -d"=" -f2 | xargs`
+ANDROID_PRIORITY=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_android_in_priority | head -1 | cut -d"=" -f2 | xargs`
+IOS_NORMAL=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_ios_in | head -1 | cut -d"=" -f2 | xargs`
+IOS_PRIORITY=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_ios_in_priority | head -1 | cut -d"=" -f2 | xargs`
 OUT=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_out | head -1 | cut -d"=" -f2 | xargs`
 
 cat << EOF > $TURTLE_DIR/.secrets.local-queue
-AWS_SQS_ANDROID_QUEUE_URL="$ANDROID"
-AWS_SQS_IOS_QUEUE_URL="$IOS"
+AWS_SQS_ANDROID_QUEUE_URL="$ANDROID_NORMAL"
+AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="$ANDROID_PRIORITY"
+AWS_SQS_IOS_QUEUE_URL="$IOS_NORMAL"
+AWS_SQS_IOS_PRIORITY_QUEUE_URL="$IOS_PRIORITY"
 AWS_SQS_OUT_QUEUE_URL="$OUT"
 EOF
 

scripts/setWorkingdirMode.sh

@@ -6,42 +6,44 @@ DOWNLOAD_DIR="${EXPO_DOWNLOAD_DIR:-$HOME/Downloads}"
 
 mkdir -p $DOWNLOAD_DIR
 
-if [ ! -f "$ROOT_DIR/.secrets.local-queue" ]; then 
-  echo "You need to run \"yarn secrets:init-private [SQS_NAME]\" or create .secrets.local-queue file yourself with AWS_SQS_ANDROID_QUEUE_URL, AWS_SQS_IOS_QUEUE_URL and AWS_SQS_OUT_QUEUE_URL variables."
+if [ ! -f "$ROOT_DIR/.secrets.local-queue" ]; then
+  echo "You need to run \"yarn secrets:init-private [SQS_NAME]\" or create the .secrets.local-queue file yourself with AWS_SQS_ANDROID_QUEUE_URL, AWS_SQS_ANDROID_PRIORITY_QUEUE_URL, AWS_SQS_IOS_QUEUE_URL, AWS_SQS_IOS_PRIORITY_QUEUE_URL and AWS_SQS_OUT_QUEUE_URL variables."
   exit 1
 fi
 
-setup_local() {
-  ensure_link_exists() {
-    if [ ! -L "$2" ]; then
-      ln -s $1 $2
-      echo "$2 link created"
-    fi
-  }
-  WORKINGDIR="$ROOT_DIR/workingdir/local"
-  
-  if [[ -d $WORKINGDIR ]]; then
-    echo "Local workingdir already prepared"
-    return
-  fi
-
-  if [[ -z "$EXPO_UNIVERSE_DIR" ]]; then
-    echo "EXPO_UNIVERSE_DIR needs to be set"
-    exit -1
-  fi
-
-  cd $EXPO_UNIVERSE_DIR/exponent && yarn
-  cd $EXPO_UNIVERSE_DIR/exponent/tools-public && yarn && ./generate-dynamic-macros-android.sh
-
-  mkdir -p $WORKINGDIR
-  echo "$WORKINGDIR directory created"
-
-  LINKS=("template-files" "package.json" "android" "expokit-npm-package" "packages" "client-builds" "cpp" "exponent-view-template" "ios" "shellAppBase-builds" "shellAppWorkspaces" "node_modules")
-
-  for LINKNAME in ${LINKS[@]}; do
-    ensure_link_exists "$EXPO_UNIVERSE_DIR/exponent/$LINKNAME" "$WORKINGDIR/$LINKNAME"
-  done
-}
+# @dsokal: this is currently broken
+#
+# setup_local() {
+#   ensure_link_exists() {
+#     if [ ! -L "$2" ]; then
+#       ln -s $1 $2
+#       echo "$2 link created"
+#     fi
+#   }
+#   WORKINGDIR="$ROOT_DIR/workingdir/local"
+#
+#   if [[ -d $WORKINGDIR ]]; then
+#     echo "Local workingdir already prepared"
+#     return
+#   fi
+#
+#   if [[ -z "$EXPO_UNIVERSE_DIR" ]]; then
+#     echo "EXPO_UNIVERSE_DIR needs to be set"
+#     exit -1
+#   fi
+#
+#   cd $EXPO_UNIVERSE_DIR/exponent && yarn
+#   cd $EXPO_UNIVERSE_DIR/exponent/tools-public && yarn && ./generate-dynamic-macros-android.sh
+#
+#   mkdir -p $WORKINGDIR
+#   echo "$WORKINGDIR directory created"
+#
+#   LINKS=("template-files" "package.json" "android" "expokit-npm-package" "packages" "client-builds" "cpp" "exponent-view-template" "ios" "shellAppBase-builds" "shellAppWorkspaces" "node_modules")
+#
+#   for LINKNAME in ${LINKS[@]}; do
+#     ensure_link_exists "$EXPO_UNIVERSE_DIR/exponent/$LINKNAME" "$WORKINGDIR/$LINKNAME"
+#   done
+# }
 
 setup_remote() {
   if [[ -d $ROOT_DIR/workingdir/android && -d $ROOT_DIR/workingdir/ios ]]; then
@@ -55,7 +57,7 @@ setup_remote() {
       $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
     fi
   fi
-  
+
   for sdk in $ROOT_DIR/workingdir/android/* ; do
     if [[ -f "$sdk/universe-package.json" ]]; then
       mv $sdk/package.json $sdk/exponent-package.json
@@ -87,8 +89,10 @@ EOF
 }
 
 if [ "$1" == "local" ]; then
-  setup_local
-  switch_local
+  # setup_local
+  # switch_local
+  echo "Setting local workingdir is currently broken, sorry :("
+  exit -1
 elif [ "$1" == "remote" ]; then
   setup_remote
   switch_remote

scripts/ssh/connectToAndroidPod.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-if [ -z "$1" ]; then
-  echo "Please provide an environment name as a first argument"
-  exit 1
-fi
-
-if [ -z "$2" ]; then
-  echo "Please provide a pod name as a second argument"
-  exit 1
-fi
-
-kubectl --namespace $1 exec -it $2 -- /bin/bash

scripts/ssh/findAndroidPods.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-ENVIRONMENT=${1:-production}
-
-kubectl get pods --namespace $ENVIRONMENT | grep "turtle-android-" | sed -e "s/  */ /g" | cut -d" " -f1

scripts/ssh/findIosIps.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-ENVIRONMENT=${1:-production}
-
-govc vm.ip -wait 5s turtle-js-"${ENVIRONMENT}"-*

shellTarballs/android/sdk33

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-86974c7df1fc9db46f02b59f4a663b45e2f7a2af.tar.gz
+s3://exp-artifacts/android-shell-builder-aebd669d6573b0b3c802871d94cdb8b4116678d2.tar.gz

src/bin/commands/build/android.ts

@@ -1,6 +1,5 @@
 import fs from 'fs-extra';
 import _ from 'lodash';
-import uuid from 'uuid';
 
 import { ErrorWithCommandHelp } from 'turtle/bin/commands/ErrorWithCommandHelp';
 import { createBuilderAction } from 'turtle/bin/utils/builder';
@@ -24,16 +23,21 @@ export default (program: any, setCommonCommandOptions: any) => {
       /^(app-bundle|apk)$/i,
       'app-bundle',
     )
+    .option(
+      '-m --mode <build>',
+      'type of build: debug|release',
+      /^(debug|release)$/i,
+      'release',
+    )
     .description(
-      'Build a standalone APK or App Bundle for your project, signed and ready for submission to the Google Play'
-      + ' Store.',
+      'Build a standalone APK or App Bundle for your project, either signed and ready for submission to'
+      + ' the Google Play Store or in debug mode.',
     )
     .asyncAction(
       createBuilderAction({
         program,
         command,
         prepareCredentials,
-        buildJobObject,
         builder,
         platform: PLATFORMS.ANDROID,
         os: ['darwin', 'linux'],
@@ -41,26 +45,6 @@ export default (program: any, setCommonCommandOptions: any) => {
     );
 };
 
-const buildJobObject = (
-  appJSON: any,
-  { releaseChannel, username, projectDir, publicUrl, buildType }: any,
-  credentials: any,
-) => ({
-  config: {
-    ..._.get(appJSON, 'expo.android.config', {}),
-    releaseChannel,
-    androidPackage: _.get(appJSON, 'expo.android.package'),
-    publicUrl,
-    buildType,
-  },
-  id: uuid.v4(),
-  platform: PLATFORMS.ANDROID,
-  sdkVersion: _.get(appJSON, 'expo.sdkVersion'),
-  projectDir,
-  experienceName: `@${username}/${_.get(appJSON, 'expo.slug')}`,
-  ...(credentials && { credentials }),
-});
-
 const prepareCredentials = async (cmd: any) => {
   const { keystorePath, keystoreAlias } = cmd;
   const keystorePassword = process.env.EXPO_ANDROID_KEYSTORE_PASSWORD;

src/bin/commands/build/ios.ts

@@ -1,6 +1,5 @@
 import fs from 'fs-extra';
 import _ from 'lodash';
-import uuid from 'uuid';
 
 import { ErrorWithCommandHelp } from 'turtle/bin/commands/ErrorWithCommandHelp';
 import { createBuilderAction } from 'turtle/bin/utils/builder';
@@ -32,7 +31,6 @@ export default (program: any, setCommonCommandOptions: any) => {
         program,
         command,
         prepareCredentials,
-        buildJobObject,
         builder,
         platform: PLATFORMS.IOS,
         os: 'darwin',
@@ -40,21 +38,6 @@ export default (program: any, setCommonCommandOptions: any) => {
     );
 };
 
-const buildJobObject = (appJSON: any, { releaseChannel, buildType, username, publicUrl }: any, credentials: any) => ({
-  config: {
-    ..._.get(appJSON, 'expo.ios.config', {}),
-    buildType,
-    releaseChannel,
-    bundleIdentifier: _.get(appJSON, 'expo.ios.bundleIdentifier'),
-    publicUrl,
-  },
-  id: uuid.v4(),
-  platform: PLATFORMS.IOS,
-  sdkVersion: _.get(appJSON, 'expo.sdkVersion'),
-  experienceName: `@${username}/${_.get(appJSON, 'expo.slug')}`,
-  ...(credentials && { credentials }),
-});
-
 const prepareCredentials = async (cmd: any) => {
   if (cmd.type !== IOS_BUILD_TYPES.ARCHIVE) {
     return null;

src/bin/setup/android/index.ts

@@ -20,10 +20,6 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
     command: 'bash',
     missingDescription: 'Please install bash',
   },
-  {
-    command: 'gulp',
-    missingDescription: 'Run `npm install -g gulp-cli` (or `yarn global add gulp-cli`) to install gulp',
-  },
   {
     command: 'javac',
     missingDescription: 'Please install JDK (version 8 or newer) - check https://jdk.java.net/',
@@ -67,7 +63,7 @@ function formatShellAppTarballUriPath(sdkMajorVersion: string) {
   return path.join(config.directories.shellTarballsDir, PLATFORMS.ANDROID, `sdk${sdkMajorVersion}`);
 }
 
-async function _shellAppPostDownloadAction(workingdir: string) {
+async function _shellAppPostDownloadAction(sdkVersion: string, workingdir: string) {
   const inWorkingdir = (filename: string) => path.join(workingdir, filename);
 
   if (await fs.pathExists(inWorkingdir('universe-package.json'))) {
@@ -82,9 +78,11 @@ async function _shellAppPostDownloadAction(workingdir: string) {
     await _installNodeModules(workingdir);
   }
 
-  // TODO: remove following lines after upgrading android shell app
-  const toolsPublicDir = path.join(workingdir, 'tools-public');
-  await _installNodeModules(toolsPublicDir);
+  // TODO: remove this after making sure that we don't need node_modules in tools-public for older sdks
+  if (ExponentTools.parseSdkMajorVersion(sdkVersion) < 33) {
+    const toolsPublicDir = path.join(workingdir, 'tools-public');
+    await _installNodeModules(toolsPublicDir);
+  }
 }
 
 async function _installNodeModules(cwd: string) {

src/bin/setup/android/ndk.ts

@@ -4,7 +4,7 @@ import path from 'path';
 import decompress from 'decompress';
 import fs from 'fs-extra';
 
-import { formatArtifactDownloadPath } from 'turtle/bin/setup/utils/common';
+import * as utils from 'turtle/bin/setup/utils/common';
 import download from 'turtle/bin/setup/utils/downloader';
 import config from 'turtle/config';
 import logger from 'turtle/logger';
@@ -16,9 +16,10 @@ const l = logger.child(LOGGER_FIELDS);
 
 export default async function ensureAndroidNDKIsPresent() {
   const androidNdkDir = path.join(config.directories.androidDependenciesDir, 'ndk');
+  await utils.removeDirectoryUnlessReady(androidNdkDir);
   if (!(await fs.pathExists(androidNdkDir))) {
     await fs.ensureDir(androidNdkDir);
-    const androidNdkDownloadPath = formatArtifactDownloadPath(ANDROID_NDK_URL);
+    const androidNdkDownloadPath = utils.formatArtifactDownloadPath(ANDROID_NDK_URL);
 
     try {
       l.info('Downloading Android NDK');
@@ -26,6 +27,7 @@ export default async function ensureAndroidNDKIsPresent() {
       await download(ANDROID_NDK_URL, androidNdkDownloadPath);
       l.info('Decompressing Android NDK');
       await decompress(androidNdkDownloadPath, androidNdkDir, { strip: 1 });
+      await utils.markDirectoryAsReady(androidNdkDir);
       l.info('Android NDK installed successfully');
     } catch (err) {
       await fs.remove(androidNdkDir);

src/bin/setup/android/sdk.ts

@@ -5,7 +5,7 @@ import { ExponentTools } from '@expo/xdl';
 import decompress from 'decompress';
 import fs from 'fs-extra';
 
-import { formatArtifactDownloadPath } from 'turtle/bin/setup/utils/common';
+import * as utils from 'turtle/bin/setup/utils/common';
 import download from 'turtle/bin/setup/utils/downloader';
 import config from 'turtle/config';
 import logger from 'turtle/logger';
@@ -17,9 +17,10 @@ const l = logger.child(LOGGER_FIELDS);
 
 export default async function ensureAndroidSDKIsPresent() {
   const androidSdkDir = path.join(config.directories.androidDependenciesDir, 'sdk');
+  await utils.removeDirectoryUnlessReady(androidSdkDir);
   if (!(await fs.pathExists(androidSdkDir))) {
     await fs.ensureDir(androidSdkDir);
-    const androidSdkDownloadPath = formatArtifactDownloadPath(ANDROID_SDK_URL);
+    const androidSdkDownloadPath = utils.formatArtifactDownloadPath(ANDROID_SDK_URL);
 
     try {
       l.info('Downloading Android SDK');
@@ -31,6 +32,7 @@ export default async function ensureAndroidSDKIsPresent() {
       await fs.remove(androidSdkDownloadPath);
       l.info('Configuring Android SDK, this may take a while');
       await _configureSdk(androidSdkDir);
+      await utils.markDirectoryAsReady(androidSdkDir);
       l.info('Android SDK installed and configured successfully');
     } catch (err) {
       await fs.remove(androidSdkDir);
@@ -52,7 +54,7 @@ async function _configureSdk(androidSdkDir: string) {
   await ExponentTools.spawnAsyncThrowError('./configureAndroidSdk.sh', [], {
     pipeToLogger: true,
     loggerFields: LOGGER_FIELDS,
-    cwd: path.join(config.directories.rootDir, 'scripts/cli'),
+    cwd: path.join(config.directories.rootDir, 'scripts/android'),
     env: {
       ...process.env,
       ANDROID_HOME: androidSdkDir,

src/bin/setup/utils/common.ts

@@ -17,7 +17,7 @@ interface IShellAppFormaters {
   formatShellAppTarballUriPath: (sdkMajorVersion: string) => string;
 }
 
-type PostDownloadAction = (workingdir: string) => void;
+type PostDownloadAction = (sdkVersion: string, workingdir: string) => void;
 
 export async function checkSystem(requiredTools: IToolDefinition[]) {
   await ensureToolsAreInstalled(requiredTools);
@@ -29,14 +29,17 @@ export async function ensureShellAppIsPresent(
   postDownloadAction?: PostDownloadAction,
 ) {
   const workingdir = formatters.formatShellAppDirectory({ sdkVersion });
+  const shellAppMetadata = await _readShellAppTarballS3Uri(sdkVersion, formatters);
+  await removeDirectoryUnlessReady(workingdir, shellAppMetadata);
   if (await fs.pathExists(workingdir)) {
     return;
   }
   l.info(`shell app for SDK ${sdkVersion} doesn't exist, downloading...`);
   await _downloadShellApp(sdkVersion, workingdir, formatters);
   if (postDownloadAction) {
-    await postDownloadAction(workingdir);
+    await postDownloadAction(sdkVersion, workingdir);
   }
+  await markDirectoryAsReady(workingdir, shellAppMetadata);
 }
 
 async function _downloadShellApp(sdkVersion: string, targetDirectory: string, formatters: IShellAppFormaters) {
@@ -65,3 +68,31 @@ async function _readShellAppTarballS3Uri(sdkVersion: string, formatters: IShellA
   const data = await fs.readFile(filePath, 'utf8');
   return data.trim();
 }
+
+export async function removeDirectoryUnlessReady(dir: string, metadata?: string) {
+  const readyFilePath = path.join(dir, '.ready');
+  const readyFileExists = await fs.pathExists(readyFilePath);
+  let shouldRemove = false;
+  if (readyFileExists) {
+    if (metadata !== undefined) {
+      const readyFileContents = (await fs.readFile(readyFilePath, 'utf-8')).trim();
+      if (readyFileContents !== metadata) {
+        shouldRemove = true;
+       }
+    }
+  } else {
+    shouldRemove = true;
+  }
+  if (shouldRemove) {
+    await fs.remove(dir);
+  }
+}
+
+export async function markDirectoryAsReady(dir: string, metadata?: string) {
+  const readyFilePath = path.join(dir, '.ready');
+  if (metadata === undefined) {
+    await fs.open(readyFilePath, 'w');
+  } else {
+    await fs.writeFile(readyFilePath, metadata);
+  }
+}

src/bin/utils/builder.ts

@@ -1,10 +1,13 @@
+import { ExponentTools } from '@expo/xdl';
 import _ from 'lodash';
+import uuid from 'uuid';
 
 import { ErrorWithCommandHelp } from 'turtle/bin/commands/ErrorWithCommandHelp';
 import { ErrorWithProgramHelp } from 'turtle/bin/commands/ErrorWithProgramHelp';
 import setup from 'turtle/bin/setup/setup';
 import * as ProjectUtils from 'turtle/bin/utils/project';
 import * as UserUtils from 'turtle/bin/utils/user';
+import { getExperienceUrl } from 'turtle/builders/utils/common';
 import logger from 'turtle/logger';
 import { sanitizeJob } from 'turtle/validator';
 
@@ -12,7 +15,6 @@ export function createBuilderAction({
   program,
   command,
   prepareCredentials,
-  buildJobObject,
   builder,
   platform,
   os,
@@ -55,6 +57,7 @@ export function createBuilderAction({
       const args = {
         releaseChannel: cmd.releaseChannel || 'default',
         buildType: cmd.type,
+        buildMode: cmd.mode,
         username: userData.username || 'anonymous',
         projectDir: ProjectUtils.resolveAbsoluteDir(projectDirArg),
         publicUrl: cmd.publicUrl,
@@ -65,7 +68,7 @@ export function createBuilderAction({
       await setup(platform, sdkVersion);
       const credentials = await prepareCredentials(cmd);
       const rawJob = {
-        ...buildJobObject(appJSON, args, credentials),
+        ...await buildJobObject(platform, appJSON, args, credentials),
         ...cmd.buildDir && { fakeUploadDir: ProjectUtils.resolveAbsoluteDir(cmd.buildDir) },
         ...cmd.output && { fakeUploadBuildPath: ProjectUtils.resolveAbsoluteDir(cmd.output) },
       };
@@ -82,3 +85,38 @@ export function createBuilderAction({
     }
   };
 }
+
+const buildJobObject = async (
+  platform: 'android' | 'ios',
+  appJSON: any,
+  { releaseChannel, buildType, buildMode, username, publicUrl, projectDir }: any,
+  credentials: any,
+) => {
+  const job = {
+    config: {
+      ..._.get(appJSON, 'expo.ios.config', {}),
+      buildType,
+      ...(platform === 'android' ? { buildMode } : {}),
+      releaseChannel,
+      ...(platform === 'ios' ? { bundleIdentifier: _.get(appJSON, 'expo.ios.bundleIdentifier') } : {}),
+      ...(platform === 'android' ? { androidPackage: _.get(appJSON, 'expo.android.package') } : {}),
+      publicUrl,
+    },
+    id: uuid.v4(),
+    platform,
+    projectDir,
+    sdkVersion: _.get(appJSON, 'expo.sdkVersion'),
+    experienceName: `@${username}/${_.get(appJSON, 'expo.slug')}`,
+    ...(credentials && { credentials }),
+  };
+  const url = getExperienceUrl(job.experienceName, job.config.publicUrl);
+
+  const manifest = await ExponentTools.getManifestAsync(url, {
+    'Exponent-SDK-Version': _.get(appJSON, 'expo.sdkVersion'),
+    'Exponent-Platform': platform,
+    'Expo-Release-Channel': releaseChannel,
+    'Accept': 'application/expo+json,application/json',
+  });
+
+  return { ...job, manifest };
+};

src/builders/android.ts

@@ -75,12 +75,12 @@ async function runShellAppBuilder(
 
   logger.info({ buildPhase: 'resolve native modules' }, 'Resolving universal modules dependencies');
   const enabledModules = semver.satisfies(sdkVersion, '>= 33.0.0')
-    ? await resolveNativeModules(workingDir, manifest.dependencies)
+    ? await resolveNativeModules(workingDir, _.get(manifest, 'dependencies'))
     : null;
 
   try {
     await AndroidShellApp.createAndroidShellAppAsync({
-      url: commonUtils.getExperienceUrl(jobData),
+      url: commonUtils.getExperienceUrl(jobData.experienceName, jobData.config.publicUrl),
       sdkVersion,
       keystore: tempKeystorePath,
       manifest,
@@ -93,6 +93,7 @@ async function runShellAppBuilder(
       outputFile: outputFilePath,
       modules: enabledModules,
       buildType: jobData.config.buildType,
+      buildMode: jobData.config.buildMode,
     });
   } catch (err) {
     commonUtils.logErrorOnce(err);

src/builders/utils/common.ts

@@ -1,13 +1,9 @@
 import * as url from 'url';
 
 import config from 'turtle/config';
-import { IJob } from 'turtle/job';
 import logger from 'turtle/logger';
 
-export function getExperienceUrl(job: IJob) {
-  const { experienceName, config: jobConfig } = job;
-  const { publicUrl } = jobConfig;
-
+export function getExperienceUrl(experienceName: string, publicUrl?: string) {
   // publicUrl is passed in if user wants to build an externally hosted app
   if (publicUrl) {
     return publicUrl;

src/builders/utils/ios/adhocBuild.ts

@@ -3,7 +3,6 @@ import path from 'path';
 import spawnAsync from '@expo/spawn-async';
 import { IosCodeSigning } from '@expo/xdl';
 import fs from 'fs-extra';
-import isEmpty from 'lodash/isEmpty';
 
 import * as sqs from 'turtle/aws/sqs';
 import BuildError, { BuildErrorReason } from 'turtle/builders/BuildError';
@@ -21,10 +20,18 @@ async function prepareAdHocBuildCredentials(job: IJob) {
   }
 
   const { bundleIdentifier } = job.config;
-  const { certP12, certPassword, teamId, appleSession, udids } = job.credentials;
+  const {
+    certP12,
+    certPassword,
+    teamId,
+    appleSession,
+    udids,
+    provisioningProfileId,
+  } = job.credentials;
 
   const certSerialNumber = IosCodeSigning.findP12CertSerialNumber(certP12, certPassword);
   const args = [
+    ...(provisioningProfileId ? ['--profile-id', provisioningProfileId] : []),
     teamId,
     udids!.join(','),
     bundleIdentifier,
@@ -32,25 +39,26 @@ async function prepareAdHocBuildCredentials(job: IJob) {
   ];
 
   try {
+    if (provisioningProfileId) {
+      logger.info(
+        `Using ad hoc provisioning profile id: ${provisioningProfileId}`,
+      );
+    }
+
     const credentials = await runFastlaneAction(
       travelingFastlane.manageAdHocProvisioningProfile,
       args,
       { env: { FASTLANE_SESSION: appleSession } },
     );
-    if (isEmpty(credentials)) {
-      if (!job.credentials.provisioningProfile) {
-        // a user has generated the provisioning profile for his app, but he probably deleted it from Expo servers, ...
-        throw new BuildError(
-          BuildErrorReason.PROVISIONING_PROFILE_MISSING,
-          `Provisioning profile is missing on Expo servers.`,
-        );
-      }
-      return;
-    }
 
     logger.info('New ad hoc provisioning profile successfully created');
     job.credentials.provisioningProfile = credentials.provisioningProfile;
 
+    if (!credentials.provisioningProfileUpdateTimestamp) {
+      //  dont need to persist profile because nothing changed
+      return;
+    }
+
     if (isOffline()) {
       const provisioningProfilePath = path.join(job.projectDir, `adhoc.mobileprovision`);
       await fs.writeFile(provisioningProfilePath, Buffer.from(credentials.provisioningProfile, 'base64'));

src/builders/utils/ios/shellAppBuilder.ts

@@ -43,7 +43,7 @@ export default async function runShellAppBuilder(ctx: IContext, job: IJob): Prom
     });
   } else {
     Object.assign(shellAppParams, {
-      url: commonUtils.getExperienceUrl(job),
+      url: commonUtils.getExperienceUrl(job.experienceName, job.config.publicUrl),
       releaseChannel,
       sdkVersion,
     });

src/builders/utils/unimodules.ts

@@ -44,9 +44,11 @@ export async function resolveNativeModules(workingdir: string, packages?: string
 
 async function readCorePackages(workingdir: string) {
   const pkgJsonPath = path.join(workingdir, 'node_modules', 'react-native-unimodules', 'package.json');
-  if (await fs.pathExists(pkgJsonPath)) {
+  const expoPkgJsonPath = path.join(workingdir, 'packages', 'expo', 'package.json');
+  if (await fs.pathExists(pkgJsonPath) && await fs.pathExists(expoPkgJsonPath)) {
     const packageJson = JSON.parse(await fs.readFile(pkgJsonPath, 'utf-8'));
-    return keys(packageJson.dependencies);
+    const expoPackageJson = JSON.parse(await fs.readFile(expoPkgJsonPath, 'utf-8'));
+    return keys(packageJson.dependencies).concat(keys(expoPackageJson.dependencies));
   } else {
     logger.warn('No core packages detected');
     return [];

src/builders/utils/version.ts

@@ -5,26 +5,32 @@ import path from 'path';
 import config from 'turtle/config';
 import { IJob } from 'turtle/job';
 
-let SUPPORTED_SDK_VERSIONS: number[];
+interface IVersionsMap {
+  [platform: string]: number[];
+}
+
+const versions: IVersionsMap = {};
 
 export async function ensureCanBuildSdkVersion(job: IJob) {
   if (config.builder.useLocalWorkingDir) {
     return;
   }
 
-  if (!SUPPORTED_SDK_VERSIONS) {
-    SUPPORTED_SDK_VERSIONS = await findSupportedSdkVersions(job.platform);
-  }
-
+  const platformVersions = await findSupportedSdkVersions(job.platform);
   const targetMajorSdkVersion = ExponentTools.parseSdkMajorVersion(job.sdkVersion || job.manifest.sdkVersion);
-  if (!SUPPORTED_SDK_VERSIONS.includes(targetMajorSdkVersion)) {
+  if (!platformVersions.includes(targetMajorSdkVersion)) {
     throw new Error(`Unsupported SDK Version!`);
   }
 }
 
-async function findSupportedSdkVersions(platform: 'android' | 'ios'): Promise<number[]> {
-  const SDK_DIR_PREFIX = 'sdk';
-  const files = await fs.readdir(path.join(config.directories.workingDir, platform));
-  const sdks = files.filter((file) => file.startsWith(SDK_DIR_PREFIX));
-  return sdks.map((sdk) => parseInt(sdk.substr(SDK_DIR_PREFIX.length), 10));
+export async function findSupportedSdkVersions(platform: 'android' | 'ios'): Promise<number[]> {
+  if (versions[platform]) {
+    return versions[platform];
+  } else {
+    const SDK_DIR_PREFIX = 'sdk';
+    const files = await fs.readdir(path.join(config.directories.workingDir, platform));
+    const sdks = files.filter((file) => file.startsWith(SDK_DIR_PREFIX));
+    versions[platform] = sdks.map((sdk) => parseInt(sdk.substr(SDK_DIR_PREFIX.length), 10));
+    return versions[platform];
+  }
 }

src/constants/index.ts

@@ -16,4 +16,9 @@ export enum ANDROID_BUILD_TYPES {
   APP_BUNDLE = 'app-bundle',
 }
 
+export enum ANDROID_BUILD_MODES {
+  DEBUG = 'debug',
+  RELEASE = 'release',
+}
+
 export { BUILD };

src/job.ts

@@ -1,4 +1,4 @@
-import { ANDROID_BUILD_TYPES, IOS_BUILD_TYPES } from 'turtle/constants';
+import { ANDROID_BUILD_MODES, ANDROID_BUILD_TYPES, IOS_BUILD_TYPES } from 'turtle/constants';
 
 export interface IJob {
   platform: 'android' | 'ios';
@@ -14,6 +14,7 @@ export interface IJob {
     bundleIdentifier?: string;
     // android
     androidPackage?: string;
+    buildMode?: ANDROID_BUILD_MODES;
   };
   credentials: {
     // android
@@ -25,6 +26,7 @@ export interface IJob {
     certP12?: string;
     certPassword?: string;
     provisioningProfile?: string;
+    provisioningProfileId?: string;
     teamId?: string;
     appleSession?: string;
     udids?: string[];

src/jobManager.ts

@@ -64,7 +64,10 @@ async function processJob(jobData: any) {
     setCurrentJobId(job.id);
     const pingerHandle = sqs.changeMessageVisibilityRecurring(priority, jobData.ReceiptHandle, job.id);
 
-    logger.info(`Doing job MessageId=${jobData.MessageId} BuildId=${job.id} ${Date.now()}`);
+    logger.info(
+      `Doing job MessageId=${jobData.MessageId} BuildId=${job.id} `
+      + `Priority=${priority.slice(0, priority.length - 8)} ${Date.now()}`,
+    );
 
     const cancelled = await redis.checkIfCancelled(job.id);
     if (cancelled) {

src/jobsSchemas/android.ts

@@ -1,6 +1,6 @@
 import Joi from 'joi';
 
-import { PLATFORMS } from 'turtle/constants';
+import { ANDROID_BUILD_MODES, ANDROID_BUILD_TYPES, PLATFORMS } from 'turtle/constants';
 import baseJobSchema from 'turtle/jobsSchemas/base';
 
 export default baseJobSchema.concat(
@@ -12,5 +12,9 @@ export default baseJobSchema.concat(
       keystoreAlias: Joi.string(),
       keystore: Joi.string(),
     }),
+    config: Joi.object().keys({
+      buildMode: Joi.string().valid(Object.values(ANDROID_BUILD_MODES)).default(ANDROID_BUILD_MODES.RELEASE),
+      buildType: Joi.string().valid(Object.values(ANDROID_BUILD_TYPES)).default(ANDROID_BUILD_TYPES.APK),
+    }),
   }),
 );

src/jobsSchemas/ios.ts

@@ -17,6 +17,7 @@ export default baseJobSchema.concat(
         teamId: Joi.string().required(),
         // it's not required for adhoc Expo Client builds
         provisioningProfile: Joi.string(),
+        provisioningProfileId: Joi.string(),
         appleSession: Joi.string(),
         udids: Joi.array().items(Joi.string()),
       })

src/logger/index.ts

@@ -17,6 +17,7 @@ class Logger {
       name: 'turtle',
       level: config.logger.level,
       platform: config.platform,
+      serializers: bunyan.stdSerializers,
       ...config.deploymentEnv && { environment: config.deploymentEnv },
       streams: Object.values(streams),
     });

src/metrics/buildersConfig.ts

@@ -12,7 +12,14 @@ const createModeMetricName = (mode: TurtleModeLabels) => `${createMetricName()}.
 const createModeTag = (mode: TurtleModeLabels) => `mode:${mode}`;
 
 export default async function sendConfigToDatadog() {
-  const configuration = await getLabeledConfiguration();
+  let configuration;
+  try {
+    configuration = await getLabeledConfiguration();
+  } catch (err) {
+    logger.warn('Failed to send the builders configuration to Datadog because it couldn\'t be fetched from Redis');
+    return;
+  }
+
   const replicaCount = await getReplicaCount();
   const currentReplicaCount = replicaCount === null ? configuration.length : replicaCount;
 

src/scripts/setConfigurations.ts

@@ -1,80 +0,0 @@
-
-import { PLATFORMS } from 'turtle/constants';
-import logger from 'turtle/logger';
-import {
-  createConfigurationsKey,
-  createDefaultConfigurationKey,
-  HIGH_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-} from 'turtle/utils/priorities';
-import { connect, RedisClient } from 'turtle/utils/redis';
-
-const TIMEOUT = 3000;
-
-// Turtle before doing anything (and after every build) checks for the first config
-// (an element in config table with key=i) that is not taken (config.i is null/outdated)
-// or is taken by this process itself.
-// If Turtle dies unexpectedly the configuration will become free after TTL expires,
-// so the state (number of HP/NP turtles) should come back to normal soon.
-
-// For example, if a configuration table looks like this:
-// [ HIGH_CONFIGURATION, NORMAL_CONFIGURATION, HIGH_CONFIGURATION ]
-// If there is one turtle it runs with High Priority first.
-// If there are two turtles, one runs with High Priority first, the second one with Normal Priority first.
-// If the first dies, and the third is started in his place, it gets 3rd config (again HP-first),
-// but after some time, because of TTL for config reservations,
-// they should come back to using configurations no. 1 and 2.
-// If there are no more available configurations from the table, the default configuration is used.
-
-const AVAILABLE_CONFIGURATIONS_IOS = [
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  // there are 5 iOS builders at the moment
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-];
-
-const AVAILABLE_CONFIGURATIONS_ANDROID = [
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  // there are 15 Android builders at the moment
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-];
-
-const DEFAULT_CONFIGURATION = HIGH_CONFIGURATION;
-
-async function run() {
-  const redis = await connect(TIMEOUT, RedisClient.Configuration);
-
-  await redis.set(createConfigurationsKey(PLATFORMS.IOS), JSON.stringify(AVAILABLE_CONFIGURATIONS_IOS));
-  await redis.set(createConfigurationsKey(PLATFORMS.ANDROID), JSON.stringify(AVAILABLE_CONFIGURATIONS_ANDROID));
-  await redis.set(createDefaultConfigurationKey(PLATFORMS.IOS), JSON.stringify(DEFAULT_CONFIGURATION));
-  await redis.set(createDefaultConfigurationKey(PLATFORMS.ANDROID), JSON.stringify(DEFAULT_CONFIGURATION));
-
-  redis.disconnect();
-}
-
-if (require.main === module) {
-  run()
-    .then(() => logger.info('All done'))
-    .catch((err) => logger.error('Error:', err));
-}

src/scripts/setReplicaCount.ts

@@ -1,38 +0,0 @@
-import isNaN from 'lodash/isNaN';
-
-import { PLATFORMS } from 'turtle/constants';
-import logger from 'turtle/logger';
-import { setReplicaCount } from 'turtle/utils/metadata';
-import { getRedisClient, RedisClient } from 'turtle/utils/redis';
-
-async function run() {
-  const platform = process.env.SCRIPT_PLATFORM;
-  const replicaCountRaw = process.env.SCRIPT_REPLICA_COUNT;
-  const replicaCount = Number(replicaCountRaw);
-
-  if (!platform || !replicaCountRaw) {
-    throw new Error('You must set both SCRIPT_PLATFORM and SCRIPT_REPLICA_COUNT env variables.');
-  }
-
-  if (!Object.values(PLATFORMS).includes(platform)) {
-    throw new Error(`Unsupported platform: ${platform}`);
-  }
-
-  if (isNaN(replicaCount)) {
-    throw new Error(`Wrong value for replica count: ${replicaCountRaw}`);
-  }
-
-  const redis = await getRedisClient(RedisClient.Configuration);
-  logger.info(`[${platform}] Setting replica count in redis to ${replicaCount}`);
-  await setReplicaCount(platform as PLATFORMS, replicaCount);
-  redis.disconnect();
-}
-
-if (require.main === module) {
-  run()
-    .then(() => logger.info('All done'))
-    .catch((err) => {
-      logger.error('Error:', err);
-      process.exit(1);
-    });
-}

src/server.ts

@@ -1,13 +1,15 @@
 import { LoggerDetach } from '@expo/xdl';
 
+import { findSupportedSdkVersions } from 'turtle/builders/utils/version';
 import config from 'turtle/config';
 import { doJob } from 'turtle/jobManager';
 import logger from 'turtle/logger';
 import setup from 'turtle/setup';
 import { checkShouldExit, setShouldExit, turtleVersion } from 'turtle/turtleContext';
-import { getRedisClient, RedisClient } from 'turtle/utils/redis';
+import { getRedisClient } from 'turtle/utils/redis';
 
 const REDIS_TURTLE_VERSION_KEY = 'turtle:version';
+const REDIS_TURTLE_SUPPORTED_SDK_VERSIONS_KEY = `turtle:${config.platform}:sdkVersions`;
 
 process.on('unhandledRejection', (err) => logger.error({ err }, 'Unhandled promise rejection'));
 
@@ -35,11 +37,18 @@ async function main() {
   }
 
   try {
-    const redis = await getRedisClient(RedisClient.Configuration);
-    await redis.set(REDIS_TURTLE_VERSION_KEY, turtleVersion);
-    logger.info(`Register Turtle version ${turtleVersion} in Redis`);
+    const redis = await getRedisClient();
+    if (redis) {
+      await redis.set(REDIS_TURTLE_VERSION_KEY, turtleVersion);
+      logger.info(`Registered Turtle version (${turtleVersion}) in Redis`);
+      const sdkVersions = (await findSupportedSdkVersions(config.platform)).join(',');
+      await redis.set(REDIS_TURTLE_SUPPORTED_SDK_VERSIONS_KEY, sdkVersions);
+      logger.info(`Registered supported SDK versions (${sdkVersions}) in Redis`);
+    } else {
+      logger.warn('Couldn\'t set supported SDK versions in Redis (failed to connect to Redis)');
+    }
   } catch (err) {
-    logger.error({ err }, 'Failed to register Turtle version in Redis');
+    logger.error({ err }, 'Failed to set versions in Redis');
   }
 
   while (true) {

src/utils/metadata.ts

@@ -5,14 +5,6 @@ import { getRedisClient, RedisClient } from 'turtle/utils/redis';
 const createKeyPrefix = (platform: PLATFORMS) => `${config.deploymentEnv}:${platform}`;
 const createReplicaCountKey = (platform: PLATFORMS) => `${createKeyPrefix(platform)}:replicaCount`;
 
-export async function setReplicaCount(platform: PLATFORMS, replicaCount: number) {
-  const redis = await getRedisClient(RedisClient.Configuration);
-  await redis.set(
-    createReplicaCountKey(platform),
-    replicaCount,
-  );
-}
-
 export async function getReplicaCount(maybePlatform?: PLATFORMS) {
   const platform = maybePlatform || config.platform;
   const redis = await getRedisClient(RedisClient.Configuration);

src/utils/priorities.ts

@@ -49,7 +49,7 @@ export function createDefaultConfigurationKey(platform = config.platform) {
 }
 
 export async function getPriorities() {
-  if (config.env === 'test') {
+  if (config.env === 'test' || config.deploymentEnv === 'development') {
     return NORMAL_CONFIGURATION;
   }
 
@@ -93,7 +93,7 @@ export async function getPriorities() {
   }
 }
 
-export async function getConfiguration(currentRedisClient: any) {
+async function getConfiguration(currentRedisClient: any) {
   const redis = currentRedisClient || await getRedisClient(RedisClient.Configuration);
   return JSON.parse(await redis.get(createConfigurationsKey()));
 }

src/utils/redis.ts

@@ -64,7 +64,7 @@ export async function checkIfCancelled(jobId: string) {
     return await redis.get(`jobs:cancelled:${jobId}`);
   } catch (err) {
     if (config.deploymentEnv === 'development') {
-      logger.warn('Did you turn on redis server? Run `yarn start-docker` in server/www');
+      logger.warn('Did you turn on redis server? Run `yarn start-docker` in $EXPO_UNIVERSE_DIR/server/www');
     }
     logger.error({ err });
     return false;

supervisord.conf

@@ -0,0 +1,17 @@
+[supervisord]
+nodaemon=true
+
+[program:turtle]
+directory=/app/turtle
+command=node ./build/server.js
+autorestart=true
+startretries=5
+startsecs=600
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[eventlistener:turtle_failure]
+events=PROCESS_STATE_FATAL
+command=/app/turtle/scripts/android/supervisordEventListener.sh

tsconfig.json

@@ -18,7 +18,8 @@
     "allowJs": true,
     "plugins": [
       { "transform": "ts-transformer-imports" }
-    ]
+    ],
+    "typeRoots": ["node_modules/@expo/babel-preset-cli/ts-declarations", "node_modules/@types"]
   },
   "include": ["src"],
   "exclude": ["node_modules", "build", "workingdir"]