fix android debug builds (#118)

* add circleci jobs for turtle deployment

* add checkout step

* remove redundant npm scripts

* remove docker:* and deploy:android npm scripts

* improve docs and remove duplicated scripts

.circleci/config.yml

@@ -137,6 +137,23 @@ commands:
           paths:
             - "<< parameters.workingdir >>/<< parameters.platform >>/<< parameters.sdk_version >>"
 
+  deploy_turtle:
+    parameters:
+      platform:
+        type: string
+      env:
+        type: string
+    steps:
+      - checkout
+      - run:
+          name: Install curl & jq
+          command: nix-env -iA nixpkgs.curl nixpkgs.jq
+      - run:
+          name: Schedule a Turtle deploy
+          command: |
+            echo "We're going to deploy << parameters.platform >> Turtles to << parameters.env >>"
+            ./scripts/deploy.sh "<< parameters.platform >>" "<< parameters.env >>"
+
 jobs:
   build:
     executor: nix
@@ -203,6 +220,34 @@ jobs:
           sdk_version: sdk32
           platform: android
 
+  android_staging_deploy:
+    executor: nix
+    steps:
+      - deploy_turtle:
+          platform: android
+          env: staging
+
+  android_production_deploy:
+    executor: nix
+    steps:
+      - deploy_turtle:
+          platform: android
+          env: production
+
+  ios_staging_deploy:
+    executor: nix
+    steps:
+      - deploy_turtle:
+          platform: ios
+          env: staging
+
+  ios_production_deploy:
+    executor: nix
+    steps:
+      - deploy_turtle:
+          platform: ios
+          env: production
+
 workflows:
   checks:
     jobs:
@@ -212,3 +257,25 @@ workflows:
       - unit_tests
       - ios_smoke_tests
       - android_smoke_tests
+  deploy:
+    jobs:
+      - android_staging_approve_deploy:
+          type: approval
+      - android_staging_deploy:
+          requires:
+            - android_staging_approve_deploy
+      - android_production_approve_deploy:
+          type: approval
+      - android_production_deploy:
+          requires:
+            - android_production_approve_deploy
+      - ios_staging_approve_deploy:
+          type: approval
+      - ios_staging_deploy:
+          requires:
+            - ios_staging_approve_deploy
+      - ios_production_approve_deploy:
+          type: approval
+      - ios_production_deploy:
+          requires:
+            - ios_production_approve_deploy

.envrc

@@ -8,16 +8,13 @@ source_local() {
   watch_file $file
 }
 
-source_up || true
 source_local
 
 export NODE_ENV=development
 
 export EXPO_TURTLE_TOOLS_DIR="$(pwd)/tools"
 
-if [ -z "$EXPO_UNIVERSE_DIR" ]; then
-  PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
-  if [ "$CI" = "true" ]; then
-    PATH_add $EXPO_TURTLE_TOOLS_DIR/ci/bin
-  fi
+PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
+if [ "$CI" = "true" ]; then
+  PATH_add $EXPO_TURTLE_TOOLS_DIR/ci/bin
 fi

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -0,0 +1,38 @@
+---
+name: "\U0001F41B Bug Report"
+about: You want to report a reproducible bug or regression in turtle-cli.
+title: ''
+labels: 'bug'
+assignees: ''
+
+---
+
+<!-- Thanks for helping expo! Before you submit your issue, please make sure you followed our checklist and check the appropriate boxes by putting an x in the [ ]: [x] -->
+
+### New Issue Checklist
+
+- [ ] Updated turtle-cli to the latest version (`yarn global add turtle-cli` or `npm install --global turtle-cli`).
+- [ ] I read the [Contribution Guidelines](https://github.com/expo/turtle/blob/master/CONTRIBUTING.md).
+- [ ] I read [docs.expo.io](https://docs.expo.io) (Managed Workflow > Distributing Your App).
+- [ ] I searched for [existing GitHub issues](https://github.com/expo/turtle/issues).
+- [ ] **I enclosed the complete output when running turtle, including the stack trace and command used.**
+
+### Issue Description
+<!-- Please include what's happening, expected behavior, and any relevant code samples -->
+
+##### Command executed
+<!-- The command you executed on the command line that resulted in an error -->
+
+##### Complete output when running turtle, including the stack trace and command used
+
+<details>
+  <pre> [PUT LOGS HERE] </pre>
+</details>
+
+### Environment
+
+<!-- Please run `expo diagnostics` and copy the output below. This will help us help you. -->
+
+<details>
+  <pre> [PUT OUTPUT HERE] </pre>
+</details>

.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md

@@ -0,0 +1,22 @@
+---
+name: "\U0001F64B Feature Request"
+about: Want us to add something to turtle-cli?
+title: ''
+labels: 'enhancement'
+assignees: ''
+
+---
+
+### Feature Request
+
+#### Motivation Behind Feature
+<!-- Why should this feature be implemented? What problem does it solve? -->
+
+#### Feature Description
+<!-- Describe your feature request in detail -->
+<!-- Please provide any code examples or screenshots of what this feature would look like -->
+<!-- Are there any drawbacks? Will this break anything for existing users? -->
+
+#### Alternatives or Workarounds
+<!-- Describe alternatives or workarounds you are currently using -->
+<!-- Are there ways to do this with existing commands? -->

.github/ISSUE_TEMPLATE/QUESTION.md

@@ -0,0 +1,21 @@
+---
+name: "\U00002753 Questions and discussions"
+about: You have a question about turtle-cli or want to discuss some aspects of turtle-cli.
+title: ''
+labels: 'question'
+assignees: ''
+
+---
+
+### Question Checklist
+
+- [ ] I read the [Contribution Guidelines](https://github.com/expo/turtle/blob/master/CONTRIBUTING.md).
+- [ ] I read [docs.expo.io](https://docs.expo.io) (Managed Workflow > Distributing Your App).
+- [ ] I searched for [existing GitHub issues](https://github.com/expo/turtle/issues).
+
+### Question Subject
+<!-- What command do you have a question about? -->
+<!-- Is this a question about documentation? -->
+
+#### Question Description
+<!-- Please include expected behavior and any relevant code samples with your question if possible -->

.github/PULL_REQUEST_TEMPLATE

@@ -1,11 +0,0 @@
-# Why
-
-Please describe the motivation for this PR, and link to relevant GitHub issues, forums posts, or feature requests.
-
-# How
-
-How did you build this feature or fix this bug and why?
-
-# Test Plan
-
-Please describe how you tested this change and how a reviewer could reproduce your test, especially if this PR does not include automated tests! If possible, please also provide terminal output and/or screenshots demonstrating your test/reproduction.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,15 @@
+<!-- Thanks for contributing to _turtle_! Before you submit your pull request, please make sure to check the following boxes by putting an x in the [ ] (don't: [x ], [ x], do: [x]) -->
+
+### Checklist
+- [ ] I've read the [Contribution Guidelines](https://github.com/expo/turtle/blob/master/CONTRIBUTING.md).
+- [ ] I've updated the [CHANGELOG](https://github.com/expo/turtle/blob/master/CHANGELOG.md) if necessary.
+- [ ] I've ensured the unit and smoke tests are still passing - either by running `yarn test:unit` and `yarn test:smoke:[ios|android]` or by checking the appropriate CircleCI builds' statuses.
+- [ ] **I've manually tested whether the changes I made work as expected.**
+
+### Motivation and Context
+<!-- Why is this change required? What problem does it solve? -->
+<!-- If it fixes an open issue, please link to the issue here. -->
+
+### Description
+<!-- Describe your changes in detail -->
+<!-- Please describe in detail how you tested your changes. -->

CHANGELOG.md

@@ -6,9 +6,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.8.11] - 2019-07-16
+### Fixed
+- Android debug builds.
+
+## [0.8.10] - 2019-07-16
+### Fixed
+- iOS simulator builds.
+
+## [0.8.9] - 2019-07-15
+### Fixed
+- User authentication.
+
+## [0.8.8] - 2019-07-11
+### Fixed
+- LocalAuthentication module on iOS (fixes [expo/expo#663](https://github.com/expo/expo/issues/663)).
+
+## [0.8.7] - 2019-07-05
+### Added
+- Adhoc Builds: use provisioningProfileId if specified.
+- Registering supported SDK versions in Redis.
+- Support for 64 bit builds in SDK33.
+
+## [0.8.6] - 2019-06-25
+### Added
+- Added support for Android build modes: debug or release.
+### Fixed
+- Upgrading the shell app for a given SDK version when the previous version is already downloaded (bug in `turtle-cli`).
+- Updated Android shell app for SDK33 to include `[email protected]` which fixes crashes on Android 9.0 when showing some Lottie animations.
+- Schema for `buildModes`.
+- Updated @expo/xdl to 55.0.7 (fixed reading from binary plist).
+
 ## [0.8.4] - 2019-06-13
 ### Changed
 - Upgraded `sharp` from `0.21.3` to `0.22.1`.
+- `gulp-cli` is no longer required to build an app with `turtle-cli`.
 
 ## [0.8.3] - 2019-06-13
 ### Added
@@ -116,7 +148,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.5.10] - 2019-02-19
 ### Added
-- Register Turtle version in Redis.
+- Registering Turtle version in Redis.
 ### Changed
 - Update Android shell app for SDK 32 to fix [update](https://github.com/expo/expo/issues/3504) and [back button](https://github.com/expo/expo/issues/1786) issues.
 

CONTRIBUTING.md

@@ -16,9 +16,6 @@ This repository contains the codebase for the `turtle-cli` NPM package. The same
 ### Branches
 The `master` branch of the repository should be kept releasable at any time. This way we can continuously release fixes and improvements without costly managing of different branches and issues will be noticed and fixed quickly. This also ensures other contributors can check out the latest version from GitHub and work on it with minimal disruption from other features in progress.
 
-## Setting up the repository for development
-*TODO*
-
 ## Submitting a pull request
 To submit a pull request:
 1. Fork the [repository](https://github.com/expo/turtle) and create a feature branch. (Existing contributors can create feature branches without forking. Prefix the branch name with `@your-github-username/`.)

DEVELOPMENT.md

@@ -0,0 +1,35 @@
+# Turtle Development
+
+## Prerequisites
+
+- Working `www` service, see [this README](https://github.com/expo/universe/tree/master/server/www/README.md) for more details.
+- Own AWS SQS queues. See [this README](https://github.com/expo/universe/tree/master/terraform/turtle-queues/README.md) to learn more.
+
+## Setup
+
+- Run `update-local-secrets` to populate the `.secrets*` files. You'll need to run this any time any secret changes (it shouldn't be very often).
+- Run `EXPO_UNIVERSE_DIR=/path/to/universe/dir yarn secrets:init-private YOUR_SQS_NICKNAME`. Alternatively, you can create `.secrets.local-queue` file with the SQS queues' urls yourself (`AWS_SQS_OUT_QUEUE_URL`, `AWS_SQS_ANDROID_QUEUE_URL`, `AWS_SQS_ANDROID_PRIORITY_QUEUE_URL`, `AWS_SQS_IOS_QUEUE_URL`, `AWS_SQS_IOS_PRIORITY_QUEUE_URL`)
+- Run `yarn set-workingdir:local` or `yarn set-workingdir:remote`:
+  - `:local` creates symlinks to the shell apps in `expo` repository (_unfortunately this is currently broken_)
+  - `:remote` downloads precompiled shell apps
+- Run `yarn` to install node modules.
+- Run `yarn start:ios` or `yarn start:android` to run Turtle.
+
+## Updating shell apps
+
+_TODO_
+
+## Generating recording files for smoke tests
+
+Smoke tests are powered by [Nock](https://www.npmjs.com/package/nock) which uses JSON files containing responses to the HTTP requests fired by the build process.
+
+To record new responses remove old `recording.json` and run tests and then manually schedule a build with `expo-cli` tool.
+
+As a precaution, the file with HTTP requests/responses contains, among other things, iOS certificates used for the test build, is
+encrypted with `git-crypt`. Remember not to revoke them, because then the tests will fail.
+
+## Other notes
+
+Turtle uses Redis to get notifications (from www) about builds cancellations. To make this feature working, you have to use the same Redis instance as www. Therefore, you need to remember about starting www before starting Turtle. If you forget about it, don't worry, Turtle will remind you to run `yarn start-docker` in `server/www` directory.
+
+Turtle uploads build artifacts to S3. This could take some time if you have a poor internet connection. To facilitate local development, there is a feature which allows Turtle to fake artifacts upload. `secrets:init-private` script, which you should have run while setting up Turtle, sets fake upload directory to your user's default `Downloads` directory (we're assuming you're using Mac). If you need to test uploading artifacts to S3, you have to edit `.secrets.local` file and comment or remove `TURTLE_FAKE_UPLOAD` and `TURTLE_FAKE_UPLOAD_DIR` environment variables.

README.md

@@ -15,21 +15,20 @@ Turtle CLI is a command line interface for building Expo standalone apps.
 You can use it both on your CI and your private computer.
 
 - [Changelog](CHANGELOG.md)
-- [Contributing to Turtle CLI](CONTRIBUTING.md)
+- [Contributing](CONTRIBUTING.md)
+- [Development](DEVELOPMENT.md)
 - [Code of Conduct](CODE_OF_CONDUCT.md)
 
-## Glossary
-- **Turtle** - script which takes a shell app, fills it with user's code and produces a standalone app
-- **Shell app** — precompiled (in case of iOS builds) Expo app which doesn't have user's code
-- **Standalone app** - .ipa/.apk file, a user uploads standalone apps to the Google Play Store/Apple App Store
-
 ## Installation
+
 [Installation instructions and documentation here.](https://docs.expo.io/versions/latest/distribution/turtle-cli)
 
 ## Usage example
+
 See the [expo/turtle-cli-example](https://github.com/expo/turtle-cli-example) repository to learn how to use Turtle CLI in order to build Expo standalone apps on your CI.
 
 ## Do you need help?
+
 - Make sure you've read [the Expo Docs](https://docs.expo.io/), especially [Building Standalone Apps on Your CI
 ](https://docs.expo.io/versions/latest/distribution/turtle-cli) and [Building Standalone Apps](https://docs.expo.io/versions/latest/distribution/building-standalone-apps) guides.
 - Search for existing GitHub issues [in this repository](https://github.com/expo/turtle/issues) and in [expo/expo](https://github.com/expo/expo/issues) repository.
@@ -39,5 +38,6 @@ See the [expo/turtle-cli-example](https://github.com/expo/turtle-cli-example) re
   * ask a question on [Expo Developers Slack](http://slack.expo.io).
 
 ## License
+
 The Turtle CLI source code is made available under the [MIT license](LICENSE).
 Some of the dependencies are licensed differently, with the BSD license, for example.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "turtle-cli",
-  "version": "0.8.4",
+  "version": "0.8.10",
   "description": "The command-line tool for building standalone Expo apps",
   "preferGlobal": true,
   "scripts": {
@@ -13,14 +13,10 @@
     "prepare": "rimraf ./build && yarn build",
     "build": "ttsc",
     "build:production": "./scripts/build.sh",
+    "watch": "ttsc --watch",
     "secrets:init-private": "./scripts/initPrivateSecrets.sh",
     "set-workingdir:remote": "./scripts/setWorkingdirMode.sh remote",
     "set-workingdir:local": "./scripts/setWorkingdirMode.sh local",
-    "docker:pull": "retry5 gcloud docker -- pull gcr.io/exponentjs/turtle-android:latest",
-    "docker:pull:tag": "retry5 gcloud docker -- pull gcr.io/exponentjs/turtle-android:${TAG}",
-    "docker:build": "yarn build:production android && gcloud docker -- build -t gcr.io/exponentjs/turtle-android:latest -t gcr.io/exponentjs/turtle-android:${TAG} .",
-    "docker:push": "retry5 gcloud docker -- push gcr.io/exponentjs/turtle-android:${TAG} && retry5 gcloud docker -- push gcr.io/exponentjs/turtle-android:latest",
-    "deploy:android": "./scripts/android/deploy.sh",
     "test": "yarn jest",
     "test:integration": "yarn test --config jest/integration-config.json",
     "test:unit": "yarn test --config jest/unit-config.json",
@@ -28,19 +24,7 @@
     "test:smoke:android": "PLATFORM=android yarn test:smoke --testNamePattern=android",
     "test:smoke:ios": "PLATFORM=ios yarn test:smoke --testNamePattern=iOS",
     "release": "./scripts/release.sh",
-    "release:beta": "./scripts/releaseBeta.sh",
-    "ssh:android:pods-names": "./scripts/ssh/findAndroidPods.sh",
-    "ssh:android:connect": "./scripts/ssh/connectToAndroidPod.sh",
-    "ssh:ios:ips": "./scripts/ssh/findIosIps.sh",
-    "set-priorities": "yarn set-priorities:local",
-    "set-priorities:default": "expenv ts-node -r tsconfig-paths/register --files src/scripts/setConfigurations.ts",
-    "set-priorities:local": "EXPENV_OVERRIDES=local-common,local,local-queue yarn set-priorities:default",
-    "set-priorities:staging": "yarn set-priorities:default",
-    "set-priorities:production": "EXPENV_OVERRIDES=prod yarn set-priorities:default",
-    "set-replica-count:default": "expenv ts-node -r tsconfig-paths/register --files src/scripts/setReplicaCount.ts | bunyan -o short -l debug",
-    "set-replica-count:local": "EXPENV_OVERRIDES=local-common,local,local-queue yarn set-replica-count:default",
-    "set-replica-count:staging": "yarn set-replica-count:default",
-    "set-replica-count:production": "EXPENV_OVERRIDES=prod yarn set-replica-count:default"
+    "release:beta": "./scripts/releaseBeta.sh"
   },
   "husky": {
     "hooks": {
@@ -79,11 +63,13 @@
     "registry": "https://registry.npmjs.org/"
   },
   "devDependencies": {
+    "@expo/babel-preset-cli": "^0.1.3",
     "@types/decompress": "^4.2.3",
     "@types/fs-extra": "^5.0.4",
     "@types/jest": "^23.3.9",
     "@types/joi": "^13.4.3",
     "@types/lodash": "^4.14.129",
+    "@types/npm-package-arg": "^6.1.0",
     "@types/progress": "^2.0.1",
     "@types/raven": "^2.5.1",
     "@types/request": "^2.47.1",
@@ -110,7 +96,7 @@
   },
   "dependencies": {
     "@expo/spawn-async": "^1.4.2",
-    "@expo/xdl": "54.1.4",
+    "@expo/xdl": "56.1.2-rc.1",
     "@google-cloud/logging-bunyan": "^0.9.5",
     "async-retry": "^1.2.1",
     "aws-sdk": "^2.226.1",
@@ -138,6 +124,6 @@
     "which": "^1.3.1"
   },
   "optionalDependencies": {
-    "@expo/traveling-fastlane-darwin": "1.9.4"
+    "@expo/traveling-fastlane-darwin": "1.9.9"
   }
 }

scripts/deploy.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+SCRIPT_NAME=`basename "$0"`
+DEPLOY_ENDPOINT_URL="https://circleci.com/api/v1.1/project/github/expo/turtle-deploy/tree/master"
+
+if [[ -z "$1" || -z "$2" ]]; then
+  echo "usage: ${bold}$SCRIPT_NAME PLATFORM ENVIRONMENT [FORCE]${normal}"
+  exit 1
+fi
+
+PLATFORM="$1"
+ENVIRONMENT="$2"
+
+if [[ "$3" = "--force" ]]; then
+  FORCE_DEPLOY="true"
+fi
+
+if [[ "$PLATFORM" != "android" && "$PLATFORM" != "ios" ]]; then
+  echo "unsupported PLATFORM passed: ${bold}$PLATFORM${normal}"
+  echo "valid values are: ${bold}android${normal}, ${bold}ios${normal}"
+  exit 1
+fi
+
+if [[ "$ENVIRONMENT" != "staging" && "$ENVIRONMENT" != "production" ]]; then
+  echo "unsupported ENVIRONMENT passed: ${bold}$ENVIRONMENT${normal}"
+  echo "valid values are: ${bold}staging${normal}, ${bold}production${normal}"
+  exit 1
+fi
+
+if [[ ! -z "$CIRCLE_SHA1" ]]; then
+  TURTLE_COMMIT="$CIRCLE_SHA1"
+elif [[ -z "$FORCE_DEPLOY" ]]; then
+  read -p "You're about to schedule a deploy, are you sure you want to deploy the current commit? (y/n) " -n 1 -r
+  echo    # (optional) move to a new line
+  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "exiting..."
+    exit 0
+  else
+    TURTLE_COMMIT=`git log --pretty=format:%H -1`
+  fi
+fi
+
+if [[ -z "$TURTLE_BOT_CIRCLE_API_USER_TOKEN" ]]; then
+  echo "Please set the TURTLE_BOT_CIRCLE_API_USER_TOKEN env variable."
+  exit 1
+fi
+
+if [[ $FORCE_DEPLOY = "true" ]]; then
+  curl -X POST \
+    --header "Content-Type: application/json" \
+    -u $TURTLE_BOT_CIRCLE_API_USER_TOKEN: \
+    -d '
+      {
+        "build_parameters": {
+          "CIRCLE_JOB": "deploy_'$PLATFORM'",
+          "TURTLE_ENV": "'$ENVIRONMENT'",
+          "TURTLE_FORCE_DEPLOY": "true"
+        }
+      }' \
+    $DEPLOY_ENDPOINT_URL | jq -r .build_url
+else
+  curl -X POST \
+    --header "Content-Type: application/json" \
+    -u $TURTLE_BOT_CIRCLE_API_USER_TOKEN: \
+    -d '
+      {
+        "build_parameters": {
+          "CIRCLE_JOB": "deploy_'$PLATFORM'",
+          "TURTLE_ENV": "'$ENVIRONMENT'",
+          "TURTLE_COMMIT": "'$TURTLE_COMMIT'"
+        }
+      }' \
+    $DEPLOY_ENDPOINT_URL | jq -r .build_url
+fi

scripts/initPrivateSecrets.sh

@@ -13,13 +13,17 @@ fi
 TURTLE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
 TERRAFORM_DIR="${EXPO_UNIVERSE_DIR}/terraform"
 
-ANDROID=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_android_in | head -1 | cut -d"=" -f2 | xargs`
-IOS=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_ios_in | head -1 | cut -d"=" -f2 | xargs`
+ANDROID_NORMAL=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_android_in | head -1 | cut -d"=" -f2 | xargs`
+ANDROID_PRIORITY=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_android_in_priority | head -1 | cut -d"=" -f2 | xargs`
+IOS_NORMAL=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_ios_in | head -1 | cut -d"=" -f2 | xargs`
+IOS_PRIORITY=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_ios_in_priority | head -1 | cut -d"=" -f2 | xargs`
 OUT=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_out | head -1 | cut -d"=" -f2 | xargs`
 
 cat << EOF > $TURTLE_DIR/.secrets.local-queue
-AWS_SQS_ANDROID_QUEUE_URL="$ANDROID"
-AWS_SQS_IOS_QUEUE_URL="$IOS"
+AWS_SQS_ANDROID_QUEUE_URL="$ANDROID_NORMAL"
+AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="$ANDROID_PRIORITY"
+AWS_SQS_IOS_QUEUE_URL="$IOS_NORMAL"
+AWS_SQS_IOS_PRIORITY_QUEUE_URL="$IOS_PRIORITY"
 AWS_SQS_OUT_QUEUE_URL="$OUT"
 EOF
 

scripts/setWorkingdirMode.sh

@@ -6,42 +6,44 @@ DOWNLOAD_DIR="${EXPO_DOWNLOAD_DIR:-$HOME/Downloads}"
 
 mkdir -p $DOWNLOAD_DIR
 
-if [ ! -f "$ROOT_DIR/.secrets.local-queue" ]; then 
-  echo "You need to run \"yarn secrets:init-private [SQS_NAME]\" or create .secrets.local-queue file yourself with AWS_SQS_ANDROID_QUEUE_URL, AWS_SQS_IOS_QUEUE_URL and AWS_SQS_OUT_QUEUE_URL variables."
+if [ ! -f "$ROOT_DIR/.secrets.local-queue" ]; then
+  echo "You need to run \"yarn secrets:init-private [SQS_NAME]\" or create the .secrets.local-queue file yourself with AWS_SQS_ANDROID_QUEUE_URL, AWS_SQS_ANDROID_PRIORITY_QUEUE_URL, AWS_SQS_IOS_QUEUE_URL, AWS_SQS_IOS_PRIORITY_QUEUE_URL and AWS_SQS_OUT_QUEUE_URL variables."
   exit 1
 fi
 
-setup_local() {
-  ensure_link_exists() {
-    if [ ! -L "$2" ]; then
-      ln -s $1 $2
-      echo "$2 link created"
-    fi
-  }
-  WORKINGDIR="$ROOT_DIR/workingdir/local"
-  
-  if [[ -d $WORKINGDIR ]]; then
-    echo "Local workingdir already prepared"
-    return
-  fi
-
-  if [[ -z "$EXPO_UNIVERSE_DIR" ]]; then
-    echo "EXPO_UNIVERSE_DIR needs to be set"
-    exit -1
-  fi
-
-  cd $EXPO_UNIVERSE_DIR/exponent && yarn
-  cd $EXPO_UNIVERSE_DIR/exponent/tools-public && yarn && ./generate-dynamic-macros-android.sh
-
-  mkdir -p $WORKINGDIR
-  echo "$WORKINGDIR directory created"
-
-  LINKS=("template-files" "package.json" "android" "expokit-npm-package" "packages" "client-builds" "cpp" "exponent-view-template" "ios" "shellAppBase-builds" "shellAppWorkspaces" "node_modules")
-
-  for LINKNAME in ${LINKS[@]}; do
-    ensure_link_exists "$EXPO_UNIVERSE_DIR/exponent/$LINKNAME" "$WORKINGDIR/$LINKNAME"
-  done
-}
+# @dsokal: this is currently broken
+#
+# setup_local() {
+#   ensure_link_exists() {
+#     if [ ! -L "$2" ]; then
+#       ln -s $1 $2
+#       echo "$2 link created"
+#     fi
+#   }
+#   WORKINGDIR="$ROOT_DIR/workingdir/local"
+#
+#   if [[ -d $WORKINGDIR ]]; then
+#     echo "Local workingdir already prepared"
+#     return
+#   fi
+#
+#   if [[ -z "$EXPO_UNIVERSE_DIR" ]]; then
+#     echo "EXPO_UNIVERSE_DIR needs to be set"
+#     exit -1
+#   fi
+#
+#   cd $EXPO_UNIVERSE_DIR/exponent && yarn
+#   cd $EXPO_UNIVERSE_DIR/exponent/tools-public && yarn && ./generate-dynamic-macros-android.sh
+#
+#   mkdir -p $WORKINGDIR
+#   echo "$WORKINGDIR directory created"
+#
+#   LINKS=("template-files" "package.json" "android" "expokit-npm-package" "packages" "client-builds" "cpp" "exponent-view-template" "ios" "shellAppBase-builds" "shellAppWorkspaces" "node_modules")
+#
+#   for LINKNAME in ${LINKS[@]}; do
+#     ensure_link_exists "$EXPO_UNIVERSE_DIR/exponent/$LINKNAME" "$WORKINGDIR/$LINKNAME"
+#   done
+# }
 
 setup_remote() {
   if [[ -d $ROOT_DIR/workingdir/android && -d $ROOT_DIR/workingdir/ios ]]; then
@@ -55,7 +57,7 @@ setup_remote() {
       $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
     fi
   fi
-  
+
   for sdk in $ROOT_DIR/workingdir/android/* ; do
     if [[ -f "$sdk/universe-package.json" ]]; then
       mv $sdk/package.json $sdk/exponent-package.json
@@ -87,8 +89,10 @@ EOF
 }
 
 if [ "$1" == "local" ]; then
-  setup_local
-  switch_local
+  # setup_local
+  # switch_local
+  echo "Setting local workingdir is currently broken, sorry :("
+  exit -1
 elif [ "$1" == "remote" ]; then
   setup_remote
   switch_remote

scripts/ssh/connectToAndroidPod.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-if [ -z "$1" ]; then
-  echo "Please provide an environment name as a first argument"
-  exit 1
-fi
-
-if [ -z "$2" ]; then
-  echo "Please provide a pod name as a second argument"
-  exit 1
-fi
-
-kubectl --namespace $1 exec -it $2 -- /bin/bash

scripts/ssh/findAndroidPods.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-ENVIRONMENT=${1:-production}
-
-kubectl get pods --namespace $ENVIRONMENT | grep "turtle-android-" | sed -e "s/  */ /g" | cut -d" " -f1

scripts/ssh/findIosIps.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-ENVIRONMENT=${1:-production}
-
-govc vm.ip -wait 5s turtle-js-"${ENVIRONMENT}"-*

shellTarballs/android/sdk33

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-e244e865ed8cb8e20d29ea00b6373845d1c61068.tar.gz
+s3://exp-artifacts/android-shell-builder-aebd669d6573b0b3c802871d94cdb8b4116678d2.tar.gz

src/bin/commands/build/android.ts

@@ -23,9 +23,15 @@ export default (program: any, setCommonCommandOptions: any) => {
       /^(app-bundle|apk)$/i,
       'app-bundle',
     )
+    .option(
+      '-m --mode <build>',
+      'type of build: debug|release',
+      /^(debug|release)$/i,
+      'release',
+    )
     .description(
-      'Build a standalone APK or App Bundle for your project, signed and ready for submission to the Google Play'
-      + ' Store.',
+      'Build a standalone APK or App Bundle for your project, either signed and ready for submission to'
+      + ' the Google Play Store or in debug mode.',
     )
     .asyncAction(
       createBuilderAction({

src/bin/setup/android/index.ts

@@ -20,10 +20,6 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
     command: 'bash',
     missingDescription: 'Please install bash',
   },
-  {
-    command: 'gulp',
-    missingDescription: 'Run `npm install -g gulp-cli` (or `yarn global add gulp-cli`) to install gulp',
-  },
   {
     command: 'javac',
     missingDescription: 'Please install JDK (version 8 or newer) - check https://jdk.java.net/',

src/bin/setup/android/ndk.ts

@@ -4,7 +4,7 @@ import path from 'path';
 import decompress from 'decompress';
 import fs from 'fs-extra';
 
-import { formatArtifactDownloadPath } from 'turtle/bin/setup/utils/common';
+import * as utils from 'turtle/bin/setup/utils/common';
 import download from 'turtle/bin/setup/utils/downloader';
 import config from 'turtle/config';
 import logger from 'turtle/logger';
@@ -16,9 +16,10 @@ const l = logger.child(LOGGER_FIELDS);
 
 export default async function ensureAndroidNDKIsPresent() {
   const androidNdkDir = path.join(config.directories.androidDependenciesDir, 'ndk');
+  await utils.removeDirectoryUnlessReady(androidNdkDir);
   if (!(await fs.pathExists(androidNdkDir))) {
     await fs.ensureDir(androidNdkDir);
-    const androidNdkDownloadPath = formatArtifactDownloadPath(ANDROID_NDK_URL);
+    const androidNdkDownloadPath = utils.formatArtifactDownloadPath(ANDROID_NDK_URL);
 
     try {
       l.info('Downloading Android NDK');
@@ -26,6 +27,7 @@ export default async function ensureAndroidNDKIsPresent() {
       await download(ANDROID_NDK_URL, androidNdkDownloadPath);
       l.info('Decompressing Android NDK');
       await decompress(androidNdkDownloadPath, androidNdkDir, { strip: 1 });
+      await utils.markDirectoryAsReady(androidNdkDir);
       l.info('Android NDK installed successfully');
     } catch (err) {
       await fs.remove(androidNdkDir);

src/bin/setup/android/sdk.ts

@@ -5,7 +5,7 @@ import { ExponentTools } from '@expo/xdl';
 import decompress from 'decompress';
 import fs from 'fs-extra';
 
-import { formatArtifactDownloadPath } from 'turtle/bin/setup/utils/common';
+import * as utils from 'turtle/bin/setup/utils/common';
 import download from 'turtle/bin/setup/utils/downloader';
 import config from 'turtle/config';
 import logger from 'turtle/logger';
@@ -17,9 +17,10 @@ const l = logger.child(LOGGER_FIELDS);
 
 export default async function ensureAndroidSDKIsPresent() {
   const androidSdkDir = path.join(config.directories.androidDependenciesDir, 'sdk');
+  await utils.removeDirectoryUnlessReady(androidSdkDir);
   if (!(await fs.pathExists(androidSdkDir))) {
     await fs.ensureDir(androidSdkDir);
-    const androidSdkDownloadPath = formatArtifactDownloadPath(ANDROID_SDK_URL);
+    const androidSdkDownloadPath = utils.formatArtifactDownloadPath(ANDROID_SDK_URL);
 
     try {
       l.info('Downloading Android SDK');
@@ -31,6 +32,7 @@ export default async function ensureAndroidSDKIsPresent() {
       await fs.remove(androidSdkDownloadPath);
       l.info('Configuring Android SDK, this may take a while');
       await _configureSdk(androidSdkDir);
+      await utils.markDirectoryAsReady(androidSdkDir);
       l.info('Android SDK installed and configured successfully');
     } catch (err) {
       await fs.remove(androidSdkDir);

src/bin/setup/utils/common.ts

@@ -29,6 +29,8 @@ export async function ensureShellAppIsPresent(
   postDownloadAction?: PostDownloadAction,
 ) {
   const workingdir = formatters.formatShellAppDirectory({ sdkVersion });
+  const shellAppMetadata = await _readShellAppTarballS3Uri(sdkVersion, formatters);
+  await removeDirectoryUnlessReady(workingdir, shellAppMetadata);
   if (await fs.pathExists(workingdir)) {
     return;
   }
@@ -37,6 +39,7 @@ export async function ensureShellAppIsPresent(
   if (postDownloadAction) {
     await postDownloadAction(sdkVersion, workingdir);
   }
+  await markDirectoryAsReady(workingdir, shellAppMetadata);
 }
 
 async function _downloadShellApp(sdkVersion: string, targetDirectory: string, formatters: IShellAppFormaters) {
@@ -65,3 +68,31 @@ async function _readShellAppTarballS3Uri(sdkVersion: string, formatters: IShellA
   const data = await fs.readFile(filePath, 'utf8');
   return data.trim();
 }
+
+export async function removeDirectoryUnlessReady(dir: string, metadata?: string) {
+  const readyFilePath = path.join(dir, '.ready');
+  const readyFileExists = await fs.pathExists(readyFilePath);
+  let shouldRemove = false;
+  if (readyFileExists) {
+    if (metadata !== undefined) {
+      const readyFileContents = (await fs.readFile(readyFilePath, 'utf-8')).trim();
+      if (readyFileContents !== metadata) {
+        shouldRemove = true;
+       }
+    }
+  } else {
+    shouldRemove = true;
+  }
+  if (shouldRemove) {
+    await fs.remove(dir);
+  }
+}
+
+export async function markDirectoryAsReady(dir: string, metadata?: string) {
+  const readyFilePath = path.join(dir, '.ready');
+  if (metadata === undefined) {
+    await fs.open(readyFilePath, 'w');
+  } else {
+    await fs.writeFile(readyFilePath, metadata);
+  }
+}

src/bin/utils/builder.ts

@@ -57,6 +57,7 @@ export function createBuilderAction({
       const args = {
         releaseChannel: cmd.releaseChannel || 'default',
         buildType: cmd.type,
+        buildMode: cmd.mode,
         username: userData.username || 'anonymous',
         projectDir: ProjectUtils.resolveAbsoluteDir(projectDirArg),
         publicUrl: cmd.publicUrl,
@@ -88,13 +89,14 @@ export function createBuilderAction({
 const buildJobObject = async (
   platform: 'android' | 'ios',
   appJSON: any,
-  { releaseChannel, buildType, username, publicUrl, projectDir }: any,
+  { releaseChannel, buildType, buildMode, username, publicUrl, projectDir }: any,
   credentials: any,
 ) => {
   const job = {
     config: {
       ..._.get(appJSON, 'expo.ios.config', {}),
       buildType,
+      ...(platform === 'android' ? { buildMode } : {}),
       releaseChannel,
       ...(platform === 'ios' ? { bundleIdentifier: _.get(appJSON, 'expo.ios.bundleIdentifier') } : {}),
       ...(platform === 'android' ? { androidPackage: _.get(appJSON, 'expo.android.package') } : {}),

src/bin/utils/user.ts

@@ -1,4 +1,4 @@
-import { User as UserManager } from '@expo/xdl';
+import { UserManager } from '@expo/xdl';
 
 import { ErrorWithProgramHelp } from 'turtle/bin/commands/ErrorWithProgramHelp';
 

src/builders/android.ts

@@ -93,6 +93,7 @@ async function runShellAppBuilder(
       outputFile: outputFilePath,
       modules: enabledModules,
       buildType: jobData.config.buildType,
+      buildMode: jobData.config.buildMode,
     });
   } catch (err) {
     commonUtils.logErrorOnce(err);

src/builders/utils/android/credentials.ts

@@ -1,6 +1,6 @@
 import path from 'path';
 
-import { Credentials } from '@expo/xdl';
+import { AndroidCredentials } from '@expo/xdl';
 import fs from 'fs-extra';
 import get from 'lodash/get';
 import uuidv4 from 'uuid/v4';
@@ -29,7 +29,7 @@ async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredential
       );
     }
 
-    await Credentials.Android.createKeystore(
+    await AndroidCredentials.createKeystore(
       {
         keystorePath: keystoreFilename,
         keystorePassword: credentials.keystorePassword,

src/builders/utils/ios/adhocBuild.ts

@@ -3,7 +3,6 @@ import path from 'path';
 import spawnAsync from '@expo/spawn-async';
 import { IosCodeSigning } from '@expo/xdl';
 import fs from 'fs-extra';
-import isEmpty from 'lodash/isEmpty';
 
 import * as sqs from 'turtle/aws/sqs';
 import BuildError, { BuildErrorReason } from 'turtle/builders/BuildError';
@@ -21,10 +20,18 @@ async function prepareAdHocBuildCredentials(job: IJob) {
   }
 
   const { bundleIdentifier } = job.config;
-  const { certP12, certPassword, teamId, appleSession, udids } = job.credentials;
+  const {
+    certP12,
+    certPassword,
+    teamId,
+    appleSession,
+    udids,
+    provisioningProfileId,
+  } = job.credentials;
 
   const certSerialNumber = IosCodeSigning.findP12CertSerialNumber(certP12, certPassword);
   const args = [
+    ...(provisioningProfileId ? ['--profile-id', provisioningProfileId] : []),
     teamId,
     udids!.join(','),
     bundleIdentifier,
@@ -32,25 +39,26 @@ async function prepareAdHocBuildCredentials(job: IJob) {
   ];
 
   try {
+    if (provisioningProfileId) {
+      logger.info(
+        `Using ad hoc provisioning profile id: ${provisioningProfileId}`,
+      );
+    }
+
     const credentials = await runFastlaneAction(
       travelingFastlane.manageAdHocProvisioningProfile,
       args,
       { env: { FASTLANE_SESSION: appleSession } },
     );
-    if (isEmpty(credentials)) {
-      if (!job.credentials.provisioningProfile) {
-        // a user has generated the provisioning profile for his app, but he probably deleted it from Expo servers, ...
-        throw new BuildError(
-          BuildErrorReason.PROVISIONING_PROFILE_MISSING,
-          `Provisioning profile is missing on Expo servers.`,
-        );
-      }
-      return;
-    }
 
     logger.info('New ad hoc provisioning profile successfully created');
     job.credentials.provisioningProfile = credentials.provisioningProfile;
 
+    if (!credentials.provisioningProfileUpdateTimestamp) {
+      //  dont need to persist profile because nothing changed
+      return;
+    }
+
     if (isOffline()) {
       const provisioningProfilePath = path.join(job.projectDir, `adhoc.mobileprovision`);
       await fs.writeFile(provisioningProfilePath, Buffer.from(credentials.provisioningProfile, 'base64'));

src/builders/utils/version.ts

@@ -5,26 +5,32 @@ import path from 'path';
 import config from 'turtle/config';
 import { IJob } from 'turtle/job';
 
-let SUPPORTED_SDK_VERSIONS: number[];
+interface IVersionsMap {
+  [platform: string]: number[];
+}
+
+const versions: IVersionsMap = {};
 
 export async function ensureCanBuildSdkVersion(job: IJob) {
   if (config.builder.useLocalWorkingDir) {
     return;
   }
 
-  if (!SUPPORTED_SDK_VERSIONS) {
-    SUPPORTED_SDK_VERSIONS = await findSupportedSdkVersions(job.platform);
-  }
-
+  const platformVersions = await findSupportedSdkVersions(job.platform);
   const targetMajorSdkVersion = ExponentTools.parseSdkMajorVersion(job.sdkVersion || job.manifest.sdkVersion);
-  if (!SUPPORTED_SDK_VERSIONS.includes(targetMajorSdkVersion)) {
+  if (!platformVersions.includes(targetMajorSdkVersion)) {
     throw new Error(`Unsupported SDK Version!`);
   }
 }
 
-async function findSupportedSdkVersions(platform: 'android' | 'ios'): Promise<number[]> {
-  const SDK_DIR_PREFIX = 'sdk';
-  const files = await fs.readdir(path.join(config.directories.workingDir, platform));
-  const sdks = files.filter((file) => file.startsWith(SDK_DIR_PREFIX));
-  return sdks.map((sdk) => parseInt(sdk.substr(SDK_DIR_PREFIX.length), 10));
+export async function findSupportedSdkVersions(platform: 'android' | 'ios'): Promise<number[]> {
+  if (versions[platform]) {
+    return versions[platform];
+  } else {
+    const SDK_DIR_PREFIX = 'sdk';
+    const files = await fs.readdir(path.join(config.directories.workingDir, platform));
+    const sdks = files.filter((file) => file.startsWith(SDK_DIR_PREFIX));
+    versions[platform] = sdks.map((sdk) => parseInt(sdk.substr(SDK_DIR_PREFIX.length), 10));
+    return versions[platform];
+  }
 }

src/constants/index.ts

@@ -16,4 +16,9 @@ export enum ANDROID_BUILD_TYPES {
   APP_BUNDLE = 'app-bundle',
 }
 
+export enum ANDROID_BUILD_MODES {
+  DEBUG = 'debug',
+  RELEASE = 'release',
+}
+
 export { BUILD };

src/job.ts

@@ -1,4 +1,4 @@
-import { ANDROID_BUILD_TYPES, IOS_BUILD_TYPES } from 'turtle/constants';
+import { ANDROID_BUILD_MODES, ANDROID_BUILD_TYPES, IOS_BUILD_TYPES } from 'turtle/constants';
 
 export interface IJob {
   platform: 'android' | 'ios';
@@ -14,6 +14,7 @@ export interface IJob {
     bundleIdentifier?: string;
     // android
     androidPackage?: string;
+    buildMode?: ANDROID_BUILD_MODES;
   };
   credentials: {
     // android
@@ -25,6 +26,7 @@ export interface IJob {
     certP12?: string;
     certPassword?: string;
     provisioningProfile?: string;
+    provisioningProfileId?: string;
     teamId?: string;
     appleSession?: string;
     udids?: string[];

src/jobsSchemas/android.ts

@@ -1,6 +1,6 @@
 import Joi from 'joi';
 
-import { PLATFORMS } from 'turtle/constants';
+import { ANDROID_BUILD_MODES, ANDROID_BUILD_TYPES, PLATFORMS } from 'turtle/constants';
 import baseJobSchema from 'turtle/jobsSchemas/base';
 
 export default baseJobSchema.concat(
@@ -12,5 +12,9 @@ export default baseJobSchema.concat(
       keystoreAlias: Joi.string(),
       keystore: Joi.string(),
     }),
+    config: Joi.object().keys({
+      buildMode: Joi.string().valid(Object.values(ANDROID_BUILD_MODES)).default(ANDROID_BUILD_MODES.RELEASE),
+      buildType: Joi.string().valid(Object.values(ANDROID_BUILD_TYPES)).default(ANDROID_BUILD_TYPES.APK),
+    }),
   }),
 );

src/jobsSchemas/ios.ts

@@ -17,6 +17,7 @@ export default baseJobSchema.concat(
         teamId: Joi.string().required(),
         // it's not required for adhoc Expo Client builds
         provisioningProfile: Joi.string(),
+        provisioningProfileId: Joi.string(),
         appleSession: Joi.string(),
         udids: Joi.array().items(Joi.string()),
       })

src/logger/index.ts

@@ -17,6 +17,7 @@ class Logger {
       name: 'turtle',
       level: config.logger.level,
       platform: config.platform,
+      serializers: bunyan.stdSerializers,
       ...config.deploymentEnv && { environment: config.deploymentEnv },
       streams: Object.values(streams),
     });

src/metrics/buildersConfig.ts

@@ -12,7 +12,14 @@ const createModeMetricName = (mode: TurtleModeLabels) => `${createMetricName()}.
 const createModeTag = (mode: TurtleModeLabels) => `mode:${mode}`;
 
 export default async function sendConfigToDatadog() {
-  const configuration = await getLabeledConfiguration();
+  let configuration;
+  try {
+    configuration = await getLabeledConfiguration();
+  } catch (err) {
+    logger.warn('Failed to send the builders configuration to Datadog because it couldn\'t be fetched from Redis');
+    return;
+  }
+
   const replicaCount = await getReplicaCount();
   const currentReplicaCount = replicaCount === null ? configuration.length : replicaCount;
 

src/scripts/setConfigurations.ts

@@ -1,80 +0,0 @@
-
-import { PLATFORMS } from 'turtle/constants';
-import logger from 'turtle/logger';
-import {
-  createConfigurationsKey,
-  createDefaultConfigurationKey,
-  HIGH_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-} from 'turtle/utils/priorities';
-import { connect, RedisClient } from 'turtle/utils/redis';
-
-const TIMEOUT = 3000;
-
-// Turtle before doing anything (and after every build) checks for the first config
-// (an element in config table with key=i) that is not taken (config.i is null/outdated)
-// or is taken by this process itself.
-// If Turtle dies unexpectedly the configuration will become free after TTL expires,
-// so the state (number of HP/NP turtles) should come back to normal soon.
-
-// For example, if a configuration table looks like this:
-// [ HIGH_CONFIGURATION, NORMAL_CONFIGURATION, HIGH_CONFIGURATION ]
-// If there is one turtle it runs with High Priority first.
-// If there are two turtles, one runs with High Priority first, the second one with Normal Priority first.
-// If the first dies, and the third is started in his place, it gets 3rd config (again HP-first),
-// but after some time, because of TTL for config reservations,
-// they should come back to using configurations no. 1 and 2.
-// If there are no more available configurations from the table, the default configuration is used.
-
-const AVAILABLE_CONFIGURATIONS_IOS = [
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  // there are 5 iOS builders at the moment
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-];
-
-const AVAILABLE_CONFIGURATIONS_ANDROID = [
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  // there are 15 Android builders at the moment
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-];
-
-const DEFAULT_CONFIGURATION = HIGH_CONFIGURATION;
-
-async function run() {
-  const redis = await connect(TIMEOUT, RedisClient.Configuration);
-
-  await redis.set(createConfigurationsKey(PLATFORMS.IOS), JSON.stringify(AVAILABLE_CONFIGURATIONS_IOS));
-  await redis.set(createConfigurationsKey(PLATFORMS.ANDROID), JSON.stringify(AVAILABLE_CONFIGURATIONS_ANDROID));
-  await redis.set(createDefaultConfigurationKey(PLATFORMS.IOS), JSON.stringify(DEFAULT_CONFIGURATION));
-  await redis.set(createDefaultConfigurationKey(PLATFORMS.ANDROID), JSON.stringify(DEFAULT_CONFIGURATION));
-
-  redis.disconnect();
-}
-
-if (require.main === module) {
-  run()
-    .then(() => logger.info('All done'))
-    .catch((err) => logger.error('Error:', err));
-}

src/scripts/setReplicaCount.ts

@@ -1,38 +0,0 @@
-import isNaN from 'lodash/isNaN';
-
-import { PLATFORMS } from 'turtle/constants';
-import logger from 'turtle/logger';
-import { setReplicaCount } from 'turtle/utils/metadata';
-import { getRedisClient, RedisClient } from 'turtle/utils/redis';
-
-async function run() {
-  const platform = process.env.SCRIPT_PLATFORM;
-  const replicaCountRaw = process.env.SCRIPT_REPLICA_COUNT;
-  const replicaCount = Number(replicaCountRaw);
-
-  if (!platform || !replicaCountRaw) {
-    throw new Error('You must set both SCRIPT_PLATFORM and SCRIPT_REPLICA_COUNT env variables.');
-  }
-
-  if (!Object.values(PLATFORMS).includes(platform)) {
-    throw new Error(`Unsupported platform: ${platform}`);
-  }
-
-  if (isNaN(replicaCount)) {
-    throw new Error(`Wrong value for replica count: ${replicaCountRaw}`);
-  }
-
-  const redis = await getRedisClient(RedisClient.Configuration);
-  logger.info(`[${platform}] Setting replica count in redis to ${replicaCount}`);
-  await setReplicaCount(platform as PLATFORMS, replicaCount);
-  redis.disconnect();
-}
-
-if (require.main === module) {
-  run()
-    .then(() => logger.info('All done'))
-    .catch((err) => {
-      logger.error('Error:', err);
-      process.exit(1);
-    });
-}

src/server.ts

@@ -1,13 +1,15 @@
 import { LoggerDetach } from '@expo/xdl';
 
+import { findSupportedSdkVersions } from 'turtle/builders/utils/version';
 import config from 'turtle/config';
 import { doJob } from 'turtle/jobManager';
 import logger from 'turtle/logger';
 import setup from 'turtle/setup';
 import { checkShouldExit, setShouldExit, turtleVersion } from 'turtle/turtleContext';
-import { getRedisClient, RedisClient } from 'turtle/utils/redis';
+import { getRedisClient } from 'turtle/utils/redis';
 
 const REDIS_TURTLE_VERSION_KEY = 'turtle:version';
+const REDIS_TURTLE_SUPPORTED_SDK_VERSIONS_KEY = `turtle:${config.platform}:sdkVersions`;
 
 process.on('unhandledRejection', (err) => logger.error({ err }, 'Unhandled promise rejection'));
 
@@ -35,11 +37,18 @@ async function main() {
   }
 
   try {
-    const redis = await getRedisClient(RedisClient.Configuration);
-    await redis.set(REDIS_TURTLE_VERSION_KEY, turtleVersion);
-    logger.info(`Register Turtle version ${turtleVersion} in Redis`);
+    const redis = await getRedisClient();
+    if (redis) {
+      await redis.set(REDIS_TURTLE_VERSION_KEY, turtleVersion);
+      logger.info(`Registered Turtle version (${turtleVersion}) in Redis`);
+      const sdkVersions = (await findSupportedSdkVersions(config.platform)).join(',');
+      await redis.set(REDIS_TURTLE_SUPPORTED_SDK_VERSIONS_KEY, sdkVersions);
+      logger.info(`Registered supported SDK versions (${sdkVersions}) in Redis`);
+    } else {
+      logger.warn('Couldn\'t set supported SDK versions in Redis (failed to connect to Redis)');
+    }
   } catch (err) {
-    logger.error({ err }, 'Failed to register Turtle version in Redis');
+    logger.error({ err }, 'Failed to set versions in Redis');
   }
 
   while (true) {

src/utils/metadata.ts

@@ -5,14 +5,6 @@ import { getRedisClient, RedisClient } from 'turtle/utils/redis';
 const createKeyPrefix = (platform: PLATFORMS) => `${config.deploymentEnv}:${platform}`;
 const createReplicaCountKey = (platform: PLATFORMS) => `${createKeyPrefix(platform)}:replicaCount`;
 
-export async function setReplicaCount(platform: PLATFORMS, replicaCount: number) {
-  const redis = await getRedisClient(RedisClient.Configuration);
-  await redis.set(
-    createReplicaCountKey(platform),
-    replicaCount,
-  );
-}
-
 export async function getReplicaCount(maybePlatform?: PLATFORMS) {
   const platform = maybePlatform || config.platform;
   const redis = await getRedisClient(RedisClient.Configuration);

src/utils/priorities.ts

@@ -49,7 +49,7 @@ export function createDefaultConfigurationKey(platform = config.platform) {
 }
 
 export async function getPriorities() {
-  if (config.env === 'test') {
+  if (config.env === 'test' || config.deploymentEnv === 'development') {
     return NORMAL_CONFIGURATION;
   }
 
@@ -93,7 +93,7 @@ export async function getPriorities() {
   }
 }
 
-export async function getConfiguration(currentRedisClient: any) {
+async function getConfiguration(currentRedisClient: any) {
   const redis = currentRedisClient || await getRedisClient(RedisClient.Configuration);
   return JSON.parse(await redis.get(createConfigurationsKey()));
 }

src/utils/redis.ts

@@ -64,7 +64,7 @@ export async function checkIfCancelled(jobId: string) {
     return await redis.get(`jobs:cancelled:${jobId}`);
   } catch (err) {
     if (config.deploymentEnv === 'development') {
-      logger.warn('Did you turn on redis server? Run `yarn start-docker` in server/www');
+      logger.warn('Did you turn on redis server? Run `yarn start-docker` in $EXPO_UNIVERSE_DIR/server/www');
     }
     logger.error({ err });
     return false;

supervisord.conf

@@ -7,6 +7,10 @@ command=node ./build/server.js
 autorestart=true
 startretries=5
 startsecs=600
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
 
 [eventlistener:turtle_failure]
 events=PROCESS_STATE_FATAL

tsconfig.json

@@ -18,7 +18,8 @@
     "allowJs": true,
     "plugins": [
       { "transform": "ts-transformer-imports" }
-    ]
+    ],
+    "typeRoots": ["node_modules/@expo/babel-preset-cli/ts-declarations", "node_modules/@types"]
   },
   "include": ["src"],
   "exclude": ["node_modules", "build", "workingdir"]