Release 0.17.3

Bumps [http-proxy](https://github.com/http-party/node-http-proxy) from 1.17.0 to 1.18.1.
- [Release notes](https://github.com/http-party/node-http-proxy/releases)
- [Changelog](https://github.com/http-party/node-http-proxy/blob/master/CHANGELOG.md)
- [Commits](https://github.com/http-party/node-http-proxy/compare/1.17.0...1.18.1)

Signed-off-by: dependabot[bot] <[email protected]>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.circleci/config.yml

@@ -9,7 +9,7 @@ executors:
     working_directory: ~/turtle
   mac:
     macos: # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions
-      xcode: '10.1.0'
+      xcode: "11.1.0"
     working_directory: /Users/distiller/project
     shell: /bin/bash -leo pipefail
   android:
@@ -57,8 +57,8 @@ commands:
       - run:
           name: Configure aws-cli
           command: |
-              aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
-              aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+            aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
+            aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
 
   setup_nix:
     steps:
@@ -77,6 +77,49 @@ commands:
             EOF
             chmod +x /bin/rm
 
+  install_nodejs_mac:
+    parameters:
+      nodejs_version:
+        type: string
+        default: 12.16.1
+      yarn_version:
+        type: string
+        default: 1.22.0
+    steps:
+      - restore_cache:
+          key: "node-js-v1-darwin-archive-<< parameters.nodejs_version >>"
+      - run:
+          name: Uninstall yarn with brew
+          command: brew uninstall --force yarn
+      - install_node_and_yarn:
+          platform: darwin
+          nodejs_version: "<< parameters.nodejs_version >>"
+          yarn_version: "<< parameters.yarn_version >>"
+      - save_cache:
+          key: "node-js-v1-darwin-archive-<< parameters.nodejs_version >>"
+          paths:
+            - /private/tmp/nodejs.tar.xz
+
+  install_node_and_yarn:
+    parameters:
+      platform:
+        type: string
+      nodejs_version:
+        type: string
+      yarn_version:
+        type: string
+    steps:
+      - run:
+          name: Install node & yarn
+          command: |
+            NODEJS_ARCHIVE="/tmp/nodejs.tar.xz"
+            if [ ! -f "$NODEJS_ARCHIVE" ]; then
+              curl -o $NODEJS_ARCHIVE -SL "https://nodejs.org/dist/v<< parameters.nodejs_version >>/node-v<< parameters.nodejs_version >>-<< parameters.platform >>-x64.tar.xz"
+            fi
+            tar -xJf $NODEJS_ARCHIVE -C /usr/local --strip-components=1 --exclude CHANGELOG.md --exclude README.md --exclude LICENSE
+            ln -s /usr/local/bin/node /usr/local/bin/nodejs
+            npm install --global yarn@<< parameters.yarn_version >>
+
   decrypt_secrets_if_possible:
     steps:
       - run:
@@ -93,18 +136,24 @@ commands:
       workingdir:
         type: string
         default: .
+      platform:
+        type: string
+        default: linux
     steps:
       - restore_cache:
-          key: yarn-v1-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
+          key: yarn-v3-<< parameters.platform >>-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
 
   save_yarn_cache:
     parameters:
       workingdir:
         type: string
         default: .
+      platform:
+        type: string
+        default: linux
     steps:
       - save_cache:
-          key: yarn-v1-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
+          key: yarn-v3-<< parameters.platform >>-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
           paths:
             - ~/.cache/yarn # `yarn cache dir`
 
@@ -115,10 +164,10 @@ commands:
         default: ios
       sdk_version:
         type: string
-        default: sdk36
+        default: sdk37
     steps:
       - restore_cache:
-          key: "workingdir-v1-<< parameters.platform >>-<< parameters.sdk_version >>"
+          key: "workingdir-v2-<< parameters.platform >>-<< parameters.sdk_version >>"
 
   save_workingdir_cache:
     parameters:
@@ -130,10 +179,10 @@ commands:
         default: ios
       sdk_version:
         type: string
-        default: sdk36
+        default: sdk37
     steps:
       - save_cache:
-          key: "workingdir-v1-<< parameters.platform >>-<< parameters.sdk_version >>"
+          key: "workingdir-v2-<< parameters.platform >>-<< parameters.sdk_version >>"
           paths:
             - "<< parameters.workingdir >>/<< parameters.platform >>/<< parameters.sdk_version >>"
 
@@ -180,20 +229,23 @@ jobs:
     executor: mac
     steps:
       - install_nix
+      - install_nodejs_mac
       - setup
       - decrypt_secrets_if_possible
-      - restore_yarn_cache
+      - restore_yarn_cache:
+          platform: darwin
       - run: yarn
-      - save_yarn_cache
+      - save_yarn_cache:
+          platform: darwin
       - restore_workingdir_cache:
-          sdk_version: sdk35
+          sdk_version: sdk37
           platform: ios
       - run:
           name: Run iOS smoke tests
-          command: ./scripts/tests/runIOSSmokeTestsCI.sh
+          command: SDK_VERSION=sdk37 ./scripts/tests/runIOSSmokeTestsCI.sh
       - save_workingdir_cache:
           workingdir: /Users/distiller/project/workingdir
-          sdk_version: sdk35
+          sdk_version: sdk37
           platform: ios
 
   android_smoke_tests:
@@ -205,11 +257,11 @@ jobs:
       - run: yarn
       - save_yarn_cache
       - restore_workingdir_cache:
-          sdk_version: sdk36
+          sdk_version: sdk37
           platform: android
       - run:
           name: Fetch Android shell app
-          command: SDK_VERSION=sdk36 ./scripts/android/fetchRemoteAndroidTarball.sh
+          command: SDK_VERSION=sdk37 ./scripts/android/fetchRemoteAndroidTarball.sh
       - run:
           name: Run Android smoke tests
           command: |
@@ -217,7 +269,7 @@ jobs:
               --run "./scripts/tests/runAndroidSmokeTestsCI.sh"
       - save_workingdir_cache:
           workingdir: ~/turtle/workingdir
-          sdk_version: sdk36
+          sdk_version: sdk37
           platform: android
 
   android_staging_deploy:

.envrc

@@ -22,6 +22,6 @@ export NODE_ENV=development
 export EXPO_TURTLE_TOOLS_DIR="$(pwd)/tools"
 
 PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
-if [ "$CI" = "true" ]; then
+if [ "$CI" = "true" ] && [[ ! $OSTYPE =~ darwin.* ]]; then
   PATH_add $EXPO_TURTLE_TOOLS_DIR/ci/bin
 fi

.gitignore

@@ -6,6 +6,7 @@ google-cloud-credentials.json
 /workingdir/*
 /artifacts/*
 .history
+.vscode/
 
 shell.apk
 .secrets*

.release-it.json

@@ -10,6 +10,9 @@
     "release": true
   },
   "hooks": {
+    "before:init": [
+      "yarn"
+    ],
     "after:release": "yarn release:update:turtle-cli-example"
   }
 }

CHANGELOG.md

@@ -9,6 +9,137 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added gradle 6.2 to `turtle-android-base`
+- Added `ANDROID_NDK_HOME` in `turtle-android-base`
+- Upgraded XDL so it processes root `build.gradle` of Android shell apps too
+
+## [0.17.2] - 2020-08-06
+
+### Changed
+
+- Updated iOS shell app for SDK 38 with fixed integer tagging in expo-gl for iOS 14. ([#245](https://github.com/expo/turtle/pull/245))
+
+## [0.17.1] - 2020-08-05
+
+### Changed
+
+- Updated Android shell app for SDK 38 with `targetSdkVersion` bumped to 29. ([#244](https://github.com/expo/turtle/pull/244))
+
+## [0.17.0] - 2020-07-16
+
+### Added
+
+- `--gradle-args` option for `turtle build:android` which makes it possible to specify custom Gradle arguments.
+
+### Changed
+
+- Gradle Wrapper doesn't print when dots when the appropriate version of Gradle is being downloaded.
+
+## [0.16.2] - 2020-07-01
+
+### Added
+
+- Added `--allow-non-https-public-url` to allow bypass of protocol validation on `--public-url`.
+- Added support for dynamic configs (`app.config.ts`, `app.config.js`).
+
+## [0.16.0] - 2020-06-25
+
+### Added
+
+- Added iOS shell app for Expo SDK38.
+- Added Android shell app for SDK 38.
+
+### Removed
+
+- Dropped support for SDK 34.
+
+## [0.15.1] - 2020-06-22
+
+### Changed
+
+- Configure Turtle to use the Redis CA certificate.
+
+### Fixed
+
+- Fix announcing supported SDK versions.
+
+## [0.15.0] - 2020-06-02
+
+### Changed
+
+- Updated `@expo/xdl` to 57.9.13 (added \*.bak files clean-up when building .ipa).
+
+## [0.14.12] - 2020-05-14
+
+### Changed
+
+- Update iOS custom client shell app with various fixes.
+
+## [0.14.11] - 2020-05-04
+
+### Changed
+
+- Update SDK 37 iOS shell app with various fixes.
+
+## [0.14.10] - 2020-05-01
+
+### Added
+
+- support for "android.enableDangerousExperimentalLeanBuilds" in app.json.
+
+## [0.14.9] - 2020-04-21
+
+### Changed
+
+- Updated `@expo/xdl` to 57.8.30 (scoped the build command so it only builds the `:app` Android sub-project and not all the sub-projects available, [PR](https://github.com/expo/expo-cli/pull/1937)).
+
+## [0.14.8] - 2020-04-20
+
+### Changed
+
+- Updated `@expo/xdl` to 57.8.29 (removed call to `check-dynamic-macros-android.sh`, [PR](https://github.com/expo/expo-cli/pull/1933)).
+
+## [0.14.7] - 2020-04-20
+
+### Changed
+
+- Turtle no longer installs all dependencies of shell apps projects — only the production dependencies are installed now.
+- Updated `@expo/xdl` to 57.8.25 (fixed unintuitive `google-services.json` handling for SDK37+, [PR](https://github.com/expo/expo-cli/pull/1897)).
+
+## [0.14.6] - 2020-04-09
+
+### Changed
+
+- Updated @expo/xdl to 57.8.17 (fixed downloading assets smaller than 10 bytes).
+
+## [0.14.5] - 2020-04-06
+
+### Changed
+
+- Made @expo/config error more meaningful.
+
+## [0.14.4] - 2020-04-06
+
+### Fixed
+
+- Determining SDK version for Expo apps using SDK >=37.
+
+## [0.14.3] - 2020-04-03
+
+### Fixed
+
+- A fix for Android SDK 37 - `check-dynamic-macros-android.sh` doesn't use `jq` anymore.
+
+## [0.14.1] - 2020-04-01
+
+### Changed
+
+- Updated iOS shell app for SDK 37 to remove unnecessary unimodules.
+
+## [0.14.0] - 2020-03-31
+
+### Added
+
 - Added iOS shell app for SDK 37.
 - Added Android shell app for SDK 37.
 

DEVELOPMENT.md

@@ -8,7 +8,7 @@
 ## Setup
 
 - Run `update-local-secrets` to populate the `.secrets*` files. You'll need to run this any time any secret changes (it shouldn't be very often).
-- Run `EXPO_UNIVERSE_DIR=/path/to/universe/dir yarn secrets:init-private YOUR_SQS_NICKNAME`. Alternatively, you can create `.secrets.local-queue` file with the SQS queues' urls yourself (`AWS_SQS_OUT_QUEUE_URL`, `AWS_SQS_ANDROID_QUEUE_URL`, `AWS_SQS_ANDROID_PRIORITY_QUEUE_URL`, `AWS_SQS_IOS_QUEUE_URL`, `AWS_SQS_IOS_PRIORITY_QUEUE_URL`)
+- Run `EXPO_UNIVERSE_DIR=/path/to/universe/dir yarn secrets:init-private YOUR_SQS_NICKNAME`. Alternatively, you can create `.secrets.local-queue` file with the SQS queues' urls yourself (`AWS_SQS_ANDROID_QUEUE_URL`, `AWS_SQS_ANDROID_PRIORITY_QUEUE_URL`, `AWS_SQS_IOS_QUEUE_URL`, `AWS_SQS_IOS_PRIORITY_QUEUE_URL`)
 - Run `yarn set-workingdir:local` or `yarn set-workingdir:remote`:
   - `:local` creates symlinks to the shell apps in `expo` repository (_unfortunately this is currently broken_)
   - `:remote` downloads precompiled shell apps

Dockerfile

@@ -1,29 +1,28 @@
 # see docker-base/Dockerfile to learn what's installed in the turtle-android-base image
-FROM gcr.io/exponentjs/turtle-android-base:0.4.0
+FROM gcr.io/exponentjs/turtle-android-base:0.5.0
 
 ADD . /app/turtle
 
-RUN mv /app/turtle/workingdir /app && \
-    for SDK_VERSION in `ls /app/workingdir/android/`; do \
-      echo "preparing $SDK_VERSION shell app" && \
-      cd /app/workingdir/android/$SDK_VERSION && \
-      if [ -f universe-package.json ]; then \
-      mv package.json exponent-package.json && \
-      mv universe-package.json package.json && \
-      yarn install && \
-      mv package.json universe-package.json && \
-      mv exponent-package.json package.json; \
-      else \
-      yarn install; \
-      fi \
-    ; done && \
-    mv /app/workingdir /app/turtle
+WORKDIR /app/turtle
+
+RUN ./scripts/fetchRemoteTarballs.sh android
+
+RUN ln -s /app/turtle/workingdir /app/workingdir && \
+  for SDK_VERSION in `ls /app/workingdir/android/`; do \
+  echo "preparing $SDK_VERSION shell app" && \
+  cd /app/workingdir/android/$SDK_VERSION && \
+  mv packages non-workspace-packages && \
+  # Keep in sync with src/bin/setup/android/index.ts#_installNodeModules
+  # --prod requires --ignore-scripts (otherwise
+  # expo-yarn-workspaces errors as missing)
+  yarn install --ignore-scripts --prod && \
+  mv non-workspace-packages packages ; \
+  done
 
 ENV NODE_ENV "production"
 ENV TURTLE_WORKING_DIR_PATH /app/turtle/workingdir/
 
-WORKDIR /app/turtle
-
 ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+ADD redislabs_ca.pem /var/run/config/redislabs-tls/redislabs_ca.pem
 
 CMD ["/usr/bin/supervisord"]

docker-base/Dockerfile

@@ -57,10 +57,23 @@ ENV PATH ${ANDROID_HOME}/build-tools/28.0.3/:${PATH}
 
 ADD configureAndroidSdk.sh /tmp/configureAndroidSdk.sh
 RUN /tmp/configureAndroidSdk.sh --install-all-platforms
+ENV ANDROID_NDK_HOME /usr/local/android-sdk-linux/ndk/17.2.4988734
 
 # Support Gradle
 ENV TERM dumb
 
+# Install gradle 6.2
+RUN wget https://services.gradle.org/distributions/gradle-6.2-all.zip
+RUN unzip -qq gradle-6.2-all.zip
+RUN cp -r gradle-6.2 /usr/local
+
+RUN mkdir -p /root/.gradle/wrapper/dists/gradle-6.2-all/dvufqs6kielxeao781pmk5huj
+RUN mv gradle-6.2-all.zip /root/.gradle/wrapper/dists/gradle-6.2-all/dvufqs6kielxeao781pmk5huj
+RUN mv gradle-6.2 /root/.gradle/wrapper/dists/gradle-6.2-all/dvufqs6kielxeao781pmk5huj
+RUN touch /root/.gradle/wrapper/dists/gradle-6.2-all/dvufqs6kielxeao781pmk5huj/gradle-6.2-all.zip.ok
+RUN touch /root/.gradle/wrapper/dists/gradle-6.2-all/dvufqs6kielxeao781pmk5huj/gradle-6.2-all.zip.lck
+RUN chmod 755 /root/.gradle/wrapper/dists/gradle-6.2-all/dvufqs6kielxeao781pmk5huj/gradle-6.2/bin/gradle
+
 # Install gradle 5.6.3
 RUN wget https://services.gradle.org/distributions/gradle-5.6.3-all.zip
 RUN unzip -qq gradle-5.6.3-all.zip

k8s/android/turtle-deployment.template.yml

@@ -88,11 +88,6 @@ spec:
               secretKeyRef:
                 name: turtle-js-config
                 key: aws_sqs_android_priority_queue_url
-          - name: AWS_SQS_OUT_QUEUE_URL
-            valueFrom:
-              secretKeyRef:
-                name: turtle-js-config
-                key: aws_sqs_out_queue_url
           - name: AWS_CLOUDWATCH_REGION
             valueFrom:
               secretKeyRef:
@@ -149,9 +144,11 @@ spec:
           - name: DISABLE_DEX_MAX_HEAP
             value: "true"
           - name: JAVA_OPTS
-            value: '-Xms2048m -XX:+HeapDumpOnOutOfMemoryError -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:+UseG1GC'
+            value: '-Xms2048m -Xmx8192m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseG1GC'
           - name: GRADLE_OPTS
-            value: '-Dorg.gradle.parallel=true -Dorg.gradle.configureondemand=true -Dorg.gradle.daemon=false -Dorg.gradle.jvmargs="-Xms2048m -XX:+HeapDumpOnOutOfMemoryError -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:+UseG1GC"'
+            value: '-Dorg.gradle.parallel=true -Dorg.gradle.configureondemand=true -Dorg.gradle.daemon=false -Dorg.gradle.jvmargs="-Xms2048m -Xmx8192m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseG1GC"'
+          - name: REDIS_CA_CERTIFICATE
+            value: '/var/run/config/redislabs-tls/redislabs_ca.pem'
         resources:
           requests:
             memory: "8192Mi"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "turtle-cli",
-  "version": "0.13.10",
+  "version": "0.17.3",
   "description": "The command-line tool for building standalone Expo apps",
   "preferGlobal": true,
   "scripts": {
@@ -70,9 +70,11 @@
     "@expo/babel-preset-cli": "^0.1.3",
     "@types/decompress": "^4.2.3",
     "@types/fs-extra": "^5.0.4",
+    "@types/ioredis": "^4.16.6",
     "@types/jest": "^23.3.9",
     "@types/joi": "^13.4.3",
     "@types/lodash": "^4.14.129",
+    "@types/node-forge": "^0.8.6",
     "@types/npm-package-arg": "^6.1.0",
     "@types/progress": "^2.0.1",
     "@types/raven": "^2.5.1",
@@ -86,24 +88,25 @@
     "@types/webpack-dev-server": "^3.1.7",
     "@types/which": "^1.3.1",
     "husky": "^1.3.1",
-    "jest": "^25.1.0",
+    "jest": "^26.1.0",
     "lint-staged": "^8.1.5",
     "nock": "^10.0.1",
     "nodemon": "^2.0.2",
     "release-it": "^12.3.4",
     "rimraf": "^2.6.2",
-    "ts-jest": "^25.2.1",
+    "ts-jest": "^26.1.2",
     "ts-node": "^8.0.3",
     "ts-transformer-imports": "^0.4.1",
     "tsconfig-paths": "^3.8.0",
     "tslint": "^5.10.0",
     "tslint-eslint-rules": "^5.4.0",
     "ttypescript": "^1.5.6",
-    "typescript": "^3.4.2"
+    "typescript": "^3.9.5"
   },
   "dependencies": {
+    "@expo/config": "^3.2.14",
     "@expo/spawn-async": "^1.4.2",
-    "@expo/xdl": "57.5.2",
+    "@expo/xdl": "57.9.35",
     "@google-cloud/logging-bunyan": "^2.0.3",
     "async-retry": "^1.2.1",
     "aws-sdk": "^2.226.1",
@@ -115,7 +118,7 @@
     "decompress": "^4.2.0",
     "dogapi": "^2.8.3",
     "fs-extra": "^6.0.0",
-    "ioredis": "^3.2.2",
+    "ioredis": "^4.17.3",
     "is-semver": "^1.0.8",
     "joi": "^13.2.0",
     "lodash": "^4.17.11",

redislabs_ca.pem

@@ -0,0 +1 @@
+this is just a placeholder to simplify deployments

scripts/build.sh

@@ -14,7 +14,7 @@ PLATFORM=$1
 yarn
 yarn run build
 if [ "$PLATFORM" = "android" ]; then
-  $ROOT_DIR/scripts/android/fetchRemoteAndroidTarball.sh
+  echo "shell apps will be fetched when building a Docker image"
 elif [ "$PLATFORM" = "ios" ]; then
   $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
 fi

scripts/initPrivateSecrets.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 if [ -z "$EXPO_UNIVERSE_DIR" ]; then
-  echo "You need to specify EXPO_UNIVERSE_DIR to use this script. Alternatively you can create .secrets.local-queue file yourself with AWS_SQS_ANDROID_QUEUE_URL, AWS_SQS_IOS_QUEUE_URL and AWS_SQS_OUT_QUEUE_URL variables."
+  echo "You need to specify EXPO_UNIVERSE_DIR to use this script. Alternatively you can create .secrets.local-queue file yourself with AWS_SQS_ANDROID_QUEUE_URL and AWS_SQS_IOS_QUEUE_URL variables."
   exit 1
 fi
 
@@ -17,13 +17,11 @@ ANDROID_NORMAL=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_loc
 ANDROID_PRIORITY=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_android_in_priority | head -1 | cut -d"=" -f2 | xargs`
 IOS_NORMAL=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_ios_in | head -1 | cut -d"=" -f2 | xargs`
 IOS_PRIORITY=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_ios_in_priority | head -1 | cut -d"=" -f2 | xargs`
-OUT=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_out | head -1 | cut -d"=" -f2 | xargs`
 
 cat << EOF > $TURTLE_DIR/.secrets.local-queue
 AWS_SQS_ANDROID_QUEUE_URL="$ANDROID_NORMAL"
 AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="$ANDROID_PRIORITY"
 AWS_SQS_IOS_QUEUE_URL="$IOS_NORMAL"
 AWS_SQS_IOS_PRIORITY_QUEUE_URL="$IOS_PRIORITY"
-AWS_SQS_OUT_QUEUE_URL="$OUT"
 EOF
 

scripts/setWorkingdirMode.sh

@@ -7,7 +7,7 @@ DOWNLOAD_DIR="${EXPO_DOWNLOAD_DIR:-$HOME/Downloads}"
 mkdir -p $DOWNLOAD_DIR
 
 if [ ! -f "$ROOT_DIR/.secrets.local-queue" ]; then
-  echo "You need to run \"yarn secrets:init-private [SQS_NAME]\" or create the .secrets.local-queue file yourself with AWS_SQS_ANDROID_QUEUE_URL, AWS_SQS_ANDROID_PRIORITY_QUEUE_URL, AWS_SQS_IOS_QUEUE_URL, AWS_SQS_IOS_PRIORITY_QUEUE_URL and AWS_SQS_OUT_QUEUE_URL variables."
+  echo "You need to run \"yarn secrets:init-private [SQS_NAME]\" or create the .secrets.local-queue file yourself with AWS_SQS_ANDROID_QUEUE_URL, AWS_SQS_ANDROID_PRIORITY_QUEUE_URL, AWS_SQS_IOS_QUEUE_URL, and AWS_SQS_IOS_PRIORITY_QUEUE_URL variables."
   exit 1
 fi
 

scripts/tests/runAndroidSmokeTestsCI.sh

@@ -2,6 +2,8 @@
 
 set -ueo pipefail
 
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
+
 export NODE_ENV=test
 
 export ENVIRONMENT="development"
@@ -11,7 +13,6 @@ export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/2742511416
 export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in_priority.fifo"
 export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in.fifo"
 export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in_priority.fifo"
-export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_out.fifo"
 export AWS_SQS_REGION="aws.west2"
 export AWS_S3_BUCKET="exp-shell-app-assets"
 export AWS_S3_REGION="us-west-1"
@@ -24,4 +25,8 @@ export REDIS_CONFIG_URL="noop"
 export DATADOG_DISABLED="true"
 export TURTLE_SDK_VERSIONS_SECRET_TOKEN="noop"
 
+pushd $ROOT_DIR/workingdir/android/${SDK_VERSION:-sdk37} >/dev/null 2>&1
+yarn
+popd >/dev/null 2>&1
+
 yarn test:smoke:android

scripts/tests/runIOSSmokeTestsCI.sh

@@ -9,7 +9,6 @@ export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/2742511416
 export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in_priority.fifo"
 export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in.fifo"
 export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in_priority.fifo"
-export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_out.fifo"
 export AWS_SQS_REGION="aws.west2"
 export AWS_S3_BUCKET="exp-shell-app-assets"
 export AWS_S3_REGION="us-west-1"
@@ -23,6 +22,6 @@ export DATADOG_DISABLED="true"
 export TURTLE_SDK_VERSIONS_SECRET_TOKEN="noop"
 
 ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
-SDK_VERSION=sdk35 $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
+$ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
 
 yarn test:smoke:ios

scripts/tests/runUnitTestsCI.sh

@@ -9,7 +9,6 @@ export AWS_SQS_IOS_QUEUE_URL="noop"
 export AWS_SQS_ANDROID_QUEUE_URL="noop"
 export AWS_SQS_IOS_PRIORITY_QUEUE_URL="noop"
 export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="noop"
-export AWS_SQS_OUT_QUEUE_URL="noop"
 export AWS_ACCESS_KEY_ID="noop"
 export AWS_SECRET_ACCESS_KEY="noop"
 export AWS_SQS_REGION="noop"

shellTarballs/android/sdk34

@@ -1 +0,0 @@
-s3://exp-artifacts/android-shell-builder-a8e204610e5712391336b378f47962bbb0c04164.tar.gz

shellTarballs/android/sdk37

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-1a6203cf1555c3b956a311ed539eafeb7594f030.tar.gz
+s3://exp-artifacts/android-shell-builder-3cf3cf0707925322bcd667d5ba79120136f9afd8.tar.gz

shellTarballs/android/sdk38

@@ -0,0 +1 @@
+s3://exp-artifacts/android-shell-builder-6cde8e00cd7bdf7a933cd8aad7f8e86981ea68a2.tar.gz

shellTarballs/ios/client

@@ -1 +1 @@
-s3://exp-artifacts/ios-expo-client-aadfc350d8e4ebf8dce9ce08723bb7e07d563846.tar.gz
+s3://exp-artifacts/ios-expo-client-18a58afe742e63f1d2ac71ba21404d8b84e6563e.tar.gz

shellTarballs/ios/sdk34

@@ -1 +0,0 @@
-s3://exp-artifacts/ios-shell-builder-sdk-latest-651588ff3445a5d87b3a5b81ec2977c02b22f3a2.tar.gz

shellTarballs/ios/sdk37

@@ -1 +1 @@
-s3://exp-artifacts/ios-shell-builder-sdk-latest-aadfc350d8e4ebf8dce9ce08723bb7e07d563846.tar.gz
+s3://exp-artifacts/ios-shell-builder-sdk-latest-30eb0558ff6a77acf56bb045e98b84f6a5e1478a.tar.gz

shellTarballs/ios/sdk38

@@ -0,0 +1 @@
+s3://exp-artifacts/ios-shell-builder-sdk-latest-6cde8e00cd7bdf7a933cd8aad7f8e86981ea68a2.tar.gz

src/__smoke__/fixtures/android/recording.json

@@ -1 +1 @@
-[{"scope":"https://sqs.us-west-2.amazonaws.com:443","method":"POST","path":"/","body":"Action=ReceiveMessage&MaxNumberOfMessages=1&QueueUrl=https%3A%2F%2Fsqs.us-west-2.amazonaws.com%2F274251141632%2Fturtle_jobs_local_dominik_android_in_priority.fifo&Version=2012-11-05&VisibilityTimeout=30","status":200,"response":"<?xml version=\"1.0\"?><ReceiveMessageResponse xmlns=\"http://queue.amazonaws.com/doc/2012-11-05/\"><ReceiveMessageResult><Message><MessageId>fc2fe53d-1447-4304-90e0-60b025ac9687</MessageId><ReceiptHandle>AQEBcOA9S4yhL3ZCaho9Cv+XVqJEbhOaaWJTsOvBD9SOa/NK/248s0+yI9bHro5Ti9WJrnD3H65SdK9eOETFeS/QDaaCMTYkxjvH/1DrSJt2rAptgItmxl8F6N3iDYLqplOQjmautdjCWYXrCLxMDihMHEjcvWNAq0XLAAjlQJvtm5Z8Vji5xXKJP3IpQOV2DvhS/Cqriteaqe6NW5xKwgKHOC52IVTPMSHiJ0ERUf55agntOOJJ7Za7vHdTr+Zjrxl6yVFWKC/cpwYI00OKGj4bArud0buiaD0rtjszLmcPgcwADRsmhfG65p3gLwIwjfclHqSLlmcuY3c+xsu/5VM1YA==</ReceiptHandle><MD5OfBody>c9e8a99bf73407dfa26a2cb81103c26a</MD5OfBody><Body>{&quot;id&quot;:&quot;b9f9f814-5603-4dd5-929d-c9eed756c591&quot;,&quot;experienceId&quot;:&quot;531b2db0-4b14-11e9-92d6-95aebd1034a3&quot;,&quot;experienceName&quot;:&quot;@dsokal-local/turtle-cli-example&quot;,&quot;platform&quot;:&quot;android&quot;,&quot;config&quot;:{&quot;releaseChannel&quot;:&quot;default&quot;},&quot;status&quot;:&quot;sent-to-queue&quot;,&quot;sdkVersion&quot;:&quot;32.0.0&quot;,&quot;accountId&quot;:&quot;4c44c9d8-3cb6-4980-8505-ca0637cf3165&quot;,&quot;priority&quot;:&quot;high&quot;,&quot;credentials&quot;:{&quot;keystore&quot;:&quot;/u3+7QAAAAIAAAABAAAAAQAscWdyemIydGhiYzFzYjJuaGJjOTBkeGowYmd1dHkyeHBsd3Y0eXcxd2JndT0AAAFpycAadgAABQIwggT+MA4GCisGAQQBKgIRAQEFAASCBOrpcy5jViAnKMlnwuvGQ2qa5UcYfuJQwaa48a4oEcvZlrPUUvQSi1T/ZOVpWc8bhiAD8xvTbTh0w1B26K4bPrbrchQGyubs5o3tZU94I+1yMR//nITj4srlUPipX7kr2KdGDqfLfS4EwK8FXACyXx3YxC30STovgtOjTctGUQRzD2iQmu9UmlCAl4szItY2vn+j51VJBpjV9SWXrWpEOmH6hkq7xlHFhbr00FpJXnzON6ifg7ymMIUbkG2nLC3VGCVTAqWBsOgEnUExMOKTedl6Rdzze5sP9CmzUk4aUpgZK1CHjvY03/RztuFky85ODEfSLrk3zBnegQgg0x54RQq1ji+/s5IyPslfpZbcVsTM1KBLEstfCkhZT8xjCZeUxD44U1UxcMlXRYsWtpHv5gfG/4vvV9xQ+pqzuCD43DLiTrqUFw9VzIzXjU7L9CXNGN0DtcmTI5usoKhLZPYgnpT546H4rnZkPzxYs3HXd+tyBmqFTWbvoRiFI2njeaMu0/RZMR17VKjv1u7rITpi7xjwX8cxPvxHLxoDyLL/0vxu45qfmH9ScOWw0QAxQ6RybMhZuZYXuHCERrJ4DLgXNqX8yDjba/C4ioF4nl4cL1S3JtUUE8aRaB6KKQaXP2Fxj+mww3eqetWBuCBD0yBHVscqCZfePh3b1hotMaVZvu/VrxkFL+BbnOkeSinwhkHB2sLFW3wzE5NQHbMi3chaHavC0jxFfaqf8aARub+5I/0+tqI3Crj46EdcxBMpPVa82HAYLkM8pSL8Tjs15bHg+v5suU+DEAymFPauDfEOIzbAJYdBmyrLiL5HBBSM39TdjdpXm1TOdBBtpOlROpJJsOpd3HRIqBF6yHyyRSrPZ5b5k42KCbMheRREXssqZoyPtZWT8aUdi9SnwhZm/fhNPVie1F7GjfsWi5U6OLpSHk2J6GOiwBFk5AXXoe4LNqcAMwYng5a23xEyAJ2JQ4sV5Hb8asNOfVjKvwLSKhRkG1wa+zsku2B0wFa7czX3r/u+NsEfOSbgo8ICya8itu8nlaoCtol/484Ab7NHiPJpF1qeZDB5n57+aODG7rjpBXJLvB0BTI4x9bPOEEzEu6cUEo6pvVIYGb5wTfgKVZVv5pA40x61/OicbG9dR8W6co/MfCDA0BMREzxO6djh8aSphR8Wd3jFYw1MI+PzCllnD3L4/6ZTc7R4sa9EUIfIxIFp7hw48HFO6I9uhp79lQySZ+EcEFFsvMK1hRGR6lfgX8ZWqgnJP1fx0n7PXyzFQvCEsw6GEesu6IVco39vNs1WL//FhhGf+HU2/Jq7wFOjN4yQtU/h2CEG5fv/aKGswoMv2Kqiihd4LjpxQlQHlaOXjp5QnBVMl6/8QVvX50JieabW3uXmcbOGnQ8qAOyg4OCgPzJmShifOeLuDPUQIvG4Tm7514lPZJ1aGFF9tj/zEBjxJfs7N30lUG+TfcPeFQDYfwmUZED2nZgsDdNQhkkucIf53Zgly9tKzXodRdAqwSv9qAeX69nbC2Db9J//E7CaDMLgm/Ij/w11MVlikMYv0FE95PdJ/vLbWPzEF1OwxXtutlg2TPksdSU7Cl7oJNk4B++fekquRmjPGzb8EEVIsTwmzZ77LOnre/F9Zy6nGX5mGCzhnYsBux6cRaDdwcYZ6UDm2PO88c5LEiIuAAAAAQAFWC41MDkAAANhMIIDXTCCAkWgAwIBAgIEHWp77zANBgkqhkiG9w0BAQsFADBfMQswCQYDVQQGEwJVUzEJMAcGA1UECBMAMQkwBwYDVQQHEwAxCTAHBgNVBAoTADEJMAcGA1UECxMAMSQwIgYDVQQDExtjb20uZXhwby50dXJ0bGUuY2xpLmV4YW1wbGUwHhcNMTkwMzI5MTQwMDAzWhcNNDYwODE0MTQwMDAzWjBfMQswCQYDVQQGEwJVUzEJMAcGA1UECBMAMQkwBwYDVQQHEwAxCTAHBgNVBAoTADEJMAcGA1UECxMAMSQwIgYDVQQDExtjb20uZXhwby50dXJ0bGUuY2xpLmV4YW1wbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCw5E26XXa9AHihXC2gDfA9Wi4kHOhLiC5D6D1iS3Ftc+UCgg1d4pEzVB1o5O81IhWyIdTINkmpZ/qgDi1DCWXEgFHSisHFWWisXy/Ab2zRSPkE0GsDlnNvq2wObLzhEnLyALZ57+5EjQRCQJlzbO3kCFPhf/hT4hnVrsF+nnObkiEnUPCGLebHYqzk6lvMQ2XERzTyIovHD+TvfbIpKWOjSeZeEQ/9DI6Yo9bkZ4rRs9Z1vX1UTMtvEEg/T9oyCY0uE5h6yz49eH3Q2M27DqYEC3VaDhSBSELnItfSBnay/6vaQSlEzB+FGW6Jt9rTDQ1tMayUGQKvxhXi+08NHm7vAgMBAAGjITAfMB0GA1UdDgQWBBQiidnk1c5N5F4Aeg/EEPiuJO6ApDANBgkqhkiG9w0BAQsFAAOCAQEAkPbJQpnfoZehkPPNOHvyFl5S/POROColqo4OwfaYF0LmFfhMOPi3B2iuEXDPc/lVliuhQ+b+kD+pQnqlcDxWXEP8ZJETFjVSMLAeZ43MoTs22DBKh6sMYrO7MyjZbIwMMALmnfniHQe58gV7ot0sgBCs0bN3hICpqHXzZn6QSLxW1F9W5/52XFEwl+6zwTKnphZPNBhOCPQ/Z6cCEmodrKD48YeEOu+laojg3TZhVEF8Wf6TdcHjgyaT2IRPF+J6aD2K9ZazLFN/tvz0j5+1EPYtm1l6iX9RMFg0GtbRngXcIhuPXrcNuThF5/c2dL13JNYtMHUe3AAOfkycMNVjeTDkzLXZiULCDIisCnPuJQ9cRz7N&quot;,&quot;keyPassword&quot;:&quot;fb7409988d78459c826cf2a438cb3a33&quot;,&quot;keystoreAlias&quot;:&quot;QGRzb2thbC1sb2NhbC90dXJ0bGUtY2xpLWV4YW1wbGU=&quot;,&quot;keystorePassword&quot;:&quot;80c3583b33b84a1d91e5cf33b2421d7c&quot;},&quot;manifest&quot;:{&quot;android&quot;:{&quot;package&quot;:&quot;com.expo.turtle.cli.example&quot;},&quot;bundledAssets&quot;:[&quot;asset_3a2ba31570920eeb9b1d217cabe58315.ttf&quot;,&quot;asset_744ce60078c17d86006dd0edabcd59a7.ttf&quot;,&qLine truncated
+[{"scope":"https://sqs.us-west-2.amazonaws.com:443","method":"POST","path":"/","body":"Action=ReceiveMessage&MaxNumberOfMessages=1&QueueUrl=https%3A%2F%2Fsqs.us-west-2.amazonaws.com%2F274251141632%2Fturtle_jobs_local_dominik_android_in.fifo&Version=2012-11-05&VisibilityTimeout=30","status":200,"response":"<?xml version=\"1.0\"?><ReceiveMessageResponse xmlns=\"http://queue.amazonaws.com/doc/2012-11-05/\"><ReceiveMessageResult><Message><MessageId>f5017f35-ce75-4a6c-acaf-4ccc3d73899f</MessageId><ReceiptHandle>AQEBQjEPeX13Rkr+iSNEuh9J+WKCinb45vRFxWMojfZlD8f5gXi1fncRIotDjji9Q5lkhEK6hjXnALwGXZTstFj3/EPMVV2MKPNN/6NA3cQ7wSd5/+EqP8ZFOC+wBBK46KdS82t46NJ3c0DqkXPImlUE7O2n1WnFiRrjJ+I/G7yYhf+rnxLoNWD8ZPFzzInr6K0Q4Fae8Wqowyc9N8dw2kksnzW3qto+W/Ped1MyF4/Ia6KBqPLQswHRI5WRrqxxaVKdi37XxHW1J71xxEskicTlSBqQmre+3gaCja6cDpXkBtQSNp1LC7nvt/OXZTmXvwtM</ReceiptHandle><MD5OfBody>792da27a1e1bc549ed89f35a5f66e38c</MD5OfBody><Body>{&quot;id&quot;:&quot;cd9ff227-575b-4eec-874f-d2b9f6187f05&quot;,&quot;experienceId&quot;:&quot;da491dac-c6bf-4d54-b67a-ba7890e22fb2&quot;,&quot;experienceName&quot;:&quot;@dsokal-local/turtle-cli-example&quot;,&quot;platform&quot;:&quot;android&quot;,&quot;config&quot;:{&quot;releaseChannel&quot;:&quot;default&quot;,&quot;buildType&quot;:&quot;apk&quot;},&quot;status&quot;:&quot;sent-to-queue&quot;,&quot;sdkVersion&quot;:&quot;37.0.0&quot;,&quot;accountId&quot;:&quot;fbd6360b-e85d-485e-bf20-ab66adfeea16&quot;,&quot;priority&quot;:&quot;normal&quot;,&quot;credentials&quot;:{&quot;keystore&quot;:&quot;/u3+7QAAAAIAAAABAAAAAQAscWd2NGNnOHRkaHZ5ZGd4bGwzcjFjbnJzenMxamJna3R6eGhoYnhic3pxPT0AAAFoV0/uKQAABQEwggT9MA4GCisGAQQBKgIRAQEFAASCBOkSPJLZo26jmvgZA63wglEdkZ2jnWIAicad3OsLcfS3h94el0TIaQ7K8BwyX02AKYIGR82+sqxYCKvgY+YIf4xOyrVLS+yMeOurLSydsf6eDVJb7Z+TDh0s46WdN/4u1wihN2/nfdyFb+oxCfGNyKFhG7At/pNYfJQFQ3dvUZf9bVYU6VKBzMY8ON4g8gV/mQVZzsdMuvEARuoTeaW84Hw6Si/QuG/cb1p1dDaCelB7DweStzLqmQMKLqKjOYA/l34/D2YS2Hte1TeMoBmLHNeXAjahkp/WfYbfY0QNMlbc/75gkavH3g9D+SPsz3wx8fzAHRb9mHewM8YD7OSBjVQQV0I6y4hHGhQTc5YUprJ7/Z2m5Miv6ERGu9K3ENIftAY+TXi8J9alrbxaCZF3dQfXLf03kDyom4Zl1z8ptADhxU4V0DWRMFWAKxkrvX1C/qiZmyaAqV4eJfXFSku9bOz0yoaUf3TQBW1shPx9o9gTUXBKdbTzl3B6mUMgG77qGfKcHh/hzrUh87McMaiHEgGzfsmHuKiNIOzCvLgmBo36GtP8zaRNox7zGQ4UF56AbOgc69yvA6W5O7fQD0G4GpqQmkKQR/3oJqE6uTULdNcq9yNku3frxLilsQDiQNffr1abTPgRtE4w5S8vi1bFhBr5o3qzwgonqH8jEK0QKHjpaASXdplCybFM1JplyZCS9DLE1dEtpbdnU47Ekx+JMB5aSJoDhmCORCcLnmqLv4m1KWDPpuib/Z4MYkJDAx/jvbSma/Yu3rXxOrMqED9SZKWaJr7LKCAwoxSwaCewRd1Ji/BP/Ibg5Jr6KWYCGVOPRZgmbtY652uAwFd4ZKIl3v8lC/A89BmqjikgxTjtuVKzLUSdvAVSp5GlDE3G7Mzswtk3VcwPNUnlBIfafry1DvSB8OJH18XoTdbP4yKdXSgYx9GhdFboTLlZTv8WA89xl9vkQK/UMVZT0oQF2c8aQ0RE7LtOOVhJ9/LpB2M64RzgQryeI1haA787y6i3TkGRU+NPmc7bnd5sr6MnzcJbm6eo+JelW/Ap+zo5B+BCkUGTzu5W0/E0HKSyeXPJpX9VdMpwuEVadKDn3mI3lrW4zgyak7y8mh7iUx/GsDD0Nv4wKf8N+/OuTqlGo18aWIuN04cpeikcTbrapvqed5m0/ZxjNaa5GGKAjpqK3STo7AfGrZf/DRSvz8MUacBHnniJFqXFyXH4TmyoBSgPo5LUqzIBnwMnhbwHYwF0GKKyht9UF9kqeZGXfUXqltQQ3G4LSKL1gcwBwlcbX5UGk0tDMq6/vEMpVmg4knh+jz+g5GQ0kN0oIrf0HkbH4NsM235YrSylVe+ncr+S4GD7oMBXICO016CQnAMdyWD0rzGGlRE6XniFvmiI0QzUTCIAsgJ3Iaufy166w0oqhoOEaKq/bQSPI7m9ATHrALy6+ACaq1Sct41fQ8J5LwPoERKmtziyi+roZ6hvn8cnAUdZoS7YjHEQ5bPSKcdPPCwoMxP/aUpYjXnAfcvR22XneuI6E9/g88a5XImSXVx7Q1zD1mAxXvNTIUy9hNKaak8qMxnkhepCcFKrofBrqG6Qi7aWdq3Dspj6NwEPLgHmUlXkfHfHWczz2YoK6qgLmdHTpwyF6GU6BzxZTe5iDxtBZSsr6s1WRk8s6KMXBUBc4y4AAAABAAVYLjUwOQAAA2EwggNdMIICRaADAgECAgRje5iNMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNVBAYTAlVTMQkwBwYDVQQIEwAxCTAHBgNVBAcTADEJMAcGA1UEChMAMQkwBwYDVQQLEwAxJDAiBgNVBAMTG2NvbS5leHBvLnR1cnRsZS5jbGkuZXhhbXBsZTAeFw0xOTAxMTYxNTM4MDJaFw00NjA2MDMxNTM4MDJaMF8xCzAJBgNVBAYTAlVTMQkwBwYDVQQIEwAxCTAHBgNVBAcTADEJMAcGA1UEChMAMQkwBwYDVQQLEwAxJDAiBgNVBAMTG2NvbS5leHBvLnR1cnRsZS5jbGkuZXhhbXBsZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJjpLpdLvSMyE37A6E1BZ6j9wfqNC09CAVZAhsuiyocYhZH/lKCvx/giht4Pf0n5rXeOMNBjsrNwl+qxylOHwhcHAFgUiXgr15tQr+wsEA0ddDztFDzlND+HORurKXxgg82WYSwpGqLmahTJ2UvfcJrPsNp+hQgjit2zt4oyLDT+LoQrR7Mw6cxhxdCCVYNqrXVusOH0D2xubh39lfA6YCOAjXSzJ+C79nuAXwNmOmSqyYaHvMf4LCQ10qmAtXy+chucW4G5xzNkbkkIzye9hfev/YF4Xyt5HMH+WfgJLY9rfiEVa2A/t2/XhNBZYzb48GUPn4FORKzT37nrhs8xdIkCAwEAAaMhMB8wHQYDVR0OBBYEFKJv1T1PHa7r0KhPYvrXL7iCEjf9MA0GCSqGSIb3DQEBCwUAA4IBAQAUlJYtz2+GwpXDeyHTewwFs7hPgQO1oMJxtHitkc+rWPR8PwGall9WH+Wz5VXq5R1ZwsodgNRw5gLbZQpLb+/qYYyk8TSiTG1W4hKjHv6mWnSRQKSve/Zr5Z9tZt12Sc1xGYQ2QlMtvDNoxvcSguOoE0CfYzeK6hQteE873HSUIdqIRilK7EkfJexkFI/+4UWTZ4OP3MkKl0GKHTrHzcovzsmY/f1dkB0aByOyFpR17P6tIzd0Xsub3kgZr71PMb/5tvo2GePTmNVb46CUAhhBvt4BZBmr5ehh7oV+pHJrrVkPltMV8ddggNxpdWwE7F4Q5wKaGu0YPpaDZYv7jIeXdT074zft6plJHUWNi9CQgArR9Cs=&quot;,&quot;keystorePassword&quot;:&quot;28eeeda02db44ed9afc1c1d4a0f2caa6&quot;,&quot;keystoreAlias&quot;:&quot;QGV4cG8tdHVydGxlL3R1cnRsZS1jbGktZXhhbXBsZQ==&quot;,&quot;keyPassword&quot;:&quot;a6a53453c5534673900dac983e40a693&quot;},&quot;manifest&quot;:{&quot;ios&quot;:{&quot;buildNumber&quot;:&quot;13&quot;,&quot;supportsTablet&quot;:true,&quot;bundleIdentifier&quot;:&quot;com.expo.turtle.cli.example&quot;},&quot;icon&quot;:&quot;./assets/icon.png&quot;,&quot;namLine truncated

src/__smoke__/smoke.test.ts

@@ -17,8 +17,7 @@ jest.mock('../builders/utils/uploader.ts', () => ({
 async function performSmokeTest(recording: string) {
   jest.setTimeout(60 * 60 * 1000);
 
-  // @ts-ignore
-  const fixtureExists = await fs.exists(recording);
+  const fixtureExists = await fs.pathExists(recording);
   if (fixtureExists) {
     logger.info('Fixture exists, test will use existing one.');
 

src/aws/cloudwatch/index.ts

@@ -136,7 +136,7 @@ function _addEmptyMetrics(metrics: IMetric[]) {
       acc.add(i.MetricName);
     }
     return acc;
-  }, new Set());
+  }, new Set<string>());
 
   const nonEmptyMetrics = _.omitBy(state.registeredMetrics, ({ addEmpty }) => !addEmpty);
   const toAdd = _.difference(Object.keys(nonEmptyMetrics), Array.from(seenMetrics));

src/aws/cloudwatch/statistics.ts

@@ -19,10 +19,10 @@ export default function calculateStatistics(metrics: IMetric[]) {
       });
     }
 
-    result.StatisticValues.Maximum = Math.max(result.StatisticValues.Maximum, i.Value);
-    result.StatisticValues.Minimum = Math.min(result.StatisticValues.Minimum, i.Value);
-    result.StatisticValues.SampleCount += 1;
-    result.StatisticValues.Sum += i.Value;
+    result!.StatisticValues.Maximum = Math.max(result!.StatisticValues.Maximum, i.Value);
+    result!.StatisticValues.Minimum = Math.min(result!.StatisticValues.Minimum, i.Value);
+    result!.StatisticValues.SampleCount += 1;
+    result!.StatisticValues.Sum += i.Value;
     return result;
   }, null);
   return [reduced];

src/aws/sqs.ts

@@ -2,9 +2,8 @@ import retry from 'async-retry';
 import AWS from 'aws-sdk';
 import _ from 'lodash';
 
-import { OUTPUT_QUEUE_URL, QUEUE_URL } from 'turtle/aws/utils';
+import { QUEUE_URL } from 'turtle/aws/utils';
 import config from 'turtle/config';
-import { JOB_UPDATE_TYPE } from 'turtle/constants/build';
 import logger from 'turtle/logger';
 import { getCurrentJobId } from 'turtle/turtleContext';
 
@@ -73,19 +72,3 @@ export function changeMessageVisibilityRecurring(priority: string, receiptHandle
     }
   }, (VISIBILITY_TIMEOUT_SEC * 1000) / 3);
 }
-
-export async function sendMessage(jobId: string, status: JOB_UPDATE_TYPE, data = {}) {
-  const params = {
-    MessageBody: JSON.stringify({ jobId, status, ...data }),
-    QueueUrl: OUTPUT_QUEUE_URL(),
-    DelaySeconds: 0,
-    MessageGroupId: jobId,
-  };
-  try {
-    return await retry(async () => {
-      return await sqs.sendMessage(params).promise();
-    });
-  } catch (err) {
-    logger.error({ err }, `Error sending SQS message: ${JSON.stringify(params)}}`);
-  }
-}

src/aws/utils.ts

@@ -1,4 +1,3 @@
 import config from 'turtle/config';
 
 export const QUEUE_URL = (priority: string) => (config.sqs.queues as any)[priority][config.platform];
-export const OUTPUT_QUEUE_URL = () => config.sqs.queues.out;

src/bin/commands/build/android.ts

@@ -29,6 +29,11 @@ export default (program: any, setCommonCommandOptions: any) => {
       /^(debug|release)$/i,
       'release',
     )
+    .option(
+      '--gradle-args <gradle-args>',
+      'optional arguments passed to gradle, make sure to surround them with double quotes'
+      + ' (e.g.: --gradle-args "--stacktrace --debug")',
+    )
     .description(
       'Build a standalone APK or App Bundle for your project, either signed and ready for submission to'
       + ' the Google Play Store or in debug mode.',

src/bin/commands/build/index.ts

@@ -15,7 +15,12 @@ function setCommonCommandOptions(cmd: any) {
     .option('-o --output <output-file-path>', 'output file path')
     .option(
       '--public-url <url>',
-      'the URL of an externally hosted manifest (for self-hosted apps), only HTTPS URLs are supported!',
+      'the URL of an externally hosted manifest (for self-hosted apps), only HTTPS URLs are supported unless '
+      + '--allow-non-https-public-url is also set!',
+    )
+    .option(
+      '--allow-non-https-public-url',
+      'allows non-HTTPS URLs in --public-url; this should only be used when building an unpublished application',
     )
     .option(
       '--release-channel <channel-name>',

src/bin/setup/android/index.ts

@@ -101,7 +101,9 @@ async function _shellAppPostDownloadAction(sdkVersion: string, workingdir: strin
 async function _installNodeModules(cwd: string) {
   l.info(`installing dependencies in ${cwd} directory...`);
   const command = await _shouldUseYarnOrNpm();
-  await ExponentTools.spawnAsyncThrowError(command, ['install'], {
+  // Keep in sync with /Dockerfile
+  // --prod requires --ignore-scripts (otherwise expo-yarn-workspaces errors as missing)
+  await ExponentTools.spawnAsyncThrowError(command, ['install', '--ignore-scripts', '--production'], {
     pipeToLogger: true,
     loggerFields: LOGGER_FIELDS,
     cwd,

src/bin/utils/builder.ts

@@ -1,5 +1,6 @@
 import _url from 'url';
 
+import { ConfigError, getExpoSDKVersion, ProjectConfig } from '@expo/config';
 import { ExponentTools } from '@expo/xdl';
 import _ from 'lodash';
 import uuid from 'uuid';
@@ -56,28 +57,29 @@ export function createBuilderAction({
         );
       }
 
-      if (cmd.publicUrl) {
+      if (cmd.publicUrl && !cmd.allowNonHttpsPublicUrl) {
         const parsedPublicUrl = _url.parse(cmd.publicUrl);
         if (parsedPublicUrl.protocol !== 'https:') {
           throw new ErrorWithCommandHelp('--public-url is invalid - only HTTPS urls are supported');
         }
       }
 
+      const projectAbsoluteDir = ProjectUtils.resolveAbsoluteDir(projectDirArg);
       const args = {
         releaseChannel: cmd.releaseChannel || 'default',
         buildType: cmd.type,
         buildMode: cmd.mode,
         username: userData.username || 'anonymous',
-        projectDir: ProjectUtils.resolveAbsoluteDir(projectDirArg),
+        projectDir: projectAbsoluteDir,
         publicUrl: cmd.publicUrl,
       };
 
-      const appJSON = await ProjectUtils.loadAppJSON(projectDirArg, cmd.config);
-      const sdkVersion = _.get(appJSON, 'expo.sdkVersion');
+      const projectConfig = await ProjectUtils.getConfig(projectDirArg, cmd.config);
+      const sdkVersion = getExpoSDKVersionSafely(projectAbsoluteDir, projectConfig);
       await setup(platform, sdkVersion);
       const credentials = await prepareCredentials(cmd);
       const rawJob = {
-        ...await buildJobObject(platform, appJSON, args, credentials),
+        ...await buildJobObject(cmd, platform, projectConfig, args, credentials, sdkVersion),
         ...cmd.buildDir && { fakeUploadDir: ProjectUtils.resolveAbsoluteDir(cmd.buildDir) },
         ...cmd.output && { fakeUploadBuildPath: ProjectUtils.resolveAbsoluteDir(cmd.output) },
       };
@@ -95,34 +97,53 @@ export function createBuilderAction({
   };
 }
 
+function getExpoSDKVersionSafely(projectDir: string, projectConfig: ProjectConfig) {
+  try {
+    return getExpoSDKVersion(projectDir, projectConfig.exp);
+  } catch (err) {
+    if (err instanceof ConfigError) {
+      throw new Error(
+        'Couldn\'t find the `expo` library in your project\'s dependencies.'
+        + ' Have you run \'yarn install\' or \'npm install\' for your project?',
+      );
+    } else {
+      throw err;
+    }
+  }
+}
+
 const buildJobObject = async (
+  cmd: any,
   platform: 'android' | 'ios',
-  appJSON: any,
+  projectConfig: ProjectConfig,
   { releaseChannel, buildType, buildMode, username, publicUrl, projectDir }: any,
   credentials: any,
+  sdkVersion: string,
 ) => {
-  const experienceName = `@${_.get(appJSON, 'expo.owner', username)}/${_.get(appJSON, 'expo.slug')}`;
+  const owner = projectConfig.exp.owner || username;
+  const experienceName = `@${owner}/${projectConfig.exp.slug}`;
   const job = {
     config: {
-      ..._.get(appJSON, `expo.${platform}.config`, {}),
+      ...(projectConfig.exp?.[platform]?.config || {}),
       buildType,
-      ...(platform === 'android' ? { buildMode } : {}),
       releaseChannel,
-      ...(platform === 'ios' ? { bundleIdentifier: _.get(appJSON, 'expo.ios.bundleIdentifier') } : {}),
-      ...(platform === 'android' ? { androidPackage: _.get(appJSON, 'expo.android.package') } : {}),
       publicUrl,
+      ...(platform === 'android' && cmd.gradleArgs ? { gradleArgs: cmd.gradleArgs.split(' ') } : {}),
+      ...(platform === 'android' ? { buildMode } : {}),
+      ...(platform === 'android' ? { androidPackage: projectConfig.exp?.android?.package } : {}),
+      ...(platform === 'ios' ? { bundleIdentifier: projectConfig.exp?.ios?.bundleIdentifier } : {}),
     },
     id: uuid.v4(),
     platform,
     projectDir,
-    sdkVersion: _.get(appJSON, 'expo.sdkVersion'),
+    sdkVersion,
     experienceName,
     ...(credentials && { credentials }),
   };
   const url = getExperienceUrl(job.experienceName, job.config.publicUrl);
 
   const manifest = await ExponentTools.getManifestAsync(url, {
-    'Exponent-SDK-Version': _.get(appJSON, 'expo.sdkVersion'),
+    'Exponent-SDK-Version': sdkVersion,
     'Exponent-Platform': platform,
     'Expo-Release-Channel': releaseChannel,
     'Accept': 'application/expo+json,application/json',

src/bin/utils/project.ts

@@ -1,20 +1,18 @@
-import fs from 'fs-extra';
 import path from 'path';
 
-export async function loadAppJSON(projectDirArg: string, config: string | null) {
-  let appJSONPath;
+import { getConfig as getProjectConfig, setCustomConfigPath } from '@expo/config';
+import fs from 'fs-extra';
+
+export async function getConfig(projectDir: string, config: string | null) {
+  projectDir = resolveAbsoluteDir(projectDir);
   if (config) {
-    appJSONPath = resolveAbsoluteDir(config);
-  } else {
-    const projectDir = resolveAbsoluteDir(projectDirArg);
-    appJSONPath = path.join(projectDir, 'app.json');
-  }
-  const appJSONExists = await fs.pathExists(appJSONPath);
-  if (!appJSONExists) {
-    throw new Error(`Couldn't find app.json.`);
-  } else {
-    return require(appJSONPath);
+    const pathToConfig = path.resolve(process.cwd(), config);
+    if (!await fs.pathExists(pathToConfig)) {
+      throw new Error(`File at provided config path does not exist: ${pathToConfig}`);
+    }
+    setCustomConfigPath(projectDir, pathToConfig);
   }
+  return await getProjectConfig(projectDir, {});
 }
 
 export const resolveAbsoluteDir = (dir: string) => {

src/builders/android.ts

@@ -3,7 +3,6 @@ import path from 'path';
 import { AndroidShellApp, ImageUtils } from '@expo/xdl';
 import fs from 'fs-extra';
 import _ from 'lodash';
-import semver from 'semver';
 import { ANDROID_BUILD_TYPES } from 'turtle/constants';
 import uuidv4 from 'uuid/v4';
 
@@ -11,7 +10,7 @@ import getOrCreateCredentials from 'turtle/builders/utils/android/credentials';
 import { formatShellAppDirectory } from 'turtle/builders/utils/android/workingdir';
 import * as commonUtils from 'turtle/builders/utils/common';
 import * as imageHelpers from 'turtle/builders/utils/image';
-import { resolveExplicitOptIn } from 'turtle/builders/utils/unimodules';
+import { resolveExplicitOptIn, resolveNativeModules } from 'turtle/builders/utils/unimodules';
 import { uploadBuildToS3 } from 'turtle/builders/utils/uploader';
 import { ensureCanBuildSdkVersion } from 'turtle/builders/utils/version';
 import config from 'turtle/config';
@@ -35,8 +34,8 @@ export default async function buildAndroid(jobData: IJob): Promise<IJobResult> {
     ...config.builder.fakeUpload && {
       fakeUploadBuildPath:
         jobData.fakeUploadBuildPath
-        ? jobData.fakeUploadBuildPath
-        : path.join(jobData.fakeUploadDir || config.directories.fakeUploadDir, fakeUploadFilename),
+          ? jobData.fakeUploadBuildPath
+          : path.join(jobData.fakeUploadDir || config.directories.fakeUploadDir, fakeUploadFilename),
     },
   });
 
@@ -73,15 +72,20 @@ async function runShellAppBuilder(
   const sdkVersion = _.get(manifest, 'sdkVersion', sdkVersionFromJob);
   const workingDir = formatShellAppDirectory({ sdkVersion });
 
-  // (2019-07-31) We are explicitly choosing to disable this option until we have more
-  // infrastructure/tooling built around optional modules and OTA updates, as right now it's
-  // very easy for developers to break apps in production with optional modules.
+  // Default behaviour:
+  // - load all modules except [expo-branch]
+  // - load expo-branch only if present in package.json dependencies
   //
-  // to enable full resolver switch resolveExplicitOptIn to resolveNativeModules
+  // Behaviour with "enableDangerousExperimentalLeanBuilds" flag enabled
+  // - add unimodules from `manifest.dependencies` (project package.json dependecies)
+  // - add unimodules from react-native-unimodules package.json
+  // - add unimodules from expo package package.json
+  // - add recusively all dependecies of already added unimodules
   logger.info({ buildPhase: 'resolve native modules' }, 'Resolving universal modules dependencies');
-  const enabledModules = semver.satisfies(sdkVersion, '>= 33.0.0')
-    ? await resolveExplicitOptIn(workingDir, _.get(manifest, 'dependencies'))
-    : null;
+  const packageJsonDependecies = _.get(manifest, 'dependencies');
+  const enabledModules = _.get(manifest, 'android.enableDangerousExperimentalLeanBuilds')
+    ? await resolveNativeModules(workingDir, packageJsonDependecies)
+    : await resolveExplicitOptIn(workingDir, packageJsonDependecies);
 
   try {
     await AndroidShellApp.createAndroidShellAppAsync({
@@ -99,6 +103,7 @@ async function runShellAppBuilder(
       modules: enabledModules,
       buildType: jobData.config.buildType,
       buildMode: jobData.config.buildMode,
+      gradleArgs: jobData.config.gradleArgs,
     });
   } catch (err) {
     commonUtils.logErrorOnce(err);

src/builders/utils/android/credentials.ts

@@ -5,11 +5,10 @@ import fs from 'fs-extra';
 import get from 'lodash/get';
 import uuidv4 from 'uuid/v4';
 
-import * as sqs from 'turtle/aws/sqs';
-import { UPDATE_CREDENTIALS } from 'turtle/constants/build';
 import { IAndroidCredentials, IJob } from 'turtle/job';
 import logger from 'turtle/logger';
 import { isOffline } from 'turtle/turtleContext';
+import { sendCredentialsUpdate } from 'turtle/utils/www';
 
 async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredentials> {
   if (!jobData.credentials) {
@@ -50,9 +49,7 @@ async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredential
       l.info(`Key password: ${credentials.keyPassword}`);
       l.info(`Please keep these credentials safe`);
     } else {
-      await sqs.sendMessage(jobData.id, UPDATE_CREDENTIALS, {
-        ...credentials,
-      });
+      await sendCredentialsUpdate(jobData.id, 'android', credentials);
       l.info('Keystore sent to storage successfully');
     }
     await fs.unlink(keystoreFilename);

src/builders/utils/ios/adhocBuild.ts

@@ -4,12 +4,11 @@ import spawnAsync from '@expo/spawn-async';
 import { IosCodeSigning } from '@expo/xdl';
 import fs from 'fs-extra';
 
-import * as sqs from 'turtle/aws/sqs';
 import BuildError, { BuildErrorReason } from 'turtle/builders/BuildError';
-import { UPDATE_CREDENTIALS } from 'turtle/constants/build';
 import { IJob } from 'turtle/job';
 import logger from 'turtle/logger';
 import { isOffline } from 'turtle/turtleContext';
+import { sendCredentialsUpdate } from 'turtle/utils/www';
 
 // tslint:disable-next-line:no-var-requires
 const travelingFastlane = process.platform === 'darwin' ? require('@expo/traveling-fastlane-darwin')() : null;
@@ -64,7 +63,7 @@ async function prepareAdHocBuildCredentials(job: IJob) {
       await fs.writeFile(provisioningProfilePath, Buffer.from(credentials.provisioningProfile, 'base64'));
       logger.info(`Saved provisioning profile to ${provisioningProfilePath}`);
     } else {
-      await sqs.sendMessage(job.id, UPDATE_CREDENTIALS, {
+      await sendCredentialsUpdate(job.id, 'ios', {
         provisioningProfileId: credentials.provisioningProfileId,
         provisioningProfile: credentials.provisioningProfile,
       });

src/config.ts

@@ -1,3 +1,4 @@
+import fs from 'fs';
 import os from 'os';
 import path from 'path';
 
@@ -28,7 +29,6 @@ export default {
         ios: env('AWS_SQS_IOS_PRIORITY_QUEUE_URL'),
         android: env('AWS_SQS_ANDROID_PRIORITY_QUEUE_URL'),
       },
-      out: env('AWS_SQS_OUT_QUEUE_URL'),
     },
   },
   cloudwatch: {
@@ -44,8 +44,17 @@ export default {
     intervalMs: env('DATADOG_INTERVAL_MS', 5 * 60 * 1000),
   },
   redis: {
-    url: env('REDIS_URL'),
-    configUrl: env('REDIS_CONFIG_URL'),
+    main: {
+      url: env('REDIS_URL'),
+      caCertificate: envTransform(
+        'REDIS_CA_CERTIFICATE',
+        null,
+        (caCertificate: string) => caCertificate && fs.readFileSync(caCertificate),
+      ),
+    },
+    turtle: {
+      url: env('REDIS_CONFIG_URL'),
+    },
   },
   logger: {
     level: env('LOGGER_LEVEL', 'info'),
@@ -81,7 +90,7 @@ export default {
     androidDependenciesDir: env(
       'TURTLE_ANDROID_DEPENDENCIES_DIR',
       path.join(os.homedir(), '.turtle/androidDependencies'),
-      ),
+    ),
     tempS3LogsDir: env('TURTLE_TEMP_S3_LOGS_DIR', '/tmp/logs'),
     fakeUploadDir: envOptional('TURTLE_FAKE_UPLOAD_DIR'),
     temporaryFilesRoot: env(
@@ -90,6 +99,10 @@ export default {
     ),
   },
   www: {
-    sdkVersionsSecretToken: env('TURTLE_SDK_VERSIONS_SECRET_TOKEN'),
+    secretToken: env('TURTLE_SDK_VERSIONS_SECRET_TOKEN'),
+  },
+  leader: {
+    redisKey: env('LEADER_REDIS_KEY', 'turtle-v1:leader'),
+    redisKeyTTLSec: envNum('LEADER_REDIS_KEY_TTL', 30),
   },
 };

src/job.ts

@@ -15,6 +15,7 @@ export interface IJob {
     // android
     androidPackage?: string;
     buildMode?: ANDROID_BUILD_MODES;
+    gradleArgs?: string[];
   };
   credentials: {
     // android

src/jobManager.ts

@@ -10,8 +10,9 @@ import logger from 'turtle/logger';
 import * as buildDurationMetric from 'turtle/metrics/buildDuration';
 import * as buildStatusMetric from 'turtle/metrics/buildStatus';
 import { checkShouldExit, setCurrentJobId, turtleVersion } from 'turtle/turtleContext';
-import { getPriorities } from 'turtle/utils/priorities';
+import { getPriorities, labelConfiguration, TurtleMode } from 'turtle/utils/priorities';
 import * as redis from 'turtle/utils/redis';
+import { sendBuildStatusUpdate } from 'turtle/utils/www';
 import { sanitizeJob } from 'turtle/validator';
 
 function _maybeExit() {
@@ -22,8 +23,8 @@ function _maybeExit() {
 }
 
 export async function doJob() {
-  const jobData = await getJob();
-  await processJob(jobData);
+  const { job, turtleMode } = await getJob();
+  await processJob(job, turtleMode);
   _maybeExit();
 }
 
@@ -34,6 +35,7 @@ export async function getJob() {
     try {
       let job = null;
       const priorities = await getPriorities();
+      const turtleMode = labelConfiguration(priorities);
       for (const priority of priorities) {
         job = await sqs.receiveMessage(priority);
         _maybeExit();
@@ -44,7 +46,7 @@ export async function getJob() {
       }
       _maybeExit();
       if (job !== null) {
-        return job;
+        return { job, turtleMode };
       }
     } catch (err) {
       logger.error({ err }, 'Error at receiving messages');
@@ -52,7 +54,7 @@ export async function getJob() {
   }
 }
 
-async function processJob(jobData: any) {
+async function processJob(jobData: any, turtleMode: TurtleMode) {
   const receiptHandle = jobData.ReceiptHandle;
   const { priority } = jobData;
 
@@ -72,7 +74,8 @@ async function processJob(jobData: any) {
       logger.error({ err, ...rawJob && { buildJobId: rawJob.id } }, 'The build job is invalid');
       // send message only if we've managed to parse the job json
       if (rawJob) {
-        await sqs.sendMessage(rawJob.id, BUILD.JOB_STATES.ERRORED, {
+        await sendBuildStatusUpdate(rawJob.id, {
+          status: BUILD.JOB_STATES.ERRORED,
           turtleVersion,
           buildDuration: 0,
           reason: 'The build job is invalid',
@@ -92,7 +95,7 @@ async function processJob(jobData: any) {
       let buildFailed = false;
       const buildType = _.get(job, 'config.buildType', 'default');
       try {
-        const status = await build(job);
+        const status = await build(job, turtleMode);
         buildStatusMetric.add(buildType, priority, status);
         buildFailed = !status;
       } catch (err) {
@@ -127,7 +130,7 @@ async function processJob(jobData: any) {
 function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any) {
   return setTimeout(async () => {
     try {
-      sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, { turtleVersion });
+      sendBuildStatusUpdate(job.id, { status: BUILD.JOB_STATES.ERRORED, turtleVersion });
       await deleteMessage(priority, receiptHandle);
     } finally {
       logger.error('Build timed out. Going to terminate turtle agent.');
@@ -137,20 +140,33 @@ function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any)
   }, config.builder.maxJobTimeMs);
 }
 
-async function build(job: any) {
+async function build(job: any, turtleMode: TurtleMode) {
   const startTimestamp = Date.now();
   const s3Url = await logger.initForJob(job);
   const calculateBuildDuration = () => Math.ceil((Date.now() - startTimestamp) / 1000);
 
+  const commonPayload = {
+    logs: {
+      url: s3Url,
+      format: 'json',
+    },
+    extraData: {
+      turtleMode,
+    },
+  };
   try {
-    await sqs.sendMessage(job.id, BUILD.JOB_STATES.IN_PROGRESS, {
-      logUrl: s3Url,
-      logFormat: 'json',
+    await sendBuildStatusUpdate(job.id, {
+      ...commonPayload,
+      status: BUILD.JOB_STATES.IN_PROGRESS,
     });
-    const result = await (builders as any)[job.platform](job);
+    const { artifactUrl } = await (builders as any)[job.platform](job);
     const buildDuration = calculateBuildDuration();
-    sqs.sendMessage(job.id, BUILD.JOB_STATES.FINISHED, {
-      ...result,
+    sendBuildStatusUpdate(job.id, {
+      ...commonPayload,
+      artifacts: {
+        url: artifactUrl,
+      },
+      status: BUILD.JOB_STATES.FINISHED,
       turtleVersion,
       buildDuration,
     });
@@ -162,7 +178,9 @@ async function build(job: any) {
     if (err instanceof BuildError) {
       reason = err.reason;
     }
-    sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, {
+    sendBuildStatusUpdate(job.id, {
+      ...commonPayload,
+      status: BUILD.JOB_STATES.ERRORED,
       turtleVersion,
       buildDuration,
       ...reason && { reason },

src/jobsSchemas/android.ts

@@ -15,6 +15,7 @@ export default baseJobSchema.concat(
     config: Joi.object().keys({
       buildMode: Joi.string().valid(Object.values(ANDROID_BUILD_MODES)).default(ANDROID_BUILD_MODES.RELEASE),
       buildType: Joi.string().valid(Object.values(ANDROID_BUILD_TYPES)).default(ANDROID_BUILD_TYPES.APK),
+      gradleArgs: Joi.array().items(Joi.string()),
     }),
   }),
 );

src/metrics/buildersConfig.ts

@@ -5,11 +5,11 @@ import take from 'lodash/take';
 import logger from 'turtle/logger';
 import { sendMetric } from 'turtle/metrics/datadog';
 import { getReplicaCount } from 'turtle/utils/metadata';
-import { getLabeledConfiguration, TurtleModeLabels } from 'turtle/utils/priorities';
+import { getLabeledConfiguration, TurtleMode } from 'turtle/utils/priorities';
 
 const createMetricName = () => `turtle.replicaCount`;
-const createModeMetricName = (mode: TurtleModeLabels) => `${createMetricName()}.${mode}`;
-const createModeTag = (mode: TurtleModeLabels) => `mode:${mode}`;
+const createModeMetricName = (mode: TurtleMode) => `${createMetricName()}.${mode}`;
+const createModeTag = (mode: TurtleMode) => `mode:${mode}`;
 
 export default async function sendConfigToDatadog() {
   let configuration;
@@ -30,28 +30,28 @@ export default async function sendConfigToDatadog() {
 
   const metricName = createMetricName();
 
-  const normalReplicaCount = replicaCounts[TurtleModeLabels.Normal] || 0;
-  const highReplicaCount = replicaCounts[TurtleModeLabels.High] || 0;
-  const highOnlyReplicaCount = replicaCounts[TurtleModeLabels.HighOnly] || 0;
+  const normalReplicaCount = replicaCounts[TurtleMode.Normal] || 0;
+  const highReplicaCount = replicaCounts[TurtleMode.High] || 0;
+  const highOnlyReplicaCount = replicaCounts[TurtleMode.HighOnly] || 0;
 
-  const normalTag = createModeTag(TurtleModeLabels.Normal);
-  const normalMetricName = createModeMetricName(TurtleModeLabels.Normal);
+  const normalTag = createModeTag(TurtleMode.Normal);
+  const normalMetricName = createModeMetricName(TurtleMode.Normal);
   logger.debug(
     `Sending ${metricName}=${normalReplicaCount} (${normalTag}), ${normalMetricName}=${normalReplicaCount} to Datadog`,
   );
   await sendMetric(metricName, normalReplicaCount, [normalTag]);
   await sendMetric(normalMetricName, normalReplicaCount);
 
-  const highTag = createModeTag(TurtleModeLabels.High);
-  const highMetricName = createModeMetricName(TurtleModeLabels.High);
+  const highTag = createModeTag(TurtleMode.High);
+  const highMetricName = createModeMetricName(TurtleMode.High);
   logger.debug(
     `Sending ${metricName}=${highReplicaCount} (${highTag}), ${highMetricName}=${highReplicaCount} to Datadog`,
   );
   await sendMetric(metricName, highReplicaCount, [highTag]);
   await sendMetric(highMetricName, highReplicaCount);
 
-  const highOnlyTag = createModeTag(TurtleModeLabels.HighOnly);
-  const highOnlyMetricName = createModeMetricName(TurtleModeLabels.HighOnly);
+  const highOnlyTag = createModeTag(TurtleMode.HighOnly);
+  const highOnlyMetricName = createModeMetricName(TurtleMode.HighOnly);
   logger.debug(
     `Sending ${metricName}=${highOnlyReplicaCount} (${highOnlyTag}),`
     + ` ${highOnlyMetricName}=${highOnlyReplicaCount} to Datadog`,

src/server.ts

@@ -6,6 +6,8 @@ import { doJob } from 'turtle/jobManager';
 import logger from 'turtle/logger';
 import setup from 'turtle/setup';
 import { checkShouldExit, setShouldExit, turtleVersion } from 'turtle/turtleContext';
+import { synchronizeFailedUpdates } from 'turtle/updatesManager';
+import Leader, { LeaderEvent } from 'turtle/utils/leader';
 import { setSupportedSdkVersions, setTurtleVersion } from 'turtle/utils/versions';
 
 process.on('unhandledRejection', (err) => logger.error({ err }, 'Unhandled promise rejection'));
@@ -33,6 +35,8 @@ async function main() {
     await setup[config.platform]();
   }
 
+  await electLeaderAndSynchronize();
+
   try {
     await setTurtleVersion(turtleVersion);
     logger.info(`Registered Turtle version (${turtleVersion}) in www`);
@@ -52,6 +56,28 @@ async function main() {
   }
 }
 
+async function electLeaderAndSynchronize() {
+  if (config.env === 'test') {
+    return;
+
+  }
+  const l = new Leader({
+    key: config.leader.redisKey,
+    id: config.hostname,
+    ttl: config.leader.redisKeyTTLSec,
+  });
+  let intervalId: NodeJS.Timeout;
+  l.on(LeaderEvent.elected, async () => {
+    intervalId = await synchronizeFailedUpdates();
+  });
+  l.on(LeaderEvent.revoked, () => {
+    if (intervalId) {
+      clearInterval(intervalId);
+    }
+  });
+  await l.elect();
+}
+
 main()
   .then(() => logger.error('This should never happen...'))
   .catch((err) => logger.error({ err }, 'Something went terribly wrong'))

src/types/declarations.d.ts

@@ -4,7 +4,6 @@ declare module 'bunyan-sentry-stream';
 declare module 'bunyan-debug-stream';
 declare module 'copy';
 declare module 'dogapi';
-declare module 'ioredis';
 declare module 'is-semver';
 declare module 'request-progress';
 declare module '@expo/xdl';

src/updatesManager.ts

@@ -0,0 +1,83 @@
+import Redis from 'ioredis';
+import chunk from 'lodash/chunk';
+
+import config from 'turtle/config';
+import { getRedisClient, RedisClient } from 'turtle/utils/redis';
+import { sendBuildStatusUpdate, sendCredentialsUpdate } from 'turtle/utils/www';
+
+const RETRY_COUNT = 5;
+
+type UpdateType = 'build' | 'credentials';
+interface IUpdate {
+  type: UpdateType;
+  buildId: string;
+  updateBody: any;
+  retried: number;
+}
+
+export async function saveUpdateFailure(buildId: string, type: UpdateType, updateBody: object): Promise<void> {
+  if (config.env === 'test') {
+    return;
+  }
+  const redis = await getRedisClient(RedisClient.Configuration) as Redis.Redis;
+  const update: IUpdate = {
+    type,
+    buildId,
+    updateBody,
+    retried: 0,
+  };
+  await redis.set(getFailedUpdateKey(buildId), JSON.stringify(update));
+}
+
+export async function synchronizeFailedUpdates(intervalMs = 3 * 60 * 1000): Promise<NodeJS.Timeout> {
+  try {
+    await _synchronizeFailedUpdates();
+  } catch (err) {
+    // don't do anything
+  }
+  return setInterval(() => _synchronizeFailedUpdates(), intervalMs);
+}
+
+async function _synchronizeFailedUpdates(): Promise<void> {
+  const redis = await getRedisClient(RedisClient.Configuration) as Redis.Redis;
+  const keys = await redis.keys(getFailedUpdateKey('*'));
+  const keyChunks = chunk(keys, 20);
+  for (const keyChunk of keyChunks) {
+    await Promise.all(keyChunk.map((key: string) => resendUpdateForKey(key)));
+  }
+}
+
+async function resendUpdateForKey(key: string): Promise<void> {
+  try {
+    const redis = await getRedisClient(RedisClient.Configuration) as Redis.Redis;
+    const updateRaw = await redis.get(key);
+    if (updateRaw) {
+      const update = JSON.parse(updateRaw) as IUpdate;
+      try {
+        await resendUpdateToWWW(update);
+        await redis.del(key);
+      } catch (err) {
+        update.retried += 1;
+        if (update.retried >= RETRY_COUNT) {
+          await redis.del(key);
+        } else {
+          await redis.set(key, JSON.stringify(update));
+        }
+      }
+    }
+  } catch (err) {
+    // don't do anything
+  }
+}
+
+async function resendUpdateToWWW(update: IUpdate): Promise<void> {
+  const { type, buildId, updateBody } = update;
+  if (type === 'build') {
+    await sendBuildStatusUpdate(buildId, updateBody);
+  } else if (type === 'credentials') {
+    const { platform, ...rest } = updateBody;
+    await sendCredentialsUpdate(buildId, platform, rest);
+  }
+}
+
+const getFailedUpdateKey = (buildId: string) => `${config.deploymentEnv}:failed-update:${buildId}`;

src/utils/leader.ts

@@ -0,0 +1,164 @@
+import { EventEmitter } from 'events';
+import Redis from 'ioredis';
+import { v4 as uuid } from 'uuid';
+
+import { getRedisClient, RedisClient } from 'turtle/utils/redis';
+
+interface ILeaderParams {
+  key: string;
+  id?: string;
+  ttl?: number;
+  wait?: number;
+}
+
+const RENEW_LEADER_SCRIPT = `
+  local leader_key = KEYS[1]
+  local leader_id = KEYS[2]
+  local leader_ttl = KEYS[3]
+
+  local current_leader
+
+  current_leader = redis.call("get", leader_key)
+  if (current_leader == leader_id) then
+    redis.call("pexpire", leader_key, leader_ttl)
+    return "ok"
+  else
+    return "you_are_not_the_leader"
+  end
+`;
+
+const STOP_BEING_LEADER_SCRIPT = `
+  local leader_key = KEYS[1]
+  local leader_id = KEYS[2]
+
+  local current_leader
+
+  current_leader = redis.call("get", leader_key)
+  if (current_leader == leader_id) then
+    redis.call("del", leader_key)
+    return "ok"
+  else
+    return "you_are_not_the_leader"
+  end
+`;
+
+interface IRedisCustom extends Redis.Redis {
+  renewLeader(key: string, id: string, ttl: number): Promise<string>;
+  stopBeingLeader(key: string, id: string): Promise<string>;
+}
+
+enum LeaderEvent {
+  elected = 'elected',
+  revoked = 'revoked',
+  error = 'error',
+}
+
+class Leader extends EventEmitter {
+  private key: string;
+  private id: string;
+  private ttl: number;
+  private wait: number;
+  private waitId?: NodeJS.Timeout;
+  private redisClientValue?: IRedisCustom;
+  private renewId?: NodeJS.Timeout;
+  private started = false;
+
+  constructor({ key, id, ttl = 30, wait = 1000 }: ILeaderParams) {
+    super();
+
+    this.key = key;
+    this.id = id || uuid();
+    this.ttl = ttl * 1000;
+    this.wait = wait;
+  }
+  public async elect(): Promise<void> {
+    if (!this.redisClientValue) {
+      await this.init();
+    }
+
+    if (!this.started) {
+      this.started = true;
+    }
+
+    try {
+      const res = await this.redisClient.set(this.key, this.id, 'PX', this.ttl, 'NX');
+      if (res === null) {
+        this.waitId = setTimeout(() => {
+          this.elect();
+        }, this.wait);
+      } else {
+        this.emit(LeaderEvent.elected);
+        this.renewId = setInterval(() => {
+          this.renew();
+        }, this.ttl / 2);
+      }
+    } catch (err) {
+      this.emit(LeaderEvent.error, err);
+    }
+  }
+
+  public async stop(): Promise<void> {
+    if (!this.started) {
+      throw new Error('You should start the election first!');
+    }
+    try {
+      const res = await this.redisClient.stopBeingLeader(this.key, this.id);
+      if (res === 'ok') {
+        this.emit(LeaderEvent.revoked);
+      }
+      if (this.renewId) {
+        clearInterval(this.renewId);
+      }
+      if (this.waitId) {
+        clearTimeout(this.waitId);
+      }
+      this.redisClientValue?.disconnect();
+      this.redisClientValue = undefined;
+    } catch (err) {
+      this.emit(LeaderEvent.error, err);
+    }
+  }
+
+  private get redisClient(): IRedisCustom {
+    if (!this.redisClientValue) {
+      throw new Error('Please run the init method first!');
+    }
+    return this.redisClientValue as IRedisCustom;
+  }
+
+  private async init(): Promise<void> {
+    if (this.redisClientValue) {
+      return;
+    }
+    this.redisClientValue = await getRedisClient(RedisClient.Configuration);
+    const redisClient = this.redisClient as IRedisCustom;
+    redisClient.defineCommand('renewLeader', {
+      numberOfKeys: 3,
+      lua: RENEW_LEADER_SCRIPT,
+    });
+    redisClient.defineCommand('stopBeingLeader', {
+      numberOfKeys: 2,
+      lua: STOP_BEING_LEADER_SCRIPT,
+    });
+  }
+
+  private async renew(): Promise<void> {
+    try {
+      const result = await this.redisClient.renewLeader(this.key, this.id, this.ttl);
+      if (result === 'you_are_not_the_leader') {
+        if (this.renewId) {
+          clearInterval(this.renewId);
+        }
+        this.waitId = setTimeout(() => {
+          this.elect();
+        }, this.wait);
+        this.emit(LeaderEvent.revoked);
+      }
+    } catch (err) {
+      this.emit(LeaderEvent.error, err);
+    }
+  }
+}
+
+export default Leader;
+export { LeaderEvent };

src/utils/priorities.ts

@@ -9,7 +9,7 @@ import { getRedisClient, RedisClient } from 'turtle/utils/redis';
 export const NORMAL = 'normalPriority';
 export const HIGH = 'highPriority';
 
-export enum TurtleModeLabels {
+export enum TurtleMode {
   Normal = 'normal',
   High = 'high',
   HighOnly = 'highOnly',
@@ -100,13 +100,15 @@ async function getConfiguration(currentRedisClient: any) {
 
 export async function getLabeledConfiguration(currentRedisClient?: any) {
   const configuration = await getConfiguration(currentRedisClient);
-  return configuration.map((i: string[]) => {
-    if (isEqual(i, NORMAL_CONFIGURATION)) {
-      return TurtleModeLabels.Normal;
-    } else if (isEqual(i, HIGH_CONFIGURATION)) {
-      return TurtleModeLabels.High;
-    } else {
-      return TurtleModeLabels.HighOnly;
-    }
-  });
+  return configuration.map((i: string[]) => labelConfiguration(i));
+}
+
+export function labelConfiguration(configuration: string[]): TurtleMode {
+  if (isEqual(configuration, NORMAL_CONFIGURATION)) {
+    return TurtleMode.Normal;
+  } else if (isEqual(configuration, HIGH_CONFIGURATION)) {
+    return TurtleMode.High;
+  } else {
+    return TurtleMode.HighOnly;
+  }
 }

src/utils/redis.ts

@@ -12,18 +12,27 @@ export enum RedisClient {
   Configuration = 'REDIS_CLIENT_CONFIGURATION',
 }
 
-interface IClientConfig { [key: string]: string; }
+interface IClientConfig {
+  [key: string]: {
+    url: string;
+    caCertificate?: string;
+  };
+}
 
-const clientToUrl: IClientConfig = {
-  [RedisClient.Subscriber]: config.redis.url,
-  [RedisClient.Default]: config.redis.url,
-  [RedisClient.Configuration]: config.redis.configUrl,
+const clientConfigs: IClientConfig = {
+  [RedisClient.Subscriber]: config.redis.main,
+  [RedisClient.Default]: config.redis.main,
+  [RedisClient.Configuration]: config.redis.turtle,
 };
 
 export function connect(timeoutMs: number, type: string): Promise<any> {
   return new Promise((resolve, reject) => {
-    const redisClient = new Redis(clientToUrl[type], {
+    const { url, caCertificate } = clientConfigs[type];
+    const redisClient = new Redis(url, {
       maxRetriesPerRequest: 2,
+      ...caCertificate && {
+        tls: { ca: caCertificate },
+      },
     });
     const timer = setTimeout(() => reject(new Error('Timeout at connecting to Redis')), timeoutMs);
     redisClient.on('ready', () => {

src/utils/versions.ts

@@ -10,7 +10,7 @@ async function setSupportedSdkVersions(platform: string, sdkVersions: string[])
     method: 'POST',
     uri: href,
     headers: {
-      'secret-token': config.www.sdkVersionsSecretToken,
+      'secret-token': config.www.secretToken,
     },
     body: {
       platform,
@@ -26,7 +26,7 @@ async function setTurtleVersion(version: string) {
     method: 'POST',
     uri: href,
     headers: {
-      'secret-token': config.www.sdkVersionsSecretToken,
+      'secret-token': config.www.secretToken,
     },
     body: {
       version,
@@ -38,9 +38,9 @@ async function setTurtleVersion(version: string) {
 function resolveWWWEndpoint() {
   const currentEnv = config.deploymentEnv;
   if (currentEnv === 'production') {
-    return 'https://expo.io';
+    return 'https://exp.host';
   } else if (currentEnv === 'staging') {
-    return 'https://staging.expo.io';
+    return 'https://staging.exp.host';
   } else {
     return 'http://127.0.0.1:3000';
   }

src/utils/www.ts

@@ -0,0 +1,56 @@
+import { URL } from 'url';
+
+import request from 'request-promise';
+
+import config from 'turtle/config';
+import { saveUpdateFailure } from 'turtle/updatesManager';
+
+async function sendBuildStatusUpdate(buildId: string, updateParams: object) {
+  const { href } = new URL(`/--/api/v2/turtle-legacy-builds/${buildId}`, resolveWWWEndpoint());
+  try {
+    await request({
+      method: 'PUT',
+      uri: href,
+      headers: {
+        'secret-token': config.www.secretToken,
+      },
+      body: updateParams,
+      json: true,
+    });
+  } catch (err) {
+    await saveUpdateFailure(buildId, 'build', updateParams);
+  }
+}
+
+async function sendCredentialsUpdate(buildId: string, platform: 'android' | 'ios', updateParams: object) {
+  const { href } = new URL(`/--/api/v2/turtle-legacy-builds/${buildId}/credentials`, resolveWWWEndpoint());
+  try {
+    await request({
+      method: 'PUT',
+      uri: href,
+      headers: {
+        'secret-token': config.www.secretToken,
+      },
+      body: {
+        platform,
+        ...updateParams,
+      },
+      json: true,
+    });
+  } catch (err) {
+    await saveUpdateFailure(buildId, 'credentials', updateParams);
+  }
+}
+
+function resolveWWWEndpoint() {
+  const currentEnv = config.deploymentEnv;
+  if (currentEnv === 'production') {
+    return 'https://exp.host';
+  } else if (currentEnv === 'staging') {
+    return 'https://staging.exp.host';
+  } else {
+    return 'http://127.0.0.1:3000';
+  }
+}
+
+export { sendBuildStatusUpdate, sendCredentialsUpdate };