Instantiate logger only at entrypoints

This ensures buildPhase, jobID, and experienceName labels are propagated when appropriate.

.circleci/config.yml

@@ -1,4 +1,139 @@
-version: 2
+version: 2.1
+
+executors:
+  nix: &nix
+    docker:
+      - image: lnl7/nix:2.1.2
+    # Symlink to the readline-enabled bash which is the default command of the container
+    shell: /run/current-system/sw/bin/bash -leo pipefail
+    working_directory: ~/turtle
+  mac:
+    macos: # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions
+      xcode: '10.1.0'
+    working_directory: /Users/distiller/project
+    shell: /bin/bash -leo pipefail
+  android:
+    <<: *nix
+    resource_class: xlarge
+
+commands:
+  install_nix:
+    steps:
+      - run:
+          name: Install nix
+          command: curl https://nixos.org/releases/nix/nix-2.2.1/install | bash -s -- --no-daemon
+      - run:
+          name: Add nix-initializing line to .bashrc
+          command: echo ". ~/.nix-profile/etc/profile.d/nix.sh" >> ~/.bashrc
+
+  setup:
+    steps:
+      - run:
+          name: Install common packages
+          command: |
+            nix-env -iA \
+              nixpkgs.binutils \
+              nixpkgs.direnv \
+              nixpkgs.findutils \
+              nixpkgs.gawk \
+              nixpkgs.git \
+              nixpkgs.gnutar \
+              nixpkgs.gzip \
+              nixpkgs.openssh \
+              nixpkgs.python \
+              nixpkgs.xz
+      - run:
+          name: Configure direnv
+          command: |
+            mkdir -p ~/.config/direnv
+            echo -e "[whitelist]\nprefix = [ \"$HOME\" ]" > ~/.config/direnv/config.toml
+            echo 'eval "$(direnv export bash)"' >> ~/.bash_profile
+      - run:
+          name: Configure yarn
+          command: echo '--frozen-lockfile true' >> ~/.yarnrc
+      - checkout
+      - run:
+          name: Configure aws-cli
+          command: |
+              aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
+              aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+
+  setup_nix:
+    steps:
+      - run:
+          name: Add cp and rm stubs
+          command: |
+            cat \<< 'EOF' >> /bin/cp
+            #! /usr/bin/env bash
+            /usr/bin/env cp "$@"
+            EOF
+            chmod +x /bin/cp
+
+            cat \<< 'EOF' >> /bin/rm
+            #! /usr/bin/env bash
+            /usr/bin/env rm "$@"
+            EOF
+            chmod +x /bin/rm
+
+  decrypt_secrets_if_possible:
+    steps:
+      - run:
+          name: Decrypt secrets if possible
+          command: |
+            if [[ -z "$TURTLE_GIT_CRYPT_KEY_BASE64" ]]; then
+              echo 'git-crypt key not present in environment'
+            else
+              git crypt unlock <(echo $TURTLE_GIT_CRYPT_KEY_BASE64 | base64 --decode)
+            fi
+
+  restore_yarn_cache:
+    parameters:
+      workingdir:
+        type: string
+        default: .
+    steps:
+      - restore_cache:
+          key: yarn-v1-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
+
+  save_yarn_cache:
+    parameters:
+      workingdir:
+        type: string
+        default: .
+    steps:
+      - save_cache:
+          key: yarn-v1-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
+          paths:
+            - ~/.cache/yarn # `yarn cache dir`
+
+  restore_workingdir_cache:
+    parameters:
+      platform:
+        type: string
+        default: ios
+      sdk_version:
+        type: string
+        default: sdk32
+    steps:
+      - restore_cache:
+          key: "workingdir-v1-<< parameters.platform >>-<< parameters.sdk_version >>"
+
+  save_workingdir_cache:
+    parameters:
+      workingdir:
+        type: string
+        default: /Users/distiller/project/workingdir
+      platform:
+        type: string
+        default: ios
+      sdk_version:
+        type: string
+        default: sdk32
+    steps:
+      - save_cache:
+          key: "workingdir-v1-<< parameters.platform >>-<< parameters.sdk_version >>"
+          paths:
+            - "<< parameters.workingdir >>/<< parameters.platform >>/<< parameters.sdk_version >>"
 
 jobs:
   build:
@@ -9,17 +144,68 @@ jobs:
       - run: yarn
       - run: yarn lint
       - run: yarn build
-  test:
-    docker:
-      - image: circleci/node:8.12
+
+  unit_tests:
+    executor: nix
     steps:
-      - checkout
+      - setup
+      - restore_yarn_cache
       - run: yarn
-      - run: ./scripts/testUnitCI.sh
+      - save_yarn_cache
+      - run:
+          name: Run iOS smoke tests
+          command: ./scripts/tests/runUnitTestsCI.sh
+
+  ios_smoke_tests:
+    executor: mac
+    steps:
+      - install_nix
+      - setup
+      - decrypt_secrets_if_possible
+      - restore_yarn_cache
+      - run: yarn
+      - save_yarn_cache
+      - restore_workingdir_cache:
+          sdk_version: sdk32
+          platform: ios
+      - run:
+          name: Run iOS smoke tests
+          command: ./scripts/tests/runIOSSmokeTestsCI.sh
+      - save_workingdir_cache:
+          workingdir: /Users/distiller/project/workingdir
+          sdk_version: sdk32
+          platform: ios
+
+  android_smoke_tests:
+    executor: android
+    steps:
+      - setup
+      - setup_nix
+      - restore_yarn_cache
+      - run: yarn
+      - save_yarn_cache
+      - restore_workingdir_cache:
+          sdk_version: sdk32
+          platform: android
+      - run:
+          name: Fetch Android shell app
+          command: SDK_VERSION=sdk32 ./scripts/android/fetchRemoteAndroidTarball.sh
+      - run:
+          name: Run Android smoke tests
+          command: |
+            nix-shell nix/android-shell.nix --pure \
+              --run "./scripts/tests/runAndroidSmokeTestsCI.sh"
+      - save_workingdir_cache:
+          workingdir: ~/turtle/workingdir
+          sdk_version: sdk32
+          platform: android
 
 workflows:
-  version: 2
-  build_and_test:
+  checks:
     jobs:
       - build
-      - test
+  tests:
+    jobs:
+      - unit_tests
+      - ios_smoke_tests
+      - android_smoke_tests

.envrc

@@ -15,4 +15,6 @@ export NODE_ENV=development
 
 export EXPO_TURTLE_TOOLS_DIR="$(pwd)/tools"
 
-PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
+if [ -z "$EXPO_UNIVERSE_DIR" ]; then
+  PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
+fi

.gitignore

@@ -1,10 +1,12 @@
 !/src/node_modules
 
+node_modules
+yarn-error.log
+
 /build/
 /workingdir/*
 /artifacts/*
 shell.apk
-node_modules
 .secrets*
 
 *.swp

.ignore

@@ -0,0 +1 @@
+fixtures/

.secrets.local-common

@@ -2,7 +2,6 @@ LOGGER_LEVEL="debug"
 API_PROTOCOL="http"
 API_HOSTNAME="localhost"
 API_PORT="3000"
-LOGGLY_TOKEN=""
 SENTRY_DSN=""
 ENVIRONMENT="development"
 REDIS_URL="redis://localhost:6379/0"

CHANGELOG.md

@@ -5,6 +5,61 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Updated iOS shell app for SDK 32 to fix AppAuth issue
+
+## [0.6.0] - 2019-04-09
+### Changed
+- Removed Branch module from Android shell apps for all SDK versions
+
+## [0.5.15] - 2019-04-03
+### Added
+- Turtle smoke tests are now being executed on CircleCI from the context of this repository.
+### Changed
+- Updated iOS shell app for SDK 32 (fixes [this loading issue for slow connections](https://github.com/expo/expo/issues/3574)).
+- Minor infra changes (doesn't affect users).
+- Error logs format - error objects are now passed in an object being passed as the first argument to logger function.
+### Removed
+- Sending logs to loggly.
+
+## [0.5.14] - 2019-03-22
+### Added
+- `--config` parameter for turtle-cli commands, which allows to specify a custom path to app.json file
+### Changed
+- Updated iOS shell app for SDK 32 (fixes [this notifications issue](https://github.com/expo/expo/issues/3663)).
+
+## [0.5.13] - 2019-03-12
+### Added
+- Android SDK version support validation.
+### Changed
+- iOS Enterprise builds don't compile bitcode now.
+
+## [0.5.12] - 2019-02-26
+### Added
+- Google Cloud logger (as a bunyan stream).
+### Changed
+- Android builder passes the manifest from a build job message if it's available.
+
+## [0.5.11] - 2019-02-22
+### Added
+- Builder for iOS client apps.
+
+## [0.5.10] - 2019-02-19
+### Added
+- Register Turtle version in Redis.
+### Changed
+- Update Android shell app for SDK 32 to fix [update](https://github.com/expo/expo/issues/3504) and [back button](https://github.com/expo/expo/issues/1786) issues.
+
+## [0.5.9] - 2019-02-06
+- Update iOS shell app for SDK 32 (fixes bare notifications [issue](https://github.com/expo/expo/issues/3223)).
+
+## [0.5.7] - 2019-01-31
+- Update Android shell app for SDK 32 to fix bundled assets issue.
+
+## [0.5.6] - 2019-01-25
+- Update iOS shell app for SDK 30, 31 and 32 (built with xcode 10.1).
+
+## [0.5.5] - 2019-01-24
 ### Added
 - Sending build duration in AWS SQS messages (of `FINISHED` and `ERRORED` types).
 

Dockerfile

@@ -102,13 +102,6 @@ ENV PATH ${GRADLE_HOME}/bin:$PATH
 # Install Gulp
 RUN npm install -g gulp-cli
 
-# Install Git
-RUN echo 'deb http://http.debian.net/debian wheezy-backports main' >> /etc/apt/sources.list && \
-  apt-get update && \
-  apt-get -t wheezy-backports install git-core && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
-
 ADD . /app
 
 RUN for SDK_VERSION in `ls /app/workingdir/android/`; do \

jest/integration-config.json

@@ -9,6 +9,6 @@
     "^.+\\.ts$": "ts-jest"
   },
   "testRegex": "(test|spec)\\.ts$",
-  "moduleFileExtensions": ["ts", "js"],
+  "moduleFileExtensions": ["ts", "js", "json"],
   "verbose": true
 }

jest/smoke-config.json

@@ -9,6 +9,6 @@
     "^.+\\.ts$": "ts-jest"
   },
   "testMatch": [ "**/__smoke__/*.test.ts" ],
-  "moduleFileExtensions": ["ts", "js"],
+  "moduleFileExtensions": ["ts", "js", "json"],
   "verbose": true
 }

jest/unit-config.json

@@ -9,6 +9,6 @@
     "^.+\\.ts$": "ts-jest"
   },
   "testMatch": [ "**/__tests__/*.ts" ],
-  "moduleFileExtensions": ["ts", "js"],
+  "moduleFileExtensions": ["ts", "js", "json"],
   "verbose": true
 }

k8s/android/turtle-deployment.template.yml

@@ -50,8 +50,6 @@ spec:
             value: exp-shell-app-assets
           - name: AWS_S3_REGION
             value: us-west-1
-          - name: LOGGLY_SUBDOMAIN
-            value: exponent
           - name: TURTLE_TEMPORARY_FILES_DIR
             value: /tmp/turtle
           # vars from k8s secrets
@@ -100,11 +98,6 @@ spec:
               secretKeyRef:
                 name: turtle-js-config
                 key: aws_cloudwatch_region
-          - name: LOGGLY_TOKEN
-            valueFrom:
-              secretKeyRef:
-                name: turtle-js-config
-                key: loggly_token
           - name: REDIS_URL
             valueFrom:
               secretKeyRef:

nix/all-packages.nix

@@ -0,0 +1,5 @@
+self: super:
+
+{
+  nodejs = super.nodejs-8_x;
+}

nix/android-shell.nix

@@ -0,0 +1,50 @@
+with (import ./default.nix {});
+
+let
+
+   ndk = androidenv.androidndk_17c.override { fullNDK = true; };
+   ndk_root = "${ndk}/libexec/${ndk.name}";
+   sdk_path = if stdenv.isDarwin
+     then "/Library/Android/sdk"
+     else "/Android/Sdk"; # intentional Capital S
+
+in
+
+mkShell rec {
+
+  LANG="en_US.UTF-8";
+  JAVA_HOME=openjdk8;
+
+  ANDROID_SDK_ROOT=builtins.getEnv("HOME") + sdk_path;
+  ANDROID_NDK_ROOT="${ANDROID_SDK_ROOT}/ndk-bundle";
+  ANDROID_HOME=ANDROID_SDK_ROOT;
+  ANDROID_NDK=ANDROID_NDK_ROOT;
+
+  nativeBuildInputs = [
+    awscli
+    curl
+    git
+    nodejs-8_x
+    openjdk8
+    yarn
+  ];
+
+  passthru = { inherit ndk; };
+
+  shellHook = ''
+    ${./install-ndk-17c.sh} ${ndk} ${ndk_root}
+    yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --licenses --sdk_root="$ANDROID_SDK_ROOT"
+    yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --sdk_root="$ANDROID_SDK_ROOT" "platforms;android-28" | grep -v =
+    yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --sdk_root="$ANDROID_SDK_ROOT" "build-tools;28.0.3" | grep -v =
+    ${lib.optionalString stdenv.isLinux ''
+      for dep in lib lib64; do
+        if [ -L /$dep ] || [ ! -e /$dep ]; then
+          echo "Creating /$dep"
+          ln -s ${stdenv.cc.libc}/$dep /$dep
+        else
+          echo "Using existing /$dep"
+        fi
+      done
+    ''}
+  '';
+}

nix/default.nix

@@ -0,0 +1,19 @@
+let
+  pin = builtins.fromJSON (builtins.readFile ./nixpkgs.json);
+
+  nixpkgs = builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs-channels/archive/${pin.rev}.tar.gz";
+    inherit (pin) sha256;
+  };
+
+in
+
+{ ... } @ args:
+
+  import nixpkgs (args // {
+    config = {
+      allowUnfree = true;
+      android_sdk.accept_license = true;
+    };
+    overlays = [ (import ./all-packages.nix) ];
+  })

nix/install-ndk-17c.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+scriptdir="$(dirname ${BASH_SOURCE[0]})"
+shellnix="$scriptdir/android-shell.nix"
+
+ndk=${1:-$(nix-build "$shellnix" -A ndk --no-out-link)}
+ndkRoot=${2:-$(nix eval --file "$shellnix" --raw ANDROID_NDK_ROOT)}
+
+mkdir -p "$ANDROID_NDK_ROOT" # Create full path if necessary
+
+# Replace any existing link or directory with link
+rm -r "$ANDROID_NDK_ROOT"
+ln -sn "$ndkRoot" "$ANDROID_NDK_ROOT"

nix/nixpkgs.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/NixOS/nixpkgs-channels.git",
+  "rev": "80738ed9dc0ce48d7796baed5364eef8072c794d",
+  "date": "2018-11-17T11:18:10+01:00",
+  "sha256": "0anmvr6b47gbbyl9v2fn86mfkcwgpbd5lf0yf3drgm8pbv57c1dc",
+  "fetchSubmodules": false
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "turtle-cli",
-  "version": "0.5.5",
+  "version": "0.6.0",
   "description": "The command-line tool for building standalone Expo apps",
   "preferGlobal": true,
   "scripts": {
@@ -10,6 +10,7 @@
     "watch": "EXPENV_OVERRIDES=local-common,local,local-queue expenv gulp watch:source",
     "bin:turtle": "./bin/turtle.js",
     "lint": "tslint -c tslint.json 'src/**/*.ts'",
+    "lint:fix": "yarn lint --fix",
     "prepare": "yarn build",
     "build": "gulp build:server",
     "build:production": "./scripts/build.sh",
@@ -26,7 +27,7 @@
     "test:integration": "yarn test --config jest/integration-config.json",
     "test:unit": "yarn test --config jest/unit-config.json",
     "test:smoke": "yarn test --config jest/smoke-config.json --forceExit",
-    "test:smoke:android": "yarn test:smoke --testNamePattern=android",
+    "test:smoke:android": "PLATFORM=android yarn test:smoke --testNamePattern=android",
     "test:smoke:ios": "PLATFORM=ios yarn test:smoke --testNamePattern=iOS",
     "release": "./scripts/release.sh",
     "release:beta": "./scripts/releaseBeta.sh",
@@ -38,6 +39,17 @@
     "set-priorities:staging": "expenv babel-node scripts/setConfigurations.js",
     "set-priorities:production": "EXPENV_OVERRIDES=prod expenv babel-node scripts/setConfigurations.js"
   },
+  "husky": {
+    "hooks": {
+      "pre-commit": "lint-staged"
+    }
+  },
+  "lint-staged": {
+    "src/**/*.ts": [
+      "yarn lint:fix",
+      "git add"
+    ]
+  },
   "bin": {
     "turtle": "./bin/turtle.js"
   },
@@ -88,7 +100,9 @@
     "gulp-plumber": "^1.2.0",
     "gulp-sourcemaps": "^2.6.4",
     "gulp-typescript": "^4.0.2",
+    "husky": "^1.3.1",
     "jest": "^23.6.0",
+    "lint-staged": "^8.1.5",
     "nock": "^10.0.1",
     "nodemon": "^1.18.7",
     "release-it": "^10.0.0",
@@ -96,14 +110,16 @@
     "source-map-loader": "^0.2.3",
     "ts-jest": "^23.10.4",
     "tslint": "^5.10.0",
+    "tslint-eslint-rules": "^5.4.0",
     "typescript": "^3.0.1"
   },
   "dependencies": {
+    "@expo/spawn-async": "^1.4.0",
+    "@google-cloud/logging-bunyan": "^0.10.1",
     "async-retry": "^1.2.1",
     "aws-sdk": "^2.226.1",
     "bunyan": "^1.8.12",
     "bunyan-debug-stream": "^1.0.8",
-    "bunyan-loggly": "^1.3.4",
     "bunyan-sentry-stream": "^1.2.1",
     "commander": "^2.15.1",
     "copy": "^0.3.1",
@@ -122,6 +138,9 @@
     "tar": "^4.4.6",
     "uuid": "^3.2.1",
     "which": "^1.3.1",
-    "xdl": "52.0.14-alpha.6+371be041"
+    "xdl": "53.1.6-alpha.2"
+  },
+  "optionalDependencies": {
+    "@expo/traveling-fastlane-darwin": "^1.8.1"
   }
 }

scripts/android/fetchRemoteAndroidTarball.sh

@@ -1,24 +1,4 @@
 #!/usr/bin/env bash
 
-set -xeo pipefail
-
 ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
-ARTIFACTS_DIR="$ROOT_DIR/artifacts"
-WORKING_DIR="$ROOT_DIR/workingdir/android"
-ANDROID_SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/android"
-
-mkdir -p $ARTIFACTS_DIR
-
-for SDK_VERSION in `ls $ANDROID_SHELL_TARBALLS_DIRECTORY`; do
-  mkdir -p $WORKING_DIR/$SDK_VERSION
-  TARBALL_URI=$(head -n 1 $ANDROID_SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
-  if command -v run  >/dev/null 2>&1 ; then
-    run expo.s3env --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --command retry5 aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
-  else
-    # outside of universe aws credentials might not be set
-    if ! aws iam get-user >/dev/null 2>&1 ; then aws configure ; fi
-    aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
-  fi
-  tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
-  rm -rf $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
-done
+$ROOT_DIR/scripts/fetchRemoteTarballs.sh android

scripts/fetchRemoteTarballs.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -xeo pipefail
+
+PLATFORM=$1
+
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
+ARTIFACTS_DIR="$ROOT_DIR/artifacts"
+WORKING_DIR="$ROOT_DIR/workingdir/$PLATFORM"
+SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/$PLATFORM"
+
+mkdir -p $ARTIFACTS_DIR
+
+convert_s3_to_https() {
+  BUCKET=`echo $1 | sed 's/s3:\/\/\([^/]*\).*/\1/'`
+  KEY=`echo $1 | sed 's/s3:\/\/[^/]*\/\(.*\)/\1/'`
+  echo "https://s3.amazonaws.com/$BUCKET/$KEY"
+}
+
+download_shell_app() {
+  SDK_VERSION=$1
+  mkdir -p $WORKING_DIR/$SDK_VERSION
+  TARBALL_S3_URL=$(head -n 1 $SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
+  TARBALL_HTTPS_URL=$(convert_s3_to_https $TARBALL_S3_URL)
+  curl -s $TARBALL_HTTPS_URL -o $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
+  tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
+  rm -rf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
+}
+
+if [[ -z "$SDK_VERSION" ]]; then
+  for SDK_VERSION in `ls $SHELL_TARBALLS_DIRECTORY`; do
+    download_shell_app $SDK_VERSION
+  done
+else
+  if [ ! -d "$WORKING_DIR/$SDK_VERSION" ]; then
+    download_shell_app $SDK_VERSION
+  fi
+fi

scripts/ios/fetchRemoteIosTarball.sh

@@ -1,24 +1,4 @@
 #!/usr/bin/env bash
 
-set -xeo pipefail
-
 ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
-ARTIFACTS_DIR="$ROOT_DIR/artifacts"
-WORKING_DIR="$ROOT_DIR/workingdir/ios"
-IOS_SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/ios"
-
-mkdir -p $ARTIFACTS_DIR
-
-for SDK_VERSION in `ls $IOS_SHELL_TARBALLS_DIRECTORY`; do
-  mkdir -p $WORKING_DIR/$SDK_VERSION
-  TARBALL_URI=$(head -n 1 $IOS_SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
-  if command -v run  >/dev/null 2>&1 ; then
-    run expo.s3env --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --command retry5 aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
-  else
-    # outside of universe aws credentials might not be set
-    if ! aws iam get-user >/dev/null 2>&1 ; then aws configure ; fi
-    aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
-  fi
-  tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
-  rm -rf $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
-done
+$ROOT_DIR/scripts/fetchRemoteTarballs.sh ios

scripts/setWorkingdirMode.sh

@@ -48,10 +48,10 @@ setup_remote() {
     echo "Shell apps already fetched"
   else
     echo "Fetching shell apps"
-    if [ "$PLATFORM" != 'ios']; then
+    if [ "$PLATFORM" != 'ios' ]; then
       $ROOT_DIR/scripts/android/fetchRemoteAndroidTarball.sh
     fi
-    if [ "$PLATFORM" != 'android']; then
+    if [ "$PLATFORM" != 'android' ]; then
       $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
     fi
   fi

scripts/tests/runAndroidSmokeTestsCI.sh

@@ -1,37 +1,25 @@
 #!/usr/bin/env bash
 
+set -ueo pipefail
+
 export NODE_ENV=test
 
 export ENVIRONMENT="development"
 export AWS_ACCESS_KEY_ID="aws_id"
 export AWS_SECRET_ACCESS_KEY="secret"
-export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_ios_in.fifo"
-export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_ios_in.fifo"
-export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_android_in.fifo"
-export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_android_in.fifo"
-export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_out.fifo"
+export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in.fifo"
+export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in_priority.fifo"
+export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in.fifo"
+export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in_priority.fifo"
+export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_out.fifo"
 export AWS_SQS_REGION="aws.west2"
 export AWS_S3_BUCKET="exp-shell-app-assets"
 export AWS_S3_REGION="us-west-1"
 export AWS_CLOUDWATCH_REGION="us-west-1"
 export API_PROTOCOL="http"
-export API_HOSTNAME="localhost"
-export API_PORT="3000"
+export API_HOSTNAME="staging.expo.io"
+export API_PORT="443"
 export REDIS_URL="noop"
 export REDIS_CONFIG_URL="noop"
 
-ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
-$ROOT_DIR/scripts/android/fetchRemoteAndroidTarball.sh
-
 yarn test:smoke:android
-smoke_result=$?
-
-yarn test:unit
-unit_result=$?
-
-[[ $smoke_result != 0 || $unit_result != 0 ]] && \
-	echo && \
-	echo "Tests failed." && \
-	exit -1
-
-exit 0

scripts/tests/runIOSSmokeTestsCI.sh

@@ -1,13 +1,15 @@
 #!/usr/bin/env bash
 
+set -ueo pipefail
+
 export NODE_ENV=test
 
 export ENVIRONMENT="development"
-export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_ios_in.fifo"
-export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_android_in.fifo"
-export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_ios_in_priority.fifo"
-export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_android_in_priority.fifo"
-export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_out.fifo"
+export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in.fifo"
+export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in_priority.fifo"
+export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in.fifo"
+export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in_priority.fifo"
+export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_out.fifo"
 export AWS_SQS_REGION="aws.west2"
 export AWS_S3_BUCKET="exp-shell-app-assets"
 export AWS_S3_REGION="us-west-1"
@@ -18,18 +20,7 @@ export API_PORT="443"
 export REDIS_URL="noop"
 export REDIS_CONFIG_URL="noop"
 
-ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
-$ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
+SDK_VERSION=sdk32 $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
 
 yarn test:smoke:ios
-smoke_result=$?
-
-yarn test:unit
-unit_result=$?
-
-[[ $smoke_result != 0 || $unit_result != 0 ]] && \
-	echo && \
-	echo "Tests failed." && \
-	exit -1
-
-exit 0

scripts/tests/runUnitTestsCI.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -ueo pipefail
+
 export NODE_ENV=test
 
 export ENVIRONMENT="development"
@@ -21,6 +23,3 @@ export REDIS_URL="noop"
 export REDIS_CONFIG_URL="noop"
 
 yarn test:unit
-unit_result=$?
-
-exit $unit_result

shellTarballs/android/sdk26

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz

shellTarballs/android/sdk27

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz

shellTarballs/android/sdk28

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz

shellTarballs/android/sdk29

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz

shellTarballs/android/sdk30

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz

shellTarballs/android/sdk31

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-5c339ff7bab6f2161d378cc70e1bf04d32ad98dd.tar.gz
+s3://exp-artifacts/android-shell-builder-89b2d24c86737d5a489c6a18467e31b2f9ce9005.tar.gz

shellTarballs/android/sdk32

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-cb03fdc12dc74f699a6d3f22219cbcd30e01ac46.tar.gz
+s3://exp-artifacts/android-shell-builder-0cab68670fc50ce1dc16b7d86ca7976870062c39.tar.gz

shellTarballs/ios/client

@@ -0,0 +1 @@
+s3://exp-artifacts/ios-expo-client-15bb883c22173d5b01df19421580170e776041d0.tar.gz

shellTarballs/ios/sdk30

@@ -1 +1 @@
-s3://exp-artifacts/ios-shell-builder-sdk-30-7d017a96d00fbf18ee7e5581ea47528b9cc78afd.tar.gz
+s3://exp-artifacts/ios-shell-builder-sdk-30-6863276e6b9605e31228c845a047387571959e9b.tar.gz

shellTarballs/ios/sdk31

@@ -1 +1 @@
-s3://exp-artifacts/ios-shell-builder-sdk-latest-56824ecea6ba10f5a9d8991545a1c47ee3284f5f.tar.gz
+s3://exp-artifacts/ios-shell-builder-sdk-latest-e1a8bd0ab2e1eba546792c0ffc282b5b202ccc39.tar.gz

shellTarballs/ios/sdk32

@@ -1 +1 @@
-s3://exp-artifacts/ios-shell-builder-sdk-latest-f3de2d71b10c058501e99e0b1ce9b5a991e291be.tar.gz
+s3://exp-artifacts/ios-shell-builder-sdk-latest-dc981c7110b7d508de8e9c352064dc3d745c9f72.tar.gz

src/__smoke__/smoke.test.ts

@@ -34,7 +34,7 @@ async function performSmokeTest(recording: string) {
     });
   }
 
-  await jobManager.doJob();
+  await jobManager.doJob(logger);
   if (!fixtureExists) {
     await fs.writeFile(recording, JSON.stringify(nock.recorder.play()));
   }

src/aws/cloudwatch/index.ts

@@ -11,7 +11,6 @@ import {
   MetricRegistrationObject,
 } from 'turtle/aws/cloudwatch/types';
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 
 interface IState {
   metrics: IMetric[];
@@ -38,12 +37,12 @@ const cloudWatch = new aws.CloudWatch({
   secretAccessKey: config.aws.secretAccessKey,
 });
 
-export function init() {
+export function init(logger: any) {
   if (config.cloudwatch.disabled) {
     return;
   }
 
-  state.intervalId = setInterval(_dumpMetrics, config.cloudwatch.intervalMs);
+  state.intervalId = setInterval(_dumpMetrics(logger), config.cloudwatch.intervalMs);
 }
 
 export function addMetricData({
@@ -111,23 +110,25 @@ function _calculateGroupHash(metricName: string, extraDimensions: any[]) {
   return `${metricName}${dimString}`;
 }
 
-function _dumpMetrics() {
-  if (!state.metrics.length && Object.keys(state.registeredMetrics).length < 0) {
-    // empty metrics list, skipping
-    return;
-  }
+function _dumpMetrics(logger: any) {
+  return () => {
+    if (!state.metrics.length && Object.keys(state.registeredMetrics).length < 0) {
+      // empty metrics list, skipping
+      return;
+    }
 
-  logger.trace(
-    '[cloudwatch] lenghts for state.metrics=%d, state.registeredMetrics=%d',
-    state.metrics.length,
-    Object.keys(state.registeredMetrics).length,
-  );
+    logger.trace(
+      '[cloudwatch] lengths for state.metrics=%d, state.registeredMetrics=%d',
+      state.metrics.length,
+      Object.keys(state.registeredMetrics).length,
+    );
 
-  _addEmptyMetrics(state.metrics);
-  const formattedMetrics = _formatMetrics(state.metrics);
-  _pushMetrics(formattedMetrics);
+    _addEmptyMetrics(state.metrics);
+    const formattedMetrics = _formatMetrics(state.metrics);
+    _pushMetrics(formattedMetrics, logger);
 
-  state.metrics = [];
+    state.metrics = [];
+  };
 }
 
 function _addEmptyMetrics(metrics: IMetric[]) {
@@ -174,12 +175,12 @@ function _reduceMetrics(metrics: IMetric[]) {
   );
 }
 
-function _pushMetrics(metrics: IMetricsChunk[]) {
+function _pushMetrics(metrics: IMetricsChunk[], logger: any) {
   logger.trace('[cloudwatch] pushing metrics', JSON.stringify(metrics, null, '\t'));
   metrics.forEach((metricsChunk) => {
     cloudWatch.putMetricData(metricsChunk, (err) => {
       if (err) {
-        logger.warn('[cloudwatch]', err);
+        logger.warn({ err }, '[cloudwatch]');
       }
     });
   });

src/aws/cloudwatch/reducers.ts

@@ -7,7 +7,7 @@ export function sum(metrics: IMetric[]) {
   const reduced = metrics.reduce((acc: IMetric | null, i: IMetric) => {
     let result = acc;
     if (!result) {
-      result = {...i};
+      result = { ...i };
     } else {
       result.Value += i.Value;
     }

src/aws/sqs.ts

@@ -5,7 +5,6 @@ import _ from 'lodash';
 import { OUTPUT_QUEUE_URL, QUEUE_URL } from 'turtle/aws/utils';
 import config from 'turtle/config';
 import { JOB_UPDATE_TYPE } from 'turtle/constants/build';
-import logger from 'turtle/logger';
 import { getCurrentJobId } from 'turtle/turtleContext';
 
 const sqs = new AWS.SQS({
@@ -16,7 +15,7 @@ const sqs = new AWS.SQS({
 
 const VISIBILITY_TIMEOUT_SEC = 30;
 
-export async function receiveMessage(priority: string) {
+export async function receiveMessage(priority: string, logger: any) {
   const params = {
     MaxNumberOfMessages: 1,
     QueueUrl: QUEUE_URL(priority),
@@ -36,7 +35,7 @@ export async function receiveMessage(priority: string) {
         }
       })
       .catch((err) => {
-        logger.error('Receive error', err);
+        logger.error({ err }, 'Receive error');
         throw err;
       });
   });
@@ -62,19 +61,19 @@ export async function changeMessageVisibility(priority: string, receiptHandle: s
 }
 
 // Every VISIBILITY_TIMEOUT_SEC / 3 seconds we are telling AWS SQS
-// that we're still proccessing the message, so it does not
+// that we're still processing the message, so it does not
 // send the build job to another turtle agent
-export function changeMessageVisibilityRecurring(priority: string, receiptHandle: string, jobId: string) {
+export function changeMessageVisibilityRecurring(priority: string, receiptHandle: string, jobId: string, logger: any) {
   return setInterval(() => {
     if (getCurrentJobId() === jobId) {
       changeMessageVisibility(priority, receiptHandle).catch((err) => {
-        logger.warn('Error at change msg visibility', err);
+        logger.warn({ err }, 'Error at change msg visibility');
       });
     }
   }, (VISIBILITY_TIMEOUT_SEC * 1000) / 3);
 }
 
-export async function sendMessage(jobId: string, status: JOB_UPDATE_TYPE, data = {}) {
+export async function sendMessage(jobId: string, status: JOB_UPDATE_TYPE, data = {}, logger: any) {
   const params = {
     MessageBody: JSON.stringify({ jobId, status, ...data }),
     QueueUrl: OUTPUT_QUEUE_URL(),
@@ -86,6 +85,6 @@ export async function sendMessage(jobId: string, status: JOB_UPDATE_TYPE, data =
       return await sqs.sendMessage(params).promise();
     });
   } catch (err) {
-    logger.error(`Error sending SQS message: ${JSON.stringify(params)}}`, err);
+    logger.error({ err }, `Error sending SQS message: ${JSON.stringify(params)}}`);
   }
 }

src/bin/commands/build/index.ts

@@ -2,8 +2,8 @@ import { default as buildAndroid } from 'turtle/bin/commands/build/android';
 import { default as buildIOS } from 'turtle/bin/commands/build/ios';
 
 function createCommand(builder: any) {
-  return (program: any) => {
-    return builder(program, setCommonCommandOptions);
+  return (logger: any) => (program: any) => {
+    return builder(program, setCommonCommandOptions, logger);
   };
 }
 
@@ -20,7 +20,8 @@ function setCommonCommandOptions(cmd: any) {
     .option(
       '--release-channel <channel-name>',
       'pull from specified release channel (default: default)',
-    );
+    )
+    .option('-c --config <config-file>', 'specify a path to app.json');
 }
 
 export default {

src/bin/commands/index.ts

@@ -1,8 +1,9 @@
 import builders from 'turtle/bin/commands/build';
 import setup from 'turtle/bin/commands/setup';
 
-export const setupIOS = setup.ios;
-export const setupAndroid = setup.android;
-
-export const buildIOS = builders.ios;
-export const buildAndroid = builders.android;
+export default {
+  setupIOS: setup.ios,
+  setupAndroid: setup.android,
+  buildIOS: builders.ios,
+  buildAndroid: builders.android,
+};

src/bin/commands/setup.ts

@@ -4,9 +4,6 @@ import { ErrorWithCommandHelp } from 'turtle/bin/commands/ErrorWithCommandHelp';
 import { ErrorWithProgramHelp } from 'turtle/bin/commands/ErrorWithProgramHelp';
 import setup from 'turtle/bin/setup/setup';
 import { PLATFORMS } from 'turtle/constants';
-import logger from 'turtle/logger';
-
-const l = logger.withFields({ buildPhase: 'setting up environment' });
 
 function createSetupCommand(platform: string, os?: string) {
   return (program: any) => {
@@ -23,18 +20,19 @@ function createSetupCommand(platform: string, os?: string) {
   };
 }
 
-async function setupAction(program: any, cmd: any, platform: string, os?: string) {
+async function setupAction(program: any, cmd: any, platform: string, logger: any, os?: string) {
   try {
     if (os && process.platform !== os) {
       throw new Error('We don\'t support running standalone app builds for this platform on your operating system');
     }
 
     const { sdkVersion } = cmd;
-    await setup(platform, sdkVersion);
+    const l = logger.child({ buildPhase: 'setting up environment' });
+
+    await setup(platform, l, sdkVersion);
     l.info('it\'s all set!');
   } catch (err) {
-    logger.error(`Failed to setup environment for ${platform} builds`);
-    logger.error(err.stack);
+    logger.error({ err }, `Failed to setup environment for ${platform} builds`);
     if (err instanceof ErrorWithCommandHelp) {
       cmd.help();
     } else if (err instanceof ErrorWithProgramHelp) {

src/bin/index.ts

@@ -4,7 +4,7 @@ import program, { Command } from 'commander';
 import fs from 'fs-extra';
 import { LoggerDetach, ModuleVersion } from 'xdl';
 
-import * as commands from 'turtle/bin/commands';
+import commands from 'turtle/bin/commands';
 import logger from 'turtle/logger';
 
 const { name, version } = JSON.parse(fs.readFileSync(path.join(__dirname, '../../package.json'), 'utf-8'));
@@ -17,23 +17,23 @@ Command.prototype.asyncAction = function asyncAction(asyncFn: (...asyncFnArgs: a
   return this.action(async (...args) => {
     try {
       await checkForUpdateAsync();
-    } catch (e) {
-      logger.warn('Failed to check for turtle-cli update.');
+    } catch (err) {
+      logger.warn({ err }, 'Failed to check for turtle-cli update.');
     }
     return await asyncFn(...args);
   });
 };
 
 export function run(programName: string) {
-  runAsync(programName).catch((e) => {
-    logger.error('Uncaught Error', e);
+  runAsync(programName).catch((err) => {
+    logger.error({ err }, 'Uncaught Error');
     process.exit(1);
   });
 }
 
 async function runAsync(programName: string) {
   program.version(version);
-  Object.values(commands).forEach((command) => registerCommand(program, command));
+  Object.values(commands).map((command) => command(logger)).forEach((command) => registerCommand(program, command));
   program.parse(process.argv);
 
   const subCommand = process.argv[2];

src/bin/setup/android/index.ts

@@ -12,7 +12,6 @@ import { IToolDefinition } from 'turtle/bin/setup/utils/toolsDetector';
 import { formatShellAppDirectory } from 'turtle/builders/utils/android/workingdir';
 import config from 'turtle/config';
 import { PLATFORMS } from 'turtle/constants';
-import logger from 'turtle/logger';
 
 const which = util.promisify(_which);
 const REQUIRED_TOOLS: IToolDefinition[] = [
@@ -40,25 +39,24 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
     },
   },
 ];
-const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
-const l = logger.withFields(LOGGER_FIELDS);
 
-export default async function setup(sdkVersion?: string) {
-  await checkSystem(REQUIRED_TOOLS);
-  await prepareAndroidEnv();
+export default async function setup(logger: any, sdkVersion?: string) {
+  await checkSystem(REQUIRED_TOOLS, logger);
+  await prepareAndroidEnv(logger);
   if (sdkVersion) {
     await ensureShellAppIsPresent(
       sdkVersion,
       { formatShellAppDirectory, formatShellAppTarballUriPath },
+      logger,
       _shellAppPostDownloadAction,
     );
   }
 }
 
-async function prepareAndroidEnv() {
+async function prepareAndroidEnv(logger: any) {
   await fs.ensureDir(config.directories.androidDependenciesDir);
-  const sdkConfig = await ensureAndroidSDKIsPresent();
-  const ndkConfig = await ensureAndroidNDKIsPresent();
+  const sdkConfig = await ensureAndroidSDKIsPresent(logger);
+  const ndkConfig = await ensureAndroidNDKIsPresent(logger);
   _setEnvVars({ ...sdkConfig.envVars, ...ndkConfig.envVars });
   _alterPath([...sdkConfig.path, ...sdkConfig.path]);
 }
@@ -67,21 +65,23 @@ function formatShellAppTarballUriPath(sdkMajorVersion: string) {
   return path.join(config.directories.shellTarballsDir, PLATFORMS.ANDROID, `sdk${sdkMajorVersion}`);
 }
 
-async function _shellAppPostDownloadAction(workingdir: string) {
+async function _shellAppPostDownloadAction(workingdir: string, logger: any) {
   const inWorkingdir = (filename: string) => path.join(workingdir, filename);
 
   await fs.move(inWorkingdir('package.json'), inWorkingdir('exponent-package.json'));
   await fs.move(inWorkingdir('universe-package.json'), inWorkingdir('package.json'));
-  await _installNodeModules(workingdir);
+  await _installNodeModules(workingdir, logger);
   await fs.move(inWorkingdir('package.json'), inWorkingdir('universe-package.json'));
   await fs.move(inWorkingdir('exponent-package.json'), inWorkingdir('package.json'));
 
   // TODO: remove following lines after upgrading android shell app
   const toolsPublicDir = path.join(workingdir, 'tools-public');
-  await _installNodeModules(toolsPublicDir);
+  await _installNodeModules(toolsPublicDir, logger);
 }
 
-async function _installNodeModules(cwd: string) {
+async function _installNodeModules(cwd: string, logger: any) {
+  const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
+  const l = logger.child(LOGGER_FIELDS);
   l.info(`installing dependencies in ${cwd} directory...`);
   const command = await _shouldUseYarnOrNpm();
   await ExponentTools.spawnAsyncThrowError(command, ['install'], {

src/bin/setup/android/ndk.ts

@@ -7,20 +7,17 @@ import fs from 'fs-extra';
 import { formatArtifactDownloadPath } from 'turtle/bin/setup/utils/common';
 import download from 'turtle/bin/setup/utils/downloader';
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 
 const ANDROID_NDK_URL = `https://dl.google.com/android/repository/android-ndk-r10e-${os.platform()}-x86_64.zip`;
 
-const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
-const l = logger.withFields(LOGGER_FIELDS);
-
-export default async function ensureAndroidNDKIsPresent() {
+export default async function ensureAndroidNDKIsPresent(logger: any) {
   const androidNdkDir = path.join(config.directories.androidDependenciesDir, 'ndk');
   if (!(await fs.pathExists(androidNdkDir))) {
     await fs.ensureDir(androidNdkDir);
     const androidNdkDownloadPath = formatArtifactDownloadPath(ANDROID_NDK_URL);
 
     try {
+      const l = logger.child({ buildPhase: 'setting up environment' });
       l.info('Downloading Android NDK');
       await fs.ensureDir(config.directories.artifactsDir);
       await download(ANDROID_NDK_URL, androidNdkDownloadPath);

src/bin/setup/android/sdk.ts

@@ -8,20 +8,17 @@ import { ExponentTools } from 'xdl';
 import { formatArtifactDownloadPath } from 'turtle/bin/setup/utils/common';
 import download from 'turtle/bin/setup/utils/downloader';
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 
 const ANDROID_SDK_URL = 'https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip';
-
 const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
-const l = logger.withFields(LOGGER_FIELDS);
-
-export default async function ensureAndroidSDKIsPresent() {
+export default async function ensureAndroidSDKIsPresent(logger: any) {
   const androidSdkDir = path.join(config.directories.androidDependenciesDir, 'sdk');
   if (!(await fs.pathExists(androidSdkDir))) {
     await fs.ensureDir(androidSdkDir);
     const androidSdkDownloadPath = formatArtifactDownloadPath(ANDROID_SDK_URL);
 
     try {
+      const l = logger.child(LOGGER_FIELDS);
       l.info('Downloading Android SDK');
       await fs.ensureDir(config.directories.artifactsDir);
       await download(ANDROID_SDK_URL, androidSdkDownloadPath);

src/bin/setup/ios.ts

@@ -34,13 +34,13 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
   },
 ];
 
-export default async function setup(sdkVersion?: string) {
-  await checkSystem(REQUIRED_TOOLS);
+export default async function setup(logger: any, sdkVersion?: string) {
+  await checkSystem(REQUIRED_TOOLS, logger);
   if (sdkVersion) {
     await ensureShellAppIsPresent(sdkVersion, {
       formatShellAppDirectory,
       formatShellAppTarballUriPath,
-    });
+    }, logger);
   }
 }
 

src/bin/setup/setup.ts

@@ -4,15 +4,15 @@ import setupAndroid from 'turtle/bin/setup/android';
 import setupIos from 'turtle/bin/setup/ios';
 import { PLATFORMS } from 'turtle/constants';
 
-export default async function setup(platform: string, sdkVersion?: string) {
+export default async function setup(platform: string, logger: any, sdkVersion?: string) {
   if (sdkVersion && !isSemver(sdkVersion)) {
     throw new Error('SDK version is not valid.');
   }
 
   if (platform === PLATFORMS.IOS) {
-    return await setupIos(sdkVersion);
+    return await setupIos(logger, sdkVersion);
   } else if (platform === PLATFORMS.ANDROID) {
-    return await setupAndroid(sdkVersion);
+    return await setupAndroid(logger, sdkVersion);
   } else {
     throw new Error('This should never happen :(');
   }

src/bin/setup/utils/common.ts

@@ -6,51 +6,56 @@ import { ExponentTools } from 'xdl';
 
 import download from 'turtle/bin/setup/utils/downloader';
 import { ensureToolsAreInstalled, IToolDefinition } from 'turtle/bin/setup/utils/toolsDetector';
+import { IShellAppDirectoryConfig } from 'turtle/builders/utils/workingdir';
 import config from 'turtle/config';
-import logger from 'turtle/logger';
-
-const l = logger.withFields({ buildPhase: 'setting up environment' });
 
 interface IShellAppFormaters {
-  formatShellAppDirectory: (sdkVersion: string) => string;
+  formatShellAppDirectory: (config: IShellAppDirectoryConfig) => string;
   formatShellAppTarballUriPath: (sdkMajorVersion: string) => string;
 }
 
-type PostDownloadAction = (workingdir: string) => void;
+type PostDownloadAction = (workingdir: string, logger: any) => void;
 
-export async function checkSystem(requiredTools: IToolDefinition[]) {
-  await ensureToolsAreInstalled(requiredTools);
+export async function checkSystem(requiredTools: IToolDefinition[], logger: any) {
+  await ensureToolsAreInstalled(requiredTools, logger);
 }
 
 export async function ensureShellAppIsPresent(
   sdkVersion: string,
   formatters: IShellAppFormaters,
+  logger: any,
   postDownloadAction?: PostDownloadAction,
 ) {
-  const workingdir = formatters.formatShellAppDirectory(sdkVersion);
+  const workingdir = formatters.formatShellAppDirectory({ sdkVersion });
+  const l = logger.child({ buildPhase: 'setting up environment' });
   if (await fs.pathExists(workingdir)) {
     return;
   }
   l.info(`shell app for SDK ${sdkVersion} doesn't exist, downloading...`);
-  await _downloadShellApp(sdkVersion, workingdir, formatters);
+  await _downloadShellApp(sdkVersion, workingdir, formatters, l);
   if (postDownloadAction) {
-    await postDownloadAction(workingdir);
+    await postDownloadAction(workingdir, logger);
   }
 }
 
-async function _downloadShellApp(sdkVersion: string, targetDirectory: string, formatters: IShellAppFormaters) {
+async function _downloadShellApp(
+  sdkVersion: string,
+  targetDirectory: string,
+  formatters: IShellAppFormaters,
+  logger: any,
+) {
   const shellAppTarballS3Uri = await _readShellAppTarballS3Uri(sdkVersion, formatters);
   const tarballDownloadTargetPath = formatArtifactDownloadPath(shellAppTarballS3Uri);
   await fs.ensureDir(config.directories.artifactsDir);
   await download(shellAppTarballS3Uri, tarballDownloadTargetPath);
-  l.info('shell app has been downloaded');
+  logger.info('shell app has been downloaded');
   await fs.ensureDir(targetDirectory);
-  l.info('extracting shell app (this may take a while)...');
+  logger.info('extracting shell app (this may take a while)...');
   await tar.x({
     file: tarballDownloadTargetPath,
     C: targetDirectory,
   });
-  l.info('shell app extracted');
+  logger.info('shell app extracted');
 }
 
 export function formatArtifactDownloadPath(uri: string) {

src/bin/setup/utils/toolsDetector.ts

@@ -1,10 +1,7 @@
 import util from 'util';
 import _which from 'which';
 
-import logger from 'turtle/logger';
-
 const which = util.promisify(_which);
-const l = logger.withFields({ buildPhase: 'setting up environment' });
 
 export interface IToolDefinition {
   command: string;
@@ -12,7 +9,7 @@ export interface IToolDefinition {
   testFn?: () => Promise<boolean>;
 }
 
-export async function ensureToolsAreInstalled(tools: IToolDefinition[]) {
+export async function ensureToolsAreInstalled(tools: IToolDefinition[], logger: any) {
   let isAnyToolMissing = false;
   for (const { command, missingDescription, testFn } of tools) {
     try {
@@ -24,11 +21,12 @@ export async function ensureToolsAreInstalled(tools: IToolDefinition[]) {
         await which(command);
       }
     } catch (err) {
+      const l = logger.child({ buildPhase: 'setting up environment' });
       isAnyToolMissing = true;
       if (!testFn) {
-        l.error(`${command} is missing in your $PATH`);
+        l.error({ err }, `${command} is missing in your $PATH`);
       }
-      l.error(missingDescription);
+      l.error({ err }, missingDescription);
     }
   }
   if (isAnyToolMissing) {

src/bin/utils/builder.ts

@@ -5,10 +5,10 @@ import { ErrorWithProgramHelp } from 'turtle/bin/commands/ErrorWithProgramHelp';
 import setup from 'turtle/bin/setup/setup';
 import * as ProjectUtils from 'turtle/bin/utils/project';
 import * as UserUtils from 'turtle/bin/utils/user';
-import logger from 'turtle/logger';
 import { sanitizeJob } from 'turtle/validator';
 
 export function createBuilderAction({
+  logger,
   program,
   command,
   prepareCredentials,
@@ -60,9 +60,9 @@ export function createBuilderAction({
         publicUrl: cmd.publicUrl,
       };
 
-      const appJSON = await ProjectUtils.loadAppJSON(projectDirArg);
+      const appJSON = await ProjectUtils.loadAppJSON(projectDirArg, cmd.config);
       const sdkVersion = _.get(appJSON, 'expo.sdkVersion');
-      await setup(platform, sdkVersion);
+      await setup(platform, logger, sdkVersion);
       const credentials = await prepareCredentials(cmd);
       const rawJob = {
         ...buildJobObject(appJSON, args, credentials),
@@ -72,8 +72,7 @@ export function createBuilderAction({
       const job = await sanitizeJob(rawJob);
       await builder(job);
     } catch (err) {
-      logger.error(`Failed to build standalone app: ${err.message}`);
-      logger.error(err.stack);
+      logger.error({ err }, 'Failed to build standalone app');
       if (err instanceof ErrorWithCommandHelp) {
         command.help();
       } else if (err instanceof ErrorWithProgramHelp) {

src/bin/utils/project.ts

@@ -1,12 +1,17 @@
 import fs from 'fs-extra';
 import path from 'path';
 
-export async function loadAppJSON(projectDirArg: string) {
-  const projectDir = resolveAbsoluteDir(projectDirArg);
-  const appJSONPath = path.join(projectDir, 'app.json');
+export async function loadAppJSON(projectDirArg: string, config: string | null) {
+  let appJSONPath;
+  if (config) {
+    appJSONPath = resolveAbsoluteDir(config);
+  } else {
+    const projectDir = resolveAbsoluteDir(projectDirArg);
+    appJSONPath = path.join(projectDir, 'app.json');
+  }
   const appJSONExists = await fs.pathExists(appJSONPath);
   if (!appJSONExists) {
-    throw new Error(`Couldn't find app.json in ${projectDir} directory.`);
+    throw new Error(`Couldn't find app.json.`);
   } else {
     return require(appJSONPath);
   }

src/builders/BuildError.ts

@@ -0,0 +1,17 @@
+class BuildError extends Error {
+  public reason: BuildErrorReason;
+
+  constructor(reason: BuildErrorReason, message: string) {
+    super(message);
+    this.name = 'BuildError';
+    this.reason = reason;
+  }
+}
+
+enum BuildErrorReason {
+  SESSION_EXPIRED = 'session-expired',
+  PROVISIONING_PROFILE_MISSING = 'provisioning-profile-missing',
+}
+
+export default BuildError;
+export { BuildErrorReason };

src/builders/android.ts

@@ -10,18 +10,20 @@ import { formatShellAppDirectory } from 'turtle/builders/utils/android/workingdi
 import * as commonUtils from 'turtle/builders/utils/common';
 import * as imageHelpers from 'turtle/builders/utils/image';
 import { uploadBuildToS3 } from 'turtle/builders/utils/uploader';
+import { ensureCanBuildSdkVersion } from 'turtle/builders/utils/version';
 import config from 'turtle/config';
 import { IAndroidCredentials, IJob, IJobResult } from 'turtle/job';
-import logger from 'turtle/logger';
 
-export default async function buildAndroid(jobData: IJob): Promise<IJobResult> {
-  const credentials = await getOrCreateCredentials(jobData);
-  const apkFilePath = await runShellAppBuilder(jobData, credentials);
+export default async function buildAndroid(jobData: IJob, logger: any): Promise<IJobResult> {
+  await ensureCanBuildSdkVersion(jobData);
+  const credentials = await getOrCreateCredentials(jobData, logger);
+  const apkFilePath = await runShellAppBuilder(jobData, credentials, logger);
   const randomHex = uuidv4().replace(/-/g, '');
   const s3Filename = `${jobData.experienceName}-${randomHex}-signed.apk`;
   const s3FileKey = `android/${s3Filename}`;
   const fakeUploadFilename = s3Filename.replace('/', '\\');
   const artifactUrl = await uploadBuildToS3({
+    logger,
     uploadPath: apkFilePath,
     s3FileKey,
     ...config.builder.fakeUpload && {
@@ -38,6 +40,7 @@ export default async function buildAndroid(jobData: IJob): Promise<IJobResult> {
 async function runShellAppBuilder(
   jobData: IJob,
   credentials: IAndroidCredentials,
+  logger: any,
 ): Promise<string> {
   const { temporaryFilesRoot } = config.directories;
   await fs.ensureDir(temporaryFilesRoot);
@@ -60,25 +63,26 @@ async function runShellAppBuilder(
 
   const outputFilePath = path.join(temporaryFilesRoot, `shell-signed-${jobData.id}.apk`);
 
-  const { manifest: { sdkVersion: sdkVersionFromManifest = null } = {}, sdkVersion: sdkVersionFromJob } = jobData;
-  const sdkVersion = sdkVersionFromJob || sdkVersionFromManifest;
-  const workingDir = formatShellAppDirectory(sdkVersion);
+  const { config: jobConfig, manifest, sdkVersion: sdkVersionFromJob } = jobData;
+  const sdkVersion = _.get(manifest, 'sdkVersion', sdkVersionFromJob);
+  const workingDir = formatShellAppDirectory({ sdkVersion });
 
   try {
     await AndroidShellApp.createAndroidShellAppAsync({
       url: commonUtils.getExperienceUrl(jobData),
-      sdkVersion: _.get(jobData, 'manifest.sdkVersion') || jobData.sdkVersion,
+      sdkVersion,
       keystore: tempKeystorePath,
+      manifest,
       alias: credentials.keystoreAlias,
       keystorePassword: credentials.keystorePassword,
       keyPassword: credentials.keyPassword,
       privateConfigFile: tempShellAppConfigPath,
-      releaseChannel: jobData.config.releaseChannel,
+      releaseChannel: jobConfig.releaseChannel,
       workingDir,
       outputFile: outputFilePath,
     });
   } catch (err) {
-    commonUtils.logErrorOnce(err);
+    commonUtils.logErrorOnce(err, logger);
     throw err;
   } finally {
     if (!config.builder.skipCleanup) {

src/builders/index.ts

@@ -1,7 +1,10 @@
+import { IJob, IJobResult } from 'turtle/job';
 import buildAndroid from './android';
 import buildIOS from './ios';
 
-export default {
+const builders: { [index: string]: (job: IJob, logger: any) => Promise<IJobResult>} = {
   ios: buildIOS,
   android: buildAndroid,
 };
+
+export default builders;

src/builders/ios/archive.ts

@@ -7,7 +7,6 @@ import * as fileUtils from 'turtle/builders/utils/file';
 import * as keychain from 'turtle/builders/utils/ios/keychain';
 import runShellAppBuilder from 'turtle/builders/utils/ios/shellAppBuilder';
 import { IJob } from 'turtle/job';
-import logger from 'turtle/logger/index';
 
 export default async function buildArchive(ctx: IContext, job: IJob) {
   let keychainInfo;
@@ -19,7 +18,7 @@ export default async function buildArchive(ctx: IContext, job: IJob) {
     const manifest = await runShellAppBuilder(ctx, job);
     await buildAndSignIPA(ctx, job, keychainInfo.path, manifest);
   } catch (err) {
-    logErrorOnce(err);
+    logErrorOnce(err, ctx.logger);
     throw err;
   } finally {
     if (keychainInfo) {
@@ -29,11 +28,11 @@ export default async function buildArchive(ctx: IContext, job: IJob) {
 }
 
 async function buildAndSignIPA(ctx: IContext, job: IJob, keychainPath: string, manifest: any) {
-  const l = logger.withFields({ buildPhase: 'building and signing IPA' });
-  l.info('building and signing IPA');
+  ctx.logger = ctx.logger.child({ buildPhase: 'building and signing IPA' });
+  ctx.logger.info('building and signing IPA');
 
   const {
-    credentials: { provisioningProfile, certPassword, teamId, password },
+    credentials: { provisioningProfile, certPassword, teamId },
     config: { bundleIdentifier: bundleIdentifierFromConfig },
   } = job;
 
@@ -42,7 +41,7 @@ async function buildAndSignIPA(ctx: IContext, job: IJob, keychainPath: string, m
 
   const { provisioningProfilePath } = ctx;
   await fileUtils.writeBase64ToBinaryFile(provisioningProfilePath, provisioningProfile as string);
-  l.info('saved provisioning profile to temporary path');
+  ctx.logger.info('saved provisioning profile to temporary path');
 
   const ipaBuilder = createIPABuilder({
     keychainPath,
@@ -51,12 +50,11 @@ async function buildAndSignIPA(ctx: IContext, job: IJob, keychainPath: string, m
     certPath: ctx.tempCertPath,
     certPassword,
     teamID: teamId,
-    password,
     bundleIdentifier,
-    workingDir: ctx.workingDir,
+    workspacePath: ctx.workspacePath,
     manifest,
   });
   await ipaBuilder.build();
 
-  l.info(`done building and signing IPA`);
+  ctx.logger.info(`done building and signing IPA`);
 }

src/builders/ios/context.ts

@@ -15,6 +15,7 @@ export interface IContext {
   baseArchiveDir: string;
   buildDir: string;
   fakeUploadBuildPath?: string;
+  logger: any;
   outputPath: string;
   provisioningProfilePath: string;
   provisioningProfileDir: string;
@@ -22,12 +23,14 @@ export interface IContext {
   tempCertPath: string;
   uploadPath: string;
   workingDir: string;
+  workspacePath: string;
+  applicationFilesSrc: string;
 }
 
-const { DEFAULT_EXPOKIT_WORKSPACE_NAME } = IosShellApp;
+const { EXPOKIT_APP, EXPONENT_APP } = IosShellApp;
 const { BUILD_TYPES } = IOS;
 
-export function createBuilderContext(job: IJob): IContext {
+export function createBuilderContext(job: IJob, logger: any): IContext {
   const { join } = path;
   const appUUID = uuidv4();
   const {
@@ -39,12 +42,13 @@ export function createBuilderContext(job: IJob): IContext {
   } = job;
 
   const sdkVersion = sdkVersionFromJob || sdkVersionFromManifest;
-  const workingDir = formatShellAppDirectory(sdkVersion);
+  const workingDir = formatShellAppDirectory({ sdkVersion, buildType: buildType! });
 
   const context: any = {
     appDir: join(config.directories.temporaryFilesRoot, appUUID),
     appUUID,
     workingDir,
+    logger,
   };
   context.buildDir = join(context.appDir, 'build');
   context.provisioningProfileDir = join(context.appDir, 'provisioning');
@@ -55,24 +59,63 @@ export function createBuilderContext(job: IJob): IContext {
   context.tempCertPath = join(context.appDir, 'cert.p12');
   context.baseArchiveDir = join(context.appDir, 'archive');
 
-  if (buildType === BUILD_TYPES.SIMULATOR) {
-    context.outputPath = join(context.appDir, 'archive.tar.gz');
-    context.uploadPath = join(context.appDir, 'archive.tar.gz');
-    context.archiveDir = join(
-      context.baseArchiveDir,
-      'Release',
-      `${DEFAULT_EXPOKIT_WORKSPACE_NAME}.app`,
+  if (buildType === BUILD_TYPES.CLIENT) {
+    context.applicationFilesSrc = join(
+      workingDir,
+      'expo-client-build',
+      '**',
+      '*',
     );
-  } else if (buildType === BUILD_TYPES.ARCHIVE) {
+  } else {
+    context.applicationFilesSrc = join(
+      workingDir,
+      'shellAppBase-builds',
+      buildType as string,
+      '**',
+      '*',
+    );
+  }
+
+  if (buildType === BUILD_TYPES.ARCHIVE) {
     context.outputPath = join(context.appDir, 'archive.xcarchive');
     context.uploadPath = join(context.buildDir, 'archive.ipa');
     context.archiveDir = join(
       context.baseArchiveDir,
       'Release',
-      `${DEFAULT_EXPOKIT_WORKSPACE_NAME}.xcarchive`,
+      `${EXPOKIT_APP}.xcarchive`,
       'Products',
       'Applications',
-      `${DEFAULT_EXPOKIT_WORKSPACE_NAME}.app`,
+      `${EXPOKIT_APP}.app`,
+    );
+    context.workspacePath = path.join(
+      workingDir,
+      'shellAppWorkspaces',
+      'ios',
+      'default',
+      `${EXPOKIT_APP}.xcworkspace`,
+    );
+  } else if (buildType === BUILD_TYPES.CLIENT) {
+    context.outputPath = join(context.appDir, 'archive.xcarchive');
+    context.uploadPath = join(context.buildDir, 'archive.ipa');
+    context.archiveDir = join(
+      context.baseArchiveDir,
+      `${EXPONENT_APP}.xcarchive`,
+      'Products',
+      'Applications',
+      `${EXPONENT_APP}.app`,
+    );
+    context.workspacePath = path.join(
+      workingDir,
+      'ios',
+      `${EXPONENT_APP}.xcworkspace`,
+    );
+  } else if (buildType === BUILD_TYPES.SIMULATOR) {
+    context.outputPath = join(context.appDir, 'archive.tar.gz');
+    context.uploadPath = join(context.appDir, 'archive.tar.gz');
+    context.archiveDir = join(
+      context.baseArchiveDir,
+      'Release',
+      `${EXPOKIT_APP}.app`,
     );
   }
 

src/builders/ios/index.ts

@@ -1,35 +1,39 @@
-import path from 'path';
-
 import fs from 'fs-extra';
 import pick from 'lodash/pick';
-import { ExponentTools } from 'xdl';
 
 import buildArchive from 'turtle/builders/ios/archive';
 import { createBuilderContext, IContext } from 'turtle/builders/ios/context';
 import buildSimulator from 'turtle/builders/ios/simulator';
 import { logErrorOnce } from 'turtle/builders/utils/common';
+import prepareAdHocBuildCredentials from 'turtle/builders/utils/ios/adhocBuild';
 import { uploadBuildToS3 } from 'turtle/builders/utils/uploader';
+import { ensureCanBuildSdkVersion } from 'turtle/builders/utils/version';
 import config from 'turtle/config';
-import { IOS, PLATFORMS } from 'turtle/constants/index';
+import { IOS } from 'turtle/constants/index';
 import { IJob, IJobResult } from 'turtle/job';
 
 const { BUILD_TYPES } = IOS;
 
-let SUPPORTED_SDK_VERSIONS: number[];
+export default async function iosBuilder(job: IJob, logger: any): Promise<IJobResult> {
+  if (job.config.buildType !== BUILD_TYPES.CLIENT) {
+    await ensureCanBuildSdkVersion(job);
+  }
 
-export default async function iosBuilder(job: IJob): Promise<IJobResult> {
-  await ensureCanBuildSdkVersion(job);
-
-  const ctx = createBuilderContext(job);
+  const ctx = createBuilderContext(job, logger);
 
   try {
     await initBuilder(ctx);
 
     const { buildType } = job.config;
-    if (buildType === BUILD_TYPES.SIMULATOR) {
-      await buildSimulator(ctx, job);
-    } else if (buildType === BUILD_TYPES.ARCHIVE) {
+
+    if (buildType === BUILD_TYPES.CLIENT) {
+      await prepareAdHocBuildCredentials(job, ctx.logger);
+    }
+
+    if ([BUILD_TYPES.ARCHIVE, BUILD_TYPES.CLIENT].includes(buildType!)) {
       await buildArchive(ctx, job);
+    } else if (buildType === BUILD_TYPES.SIMULATOR) {
+      await buildSimulator(ctx, job);
     } else {
       throw new Error(`Unsupported iOS build type: ${buildType}`);
     }
@@ -37,7 +41,7 @@ export default async function iosBuilder(job: IJob): Promise<IJobResult> {
     const artifactUrl = await uploadBuildToS3(ctx);
     return { artifactUrl };
   } catch (err) {
-    logErrorOnce(err);
+    logErrorOnce(err, ctx.logger);
     throw err;
   } finally {
     await cleanup(ctx);
@@ -60,25 +64,3 @@ async function cleanup(ctx: IContext) {
 
 const getTemporaryDirs = (ctx: IContext) =>
   Object.values(pick(ctx, ['appDir', 'provisioningProfileDir']));
-
-async function ensureCanBuildSdkVersion(job: IJob) {
-  if (config.builder.useLocalWorkingDir) {
-    return;
-  }
-
-  if (!SUPPORTED_SDK_VERSIONS) {
-    SUPPORTED_SDK_VERSIONS = await findSupportedSdkVersions();
-  }
-
-  const targetMajorSdkVersion = ExponentTools.parseSdkMajorVersion(job.sdkVersion || job.manifest.sdkVersion);
-  if (!SUPPORTED_SDK_VERSIONS.includes(targetMajorSdkVersion)) {
-    throw new Error(`Unsupported SDK Version!`);
-  }
-}
-
-async function findSupportedSdkVersions(): Promise<number[]> {
-  const SDK_DIR_PREFIX = 'sdk';
-  const files = await fs.readdir(path.join(config.directories.workingDir, PLATFORMS.IOS));
-  const sdks = files.filter((file) => file.startsWith(SDK_DIR_PREFIX));
-  return sdks.map((sdk) => parseInt(sdk.substr(SDK_DIR_PREFIX.length), 10));
-}

src/builders/ios/simulator.ts

@@ -1,11 +1,10 @@
 import { IContext } from 'turtle/builders/ios/context';
 import runShellAppBuilder from 'turtle/builders/utils/ios/shellAppBuilder';
 import { IJob } from 'turtle/job';
-import logger from 'turtle/logger';
 
 export default async function buildSimulator(ctx: IContext, job: IJob) {
-  const l = logger.withFields({ buildPhase: 'running simulator builder' });
-  l.info('running simulator build');
+  ctx.logger = ctx.logger.child({ buildPhase: 'running simulator builder' });
+  ctx.logger.info('running simulator build');
   await runShellAppBuilder(ctx, job);
-  l.info('build complete');
+  ctx.logger.info('build complete');
 }

src/builders/utils/android/credentials.ts

@@ -2,17 +2,16 @@ import path from 'path';
 
 import fs from 'fs-extra';
 import uuidv4 from 'uuid/v4';
-import { AndroidKeystore } from 'xdl';
+import { Credentials } from 'xdl';
 
 import * as sqs from 'turtle/aws/sqs';
 import { UPDATE_CREDENTIALS } from 'turtle/constants/build';
 import { IAndroidCredentials, IJob } from 'turtle/job';
-import logger from 'turtle/logger';
 import { isOffline } from 'turtle/turtleContext';
 
-async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredentials> {
+async function getOrCreateCredentials(jobData: IJob, logger: any): Promise<IAndroidCredentials> {
   if (!jobData.credentials) {
-    const l = logger.withFields({ buildPhase: 'generating keystore' });
+    const l = logger.child({ buildPhase: 'generating keystore' });
     const credentials: any = {};
     l.info('Creating keystore');
     const keystoreFilename = `/tmp/keystore-${jobData.id}.tmp.jks`;
@@ -21,13 +20,15 @@ async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredential
     credentials.keystoreAlias = Buffer.from(jobData.experienceName).toString('base64');
     const androidPackage = jobData.manifest.android.package;
 
-    await AndroidKeystore.createKeystore({
-      keystorePassword: credentials.keystorePassword,
-      keyPassword: credentials.keyPassword,
-      keystoreFilename,
-      keystoreAlias: credentials.keystoreAlias,
+    await Credentials.Android.createKeystore(
+      {
+        keystorePath: keystoreFilename,
+        keystorePassword: credentials.keystorePassword,
+        keyPassword: credentials.keyPassword,
+        keyAlias: credentials.keystoreAlias,
+      },
       androidPackage,
-    });
+    );
 
     credentials.keystore = (await fs.readFile(keystoreFilename)).toString('base64');
     l.info('Keystore created successfully');
@@ -42,7 +43,7 @@ async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredential
     } else {
       await sqs.sendMessage(jobData.id, UPDATE_CREDENTIALS, {
         ...credentials,
-      });
+      }, l);
       l.info('Keystore sent to storage successfully');
     }
     await fs.unlink(keystoreFilename);

src/builders/utils/android/workingdir.ts

@@ -2,10 +2,11 @@ import path from 'path';
 
 import { ExponentTools } from 'xdl';
 
+import { IShellAppDirectoryConfig } from 'turtle/builders/utils/workingdir';
 import config from 'turtle/config';
 import { PLATFORMS } from 'turtle/constants';
 
-export function formatShellAppDirectory(sdkVersion: string) {
+export function formatShellAppDirectory({ sdkVersion }: IShellAppDirectoryConfig) {
   const majorSdkVersion = ExponentTools.parseSdkMajorVersion(sdkVersion);
   return config.builder.useLocalWorkingDir
     ? path.join(config.directories.workingDir, 'local')

src/builders/utils/common.ts

@@ -2,7 +2,6 @@ import * as url from 'url';
 
 import config from 'turtle/config';
 import { IJob } from 'turtle/job';
-import logger from 'turtle/logger';
 
 export function getExperienceUrl(job: IJob) {
   const { experienceName, config: jobConfig } = job;
@@ -18,7 +17,7 @@ export function getExperienceUrl(job: IJob) {
 
 const alreadyLoggedError = Symbol('alreadyLoggedError');
 
-export function logErrorOnce(err: any) {
+export function logErrorOnce(err: any, logger: any) {
   if (!err[alreadyLoggedError]) {
     logger.error(err.stack);
     err[alreadyLoggedError] = true;

src/builders/utils/image.ts

@@ -18,15 +18,15 @@ export async function resizeIconWithSharpAsync(iconSizePx: number, iconFilename:
   await fs.writeFile(filename, buffer);
 }
 
-export async function getImageDimensionsWithSharpAsync(basename: string, dirname: string) {
-  const filename = path.join(dirname, basename);
+export async function getImageDimensionsWithSharpAsync(dirname: string, filename: string) {
+  const filepath = path.join(dirname, filename);
 
   // PLEASE DON'T REMOVE THE FOLLOWING LINE (sharp caches files taking path as the cache key)
   sharp.cache(false);
 
   try {
-    const meta = await sharp(filename).metadata();
-    return [meta.width, meta.height];
+    const { width, height } = await sharp(filepath).metadata();
+    return { width, height };
   } catch (e) {
     return null;
   }

src/builders/utils/ios/adhocBuild.ts

@@ -0,0 +1,98 @@
+import path from 'path';
+
+import spawnAsync from '@expo/spawn-async';
+import fs from 'fs-extra';
+import isEmpty from 'lodash/isEmpty';
+import { IosCodeSigning } from 'xdl';
+
+import * as sqs from 'turtle/aws/sqs';
+import BuildError, { BuildErrorReason } from 'turtle/builders/BuildError';
+import { UPDATE_CREDENTIALS } from 'turtle/constants/build';
+import { IJob } from 'turtle/job';
+import { isOffline } from 'turtle/turtleContext';
+
+// tslint:disable-next-line:no-var-requires
+const travelingFastlane = process.platform === 'darwin' ? require('@expo/traveling-fastlane-darwin')() : null;
+
+async function prepareAdHocBuildCredentials(job: IJob, logger: any) {
+  if (process.platform !== 'darwin') {
+    throw new Error('This function should be called only on macOS!');
+  }
+
+  const { bundleIdentifier } = job.config;
+  const { certP12, certPassword, teamId, appleSession, udids } = job.credentials;
+
+  const certSerialNumber = IosCodeSigning.findP12CertSerialNumber(certP12, certPassword);
+  const args = [
+    teamId,
+    udids!.join(','),
+    bundleIdentifier,
+    certSerialNumber || '__last__',
+  ];
+
+  try {
+    const credentials = await runFastlaneAction(
+      travelingFastlane.manageAdHocProvisioningProfile,
+      args,
+      { env: { FASTLANE_SESSION: appleSession } },
+    );
+    if (isEmpty(credentials)) {
+      if (!job.credentials.provisioningProfile) {
+        // a user has generated the provisioning profile for his app, but he probably deleted it from Expo servers, ...
+        throw new BuildError(
+          BuildErrorReason.PROVISIONING_PROFILE_MISSING,
+          `Provisioning profile is missing on Expo servers.`,
+        );
+      }
+      return;
+    }
+
+    logger.info('New ad hoc provisioning profile successfully created');
+    job.credentials.provisioningProfile = credentials.provisioningProfile;
+
+    if (isOffline()) {
+      const provisioningProfilePath = path.join(job.projectDir, `adhoc.mobileprovision`);
+      await fs.writeFile(provisioningProfilePath, Buffer.from(credentials.provisioningProfile, 'base64'));
+      logger.info(`Saved provisioning profile to ${provisioningProfilePath}`);
+    } else {
+      await sqs.sendMessage(job.id, UPDATE_CREDENTIALS, {
+        provisioningProfileId: credentials.provisioningProfileId,
+        provisioningProfile: credentials.provisioningProfile,
+      }, logger);
+      logger.info('Ad Hoc provisioning profile sent to storage successfully');
+    }
+  } catch (e) {
+    if (e instanceof BuildError) {
+      logger.error('Apple Session expired! Please authenticate again using expo-cli.');
+    }
+    throw e;
+  }
+}
+
+async function runFastlaneAction(action: string, args: string[], { env }: any) {
+  const { stderr } = await spawnAsync(
+    action,
+    args,
+    {
+      stdio: ['inherit', 'pipe', 'pipe'],
+      env: { ...process.env, ...env },
+    },
+  );
+  const { result, ...rest } = JSON.parse(stderr.trim());
+  if (result === 'failure') {
+    const { reason, rawDump, type } = rest;
+    let errorMsg = `Reason: ${reason}`;
+    if (rawDump) {
+      errorMsg += `, raw: ${JSON.stringify(rawDump)}`;
+    }
+    if (type === 'session-expired') {
+      throw new BuildError(BuildErrorReason.SESSION_EXPIRED, errorMsg);
+    } else {
+      throw new Error(errorMsg);
+    }
+  } else {
+    return rest;
+  }
+}
+
+export default prepareAdHocBuildCredentials;

src/builders/utils/ios/keychain.ts

@@ -2,27 +2,26 @@ import { IosKeychain } from 'xdl';
 
 import { IContext } from 'turtle/builders/ios/context';
 import * as fileUtils from 'turtle/builders/utils/file';
-import logger from 'turtle/logger';
 
 interface IKeychain {
   path: string;
 }
 
 export async function create(ctx: IContext): Promise<IKeychain> {
-  const l = logger.withFields({ buildPhase: 'creating keychain' });
+  const l = ctx.logger.child({ buildPhase: 'creating keychain' });
   try {
     l.info('creating keychain...');
     const keychainInfo = await IosKeychain.createKeychain(ctx.appUUID, false);
     l.info('done creating keychain');
     return keychainInfo;
   } catch (err) {
-    l.error('unable to create keychain');
+    l.error({ err }, 'unable to create keychain');
     throw err;
   }
 }
 
 export async function remove(ctx: IContext, keychainPath: string) {
-  const l = logger.withFields({ buildPhase: 'deleting keychain' });
+  const l = ctx.logger.child({ buildPhase: 'deleting keychain' });
   try {
     l.info('delete keychain...');
     const keychainInfo = await IosKeychain.deleteKeychain({
@@ -32,7 +31,7 @@ export async function remove(ctx: IContext, keychainPath: string) {
     l.info('done deleting keychain');
     return keychainInfo;
   } catch (err) {
-    l.error('unable to delete keychain');
+    l.error({ err }, 'unable to delete keychain');
     throw err;
   }
 }
@@ -49,7 +48,7 @@ export async function importCert(
     certPassword,
   }: { keychainPath: string; certP12: string; certPassword: string },
 ) {
-  const l = logger.withFields({ buildPhase: 'importing certificate into keychain' });
+  const l = ctx.logger.child({ buildPhase: 'importing certificate into keychain' });
   try {
     l.info('importing distribution certificate into keychain...');
     const { tempCertPath: certPath } = ctx;
@@ -57,7 +56,7 @@ export async function importCert(
     await IosKeychain.importIntoKeychain({ keychainPath, certPath, certPassword });
     l.info('done importing distribution certificate into keychain');
   } catch (err) {
-    l.error('unable to import distribution certificate into keychain');
+    l.error({ err }, 'unable to import distribution certificate into keychain');
     throw err;
   }
 }

src/builders/utils/ios/shellAppBuilder.ts

@@ -8,9 +8,8 @@ import { ImageUtils, IosShellApp } from 'xdl';
 import { IContext } from 'turtle/builders/ios/context';
 import * as commonUtils from 'turtle/builders/utils/common';
 import * as imageHelpers from 'turtle/builders/utils/image';
-import { PLATFORMS } from 'turtle/constants';
+import { IOS } from 'turtle/constants/index';
 import { IJob } from 'turtle/job';
-import logger from 'turtle/logger/index';
 
 const copyAsync = util.promisify(copy);
 
@@ -18,35 +17,37 @@ export default async function runShellAppBuilder(ctx: IContext, job: IJob): Prom
   const { config: jobConfig, manifest, sdkVersion: sdkVersionFromJob } = job;
   const { buildType, releaseChannel } = jobConfig;
   const sdkVersion = _.get(manifest, 'sdkVersion', sdkVersionFromJob);
-  const applicationFilesSrc = path.join(
-    ctx.workingDir,
-    'shellAppBase-builds',
-    buildType as string,
-    '**',
-    '*',
-  );
-  await copyAsync(applicationFilesSrc, ctx.baseArchiveDir);
 
-  logger.info(
+  await copyAsync(ctx.applicationFilesSrc, ctx.baseArchiveDir);
+
+  ctx.logger.info(
     { buildPhase: 'icons setup' },
     'ImageUtils: setting image functions to alternative sharp implementations',
   );
   ImageUtils.setResizeImageFunction(imageHelpers.resizeIconWithSharpAsync);
   ImageUtils.setGetImageDimensionsFunction(imageHelpers.getImageDimensionsWithSharpAsync);
   const shellAppParams = {
-    url: commonUtils.getExperienceUrl(job),
-    sdkVersion,
     action: 'configure',
     type: buildType,
-    releaseChannel,
     archivePath: ctx.archiveDir,
     privateConfigData: job.config,
-    verbose: true,
-    output: ctx.outputPath,
-    expoSourcePath: path.join(ctx.workingDir, PLATFORMS.IOS),
+    expoSourcePath: path.join(ctx.workingDir, 'ios'),
     manifest,
+    output: ctx.outputPath,
+    verbose: true,
   };
+  if (buildType === IOS.BUILD_TYPES.CLIENT) {
+    Object.assign(shellAppParams, {
+      workspacePath: path.join(ctx.workingDir, 'ios'),
+    });
+  } else {
+    Object.assign(shellAppParams, {
+      url: commonUtils.getExperienceUrl(job),
+      releaseChannel,
+      sdkVersion,
+    });
+  }
 
-  logger.info({ buildPhase: 'configuring NSBundle' }, 'configuring NSBundle...');
+  ctx.logger.info({ buildPhase: 'configuring NSBundle' }, 'configuring NSBundle...');
   return await IosShellApp.configureAndCopyArchiveAsync(shellAppParams);
 }

src/builders/utils/ios/workingdir.ts

@@ -2,12 +2,18 @@ import path from 'path';
 
 import { ExponentTools } from 'xdl';
 
+import { IShellAppDirectoryConfig } from 'turtle/builders/utils/workingdir';
 import config from 'turtle/config';
-import { PLATFORMS } from 'turtle/constants';
+import { IOS, PLATFORMS } from 'turtle/constants';
 
-export function formatShellAppDirectory(sdkVersion: string) {
-  const majorVersion = ExponentTools.parseSdkMajorVersion(sdkVersion);
-  return config.builder.useLocalWorkingDir
-    ? path.join(config.directories.workingDir, 'local')
-    : path.join(config.directories.workingDir, PLATFORMS.IOS, `sdk${majorVersion}`);
+export function formatShellAppDirectory({ sdkVersion, buildType }: IShellAppDirectoryConfig) {
+  const { workingDir } = config.directories;
+  if (buildType === IOS.BUILD_TYPES.CLIENT) {
+    return path.join(workingDir, PLATFORMS.IOS, 'client');
+  } else if (config.builder.useLocalWorkingDir) {
+    return path.join(workingDir, 'local');
+  } else {
+    const majorVersion = ExponentTools.parseSdkMajorVersion(sdkVersion);
+    return path.join(workingDir, PLATFORMS.IOS, `sdk${majorVersion}`);
+  }
 }

src/builders/utils/uploader.ts

@@ -2,24 +2,24 @@ import * as fs from 'fs-extra';
 
 import { uploadFile } from 'turtle/aws/s3';
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 
 interface IUploadCtx {
   fakeUploadBuildPath?: string;
   s3FileKey?: string;
   uploadPath: string;
+  logger: any;
 }
 
 export async function uploadBuildToS3(ctx: IUploadCtx) {
   if (config.builder.fakeUpload) {
-    const l = logger.withFields({ buildPhase: 'copying build artifact' });
+    const l = ctx.logger.child({ buildPhase: 'copying build artifact' });
     const { fakeUploadBuildPath, uploadPath } = ctx;
     l.info('copying build to fake upload directory');
     await fs.copy(uploadPath, fakeUploadBuildPath as string);
     l.info(`copied build to ${fakeUploadBuildPath}`);
     return fakeUploadBuildPath as string;
   } else {
-    const l = logger.withFields({ buildPhase: 'uploading to S3' });
+    const l = ctx.logger.child({ buildPhase: 'uploading to S3' });
     l.info('uploading build artifact to S3');
     const { Location: fileLocation } = await uploadFile({
       key: ctx.s3FileKey as string,

src/builders/utils/version.ts

@@ -0,0 +1,30 @@
+import fs from 'fs-extra';
+import path from 'path';
+import { ExponentTools } from 'xdl';
+
+import config from 'turtle/config';
+import { IJob } from 'turtle/job';
+
+let SUPPORTED_SDK_VERSIONS: number[];
+
+export async function ensureCanBuildSdkVersion(job: IJob) {
+  if (config.builder.useLocalWorkingDir) {
+    return;
+  }
+
+  if (!SUPPORTED_SDK_VERSIONS) {
+    SUPPORTED_SDK_VERSIONS = await findSupportedSdkVersions(job.platform);
+  }
+
+  const targetMajorSdkVersion = ExponentTools.parseSdkMajorVersion(job.sdkVersion || job.manifest.sdkVersion);
+  if (!SUPPORTED_SDK_VERSIONS.includes(targetMajorSdkVersion)) {
+    throw new Error(`Unsupported SDK Version!`);
+  }
+}
+
+async function findSupportedSdkVersions(platform: 'android' | 'ios'): Promise<number[]> {
+  const SDK_DIR_PREFIX = 'sdk';
+  const files = await fs.readdir(path.join(config.directories.workingDir, platform));
+  const sdks = files.filter((file) => file.startsWith(SDK_DIR_PREFIX));
+  return sdks.map((sdk) => parseInt(sdk.substr(SDK_DIR_PREFIX.length), 10));
+}

src/builders/utils/workingdir.ts

@@ -0,0 +1,6 @@
+import { IOS } from 'turtle/constants';
+
+export interface IShellAppDirectoryConfig {
+  sdkVersion: string;
+  buildType?: IOS.BUILD_TYPES;
+}

src/config.ts

@@ -8,7 +8,7 @@ export default {
   env: env('NODE_ENV'),
   hostname: env('HOSTNAME', os.hostname()),
   deploymentEnv: env('ENVIRONMENT'),
-  platform: env('PLATFORM', PLATFORMS.IOS),
+  platform: env('PLATFORM', PLATFORMS.IOS) as PLATFORMS,
   aws: {
     accessKeyId: env('AWS_ACCESS_KEY_ID'),
     secretAccessKey: env('AWS_SECRET_ACCESS_KEY'),
@@ -47,11 +47,9 @@ export default {
     client: {
       level: env('CLIENT_LOGGER_LEVEL', 'info'),
     },
-    loggly: {
-      token: env('LOGGLY_TOKEN', ''),
-      subdomain: env('LOGGLY_SUBDOMAIN', ''),
-      buffer: envNum('LOGGLY_BUFFER', 100),
-    },
+  },
+  google: {
+    credentials: env('GOOGLE_APPLICATION_CREDENTIALS', ''),
   },
   sentry: {
     dsn: env('SENTRY_DSN', ''),

src/constants/index.ts

@@ -1,9 +1,9 @@
 import * as BUILD from './build';
 import * as IOS from './ios';
 
-export const PLATFORMS = {
-  IOS: 'ios',
-  ANDROID: 'android',
-};
+export enum PLATFORMS {
+  IOS = 'ios',
+  ANDROID = 'android',
+}
 
 export { IOS, BUILD };

src/constants/ios.ts

@@ -1,4 +1,5 @@
-export const BUILD_TYPES = {
-  ARCHIVE: 'archive',
-  SIMULATOR: 'simulator',
-};
+export enum BUILD_TYPES {
+  ARCHIVE = 'archive',
+  CLIENT = 'client',
+  SIMULATOR = 'simulator',
+}

src/job.ts

@@ -1,3 +1,5 @@
+import { IOS } from 'turtle/constants';
+
 export interface IJob {
   platform: 'android' | 'ios';
   experienceName: string;
@@ -8,7 +10,7 @@ export interface IJob {
     releaseChannel: string;
     publicUrl?: string;
     // ios
-    buildType?: string;
+    buildType?: IOS.BUILD_TYPES;
     bundleIdentifier?: string;
   };
   credentials: {
@@ -16,13 +18,14 @@ export interface IJob {
     keystore?: string;
     keystoreAlias?: string;
     keystorePassword?: string;
-    // ios
     keyPassword?: string;
+    // ios
     certP12?: string;
     certPassword?: string;
     provisioningProfile?: string;
-    password?: string;
     teamId?: string;
+    appleSession?: string;
+    udids?: string[];
   };
   projectDir: string;
   fakeUploadDir?: string;

src/jobManager.ts

@@ -1,85 +1,80 @@
-import path from 'path';
-
-import fs from 'fs-extra';
 import _ from 'lodash';
 
 import * as sqs from 'turtle/aws/sqs';
 import builders from 'turtle/builders';
+import BuildError from 'turtle/builders/BuildError';
 import { logErrorOnce } from 'turtle/builders/utils/common';
 import config from 'turtle/config';
 import { BUILD } from 'turtle/constants/index';
-import logger from 'turtle/logger';
 import * as buildDurationMetric from 'turtle/metrics/buildDuration';
 import * as buildStatusMetric from 'turtle/metrics/buildStatus';
-import { checkShouldExit, setCurrentJobId } from 'turtle/turtleContext';
+import { checkShouldExit, setCurrentJobId, turtleVersion } from 'turtle/turtleContext';
 import { getPriorities } from 'turtle/utils/priorities';
 import * as redis from 'turtle/utils/redis';
 import { sanitizeJob } from 'turtle/validator';
 
-const { version: turtleVersion } = JSON.parse(fs.readFileSync(path.join(__dirname, '../package.json'), 'utf-8'));
-
-function _maybeExit() {
+function _maybeExit(logger: any) {
   if (checkShouldExit()) {
     logger.warn('Exiting due to previously received termination signal.');
     process.exit();
   }
 }
 
-export async function doJob() {
-  const jobData = await getJob();
-  await processJob(jobData);
-  _maybeExit();
+export async function doJob(logger: any) {
+  const jobData = await getJob(logger);
+  await processJob(jobData, logger);
+  _maybeExit(logger);
 }
 
-export async function getJob() {
-  _maybeExit();
+export async function getJob(logger: any) {
+  _maybeExit(logger);
   logger.info('Fetching job');
   while (true) {
     try {
       let job = null;
-      const priorities = await getPriorities();
+      const priorities = await getPriorities(logger);
       for (const priority of priorities) {
-        job = await sqs.receiveMessage(priority);
-        _maybeExit();
+        job = await sqs.receiveMessage(priority, logger);
+        _maybeExit(logger);
         if (job != null) {
           job.priority = priority;
           break;
         }
       }
-      _maybeExit();
+      _maybeExit(logger);
       if (job !== null) {
         return job;
       }
     } catch (err) {
-      logger.error('Error at receiving messages', err);
+      logger.error({ err }, 'Error at receiving messages');
     }
   }
 }
 
-async function processJob(jobData: any) {
+async function processJob(jobData: any, logger: any) {
   const receiptHandle = jobData.ReceiptHandle;
   const { priority } = jobData;
   let timeoutId;
   try {
     const rawJob = JSON.parse(jobData.Body);
-    timeoutId = failAfterMaxJobTime(priority, receiptHandle, rawJob);
+    timeoutId = failAfterMaxJobTime(priority, receiptHandle, rawJob, logger);
     const job = await sanitizeJob(rawJob);
 
     setCurrentJobId(job.id);
-    const pingerHandle = sqs.changeMessageVisibilityRecurring(priority, jobData.ReceiptHandle, job.id);
+    const pingerHandle = sqs.changeMessageVisibilityRecurring(priority, jobData.ReceiptHandle, job.id, logger);
 
     logger.info(`Doing job MessageId=${jobData.MessageId} BuildId=${job.id} ${Date.now()}`);
 
-    const cancelled = await redis.checkIfCancelled(job.id);
+    const cancelled = await redis.checkIfCancelled(job.id, logger);
     if (cancelled) {
       logger.info('The job has been cancelled');
     } else {
-      redis.registerListener(job.id, () => deleteMessage(priority, receiptHandle));
+      redis.registerListener(job.id, () => deleteMessage(priority, receiptHandle, logger), logger);
       const startTimestamp = Date.now();
       let buildFailed = false;
       const buildType = _.get(job, 'config.buildType', 'default');
       try {
-        const status = await build(job);
+        const status = await build(job, logger);
         buildStatusMetric.add(buildType, status);
         buildFailed = !status;
       } catch (err) {
@@ -97,13 +92,13 @@ async function processJob(jobData: any) {
           );
           buildDurationMetric.addTotalDuration(buildType, totalBuildDurationSecs, !buildFailed);
         }
-        redis.unregisterListeners();
+        redis.unregisterListeners(logger);
       }
       logger.info(`Job done MessageId=${jobData.MessageId} BuildId=${job.id} ${Date.now()}`);
     }
     clearInterval(pingerHandle);
   } finally {
-    await deleteMessage(priority, receiptHandle);
+    await deleteMessage(priority, receiptHandle, logger);
     if (timeoutId) {
       logger.info('Clearing job failer timeout...');
       clearTimeout(timeoutId);
@@ -111,11 +106,11 @@ async function processJob(jobData: any) {
   }
 }
 
-function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any) {
+function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any, logger: any) {
   return setTimeout(async () => {
     try {
-      sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, { turtleVersion });
-      await deleteMessage(priority, receiptHandle);
+      sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, { turtleVersion }, logger);
+      await deleteMessage(priority, receiptHandle, logger);
     } finally {
       logger.error('Going to terminate turtle agent, just in case...');
       process.exit(1);
@@ -123,39 +118,48 @@ function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any)
   }, config.builder.maxJobTimeMs);
 }
 
-async function build(job: any) {
+async function build(job: any, logger: any) {
   const startTimestamp = Date.now();
   const s3Url = await logger.init(job);
+  const buildLogger = logger.child({ jobID: job.id, experienceName: job.experienceName });
   const calculateBuildDuration = () => Math.ceil((Date.now() - startTimestamp) / 1000);
 
   try {
     await sqs.sendMessage(job.id, BUILD.JOB_STATES.IN_PROGRESS, {
       logUrl: s3Url,
       logFormat: 'json',
-    });
-    const result = await (builders as any)[job.platform](job);
+    }, buildLogger);
+    const result = await builders[job.platform](job, buildLogger);
     const buildDuration = calculateBuildDuration();
     sqs.sendMessage(job.id, BUILD.JOB_STATES.FINISHED, {
       ...result,
       turtleVersion,
       buildDuration,
-    });
+    }, buildLogger);
     return true;
   } catch (err) {
-    logErrorOnce(err);
+    logErrorOnce(err, buildLogger);
     const buildDuration = calculateBuildDuration();
-    sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, { turtleVersion, buildDuration });
+    let reason;
+    if (err instanceof BuildError) {
+      reason = err.reason;
+    }
+    sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, {
+      turtleVersion,
+      buildDuration,
+      ...reason && { reason },
+    }, buildLogger);
     return false;
   } finally {
     await logger.cleanup(job);
   }
 }
 
-async function deleteMessage(priority: string, receiptHandle: string) {
+async function deleteMessage(priority: string, receiptHandle: string, logger: any) {
   try {
     setCurrentJobId(null);
     await sqs.deleteMessage(priority, receiptHandle);
   } catch (err) {
-    logger.error('Error at deleting msg', err);
+    logger.error({ err }, 'Error at deleting msg');
   }
 }

src/jobsSchemas/ios.ts

@@ -12,11 +12,13 @@ export default baseJobSchema.concat(
     }),
     credentials: Joi.object()
       .keys({
-        certP12: Joi.string(),
-        certPassword: Joi.string(),
-        teamId: Joi.string(),
+        certP12: Joi.string().required(),
+        certPassword: Joi.string().required(),
+        teamId: Joi.string().required(),
+        // it's not required for adhoc Expo Client builds
         provisioningProfile: Joi.string(),
-        password: Joi.string(),
+        appleSession: Joi.string(),
+        udids: Joi.array().items(Joi.string()),
       })
       .when('config.buildType', {
         is: IOS.BUILD_TYPES.SIMULATOR,

src/logger/hackyLogglyStream.ts

@@ -1,48 +0,0 @@
-import EventEmitter from 'events';
-
-import BunyanLoggly from 'bunyan-loggly';
-
-import config from 'turtle/config';
-import * as constants from 'turtle/constants/logger';
-import { IJob } from 'turtle/job';
-
-export interface ILogglyConfig {
-  token: string;
-  subdomain: string;
-  tags: string[];
-}
-
-interface ILogglyExtraFields {
-  jobID: string;
-  experienceName: string;
-}
-
-export default class HackyLogglyStream extends EventEmitter {
-  private logglyStream: BunyanLoggly;
-  private extraFields: ILogglyExtraFields | null;
-
-  constructor(logglyConfig: ILogglyConfig) {
-    super();
-    this.logglyStream = new BunyanLoggly(logglyConfig, config.logger.loggly.buffer);
-    this.extraFields = null;
-  }
-  public init(job: IJob) {
-    this.extraFields = {
-      jobID: job.id,
-      experienceName: job.experienceName,
-    };
-  }
-  public cleanup() {
-    this.extraFields = null;
-  }
-  public write(rec: any) {
-    const { msg: message, hostname: host, level, ...other } = rec;
-    this.logglyStream.write({
-      message,
-      level: constants.LEVELS_MAPPING[level],
-      host,
-      ...other,
-      data: this.extraFields || {},
-    });
-  }
-}

src/logger/index.ts

@@ -4,12 +4,15 @@ import { SentryStream } from 'bunyan-sentry-stream';
 import { Client as RavenClient } from 'raven';
 
 import config from 'turtle/config';
-import * as constants from 'turtle/constants/logger';
 import { IJob } from 'turtle/job';
-import HackyLogglyStream from 'turtle/logger/hackyLogglyStream';
 import S3Stream from 'turtle/logger/s3Stream';
 import { isOffline } from 'turtle/turtleContext';
 
+// type error when using import
+// https://github.com/googleapis/nodejs-logging-bunyan/issues/241
+// tslint:disable-next-line:no-var-requires
+const { LoggingBunyan } = require('@google-cloud/logging-bunyan');
+
 interface IStream {
   stream: any;
   type?: string;
@@ -18,7 +21,6 @@ interface IStream {
 }
 
 export const s3logger = new S3Stream();
-let logglyStream: HackyLogglyStream;
 
 const streams: IStream[] = [];
 
@@ -47,53 +49,34 @@ if (config.sentry.dsn && config.deploymentEnv !== 'development') {
   });
 }
 
-if (config.logger.loggly.token) {
-  const logglyConfig = {
-    token: config.logger.loggly.token,
-    subdomain: config.logger.loggly.subdomain,
-    // prettier-ignore
-    tags: [
-      'app-shell-apps',
-      `platform.${config.platform}`,
-      `environment.${config.deploymentEnv}`,
-    ],
+if (config.google.credentials) {
+  const resource = {
+    type: 'generic_node',
+    labels: {
+      node_id: config.hostname,
+      location: '', // default value
+      namespace: '', // default value
+    },
   };
-  logglyStream = new HackyLogglyStream(logglyConfig);
-  streams.push({
-    type: 'raw',
-    stream: logglyStream,
-  });
+  streams.push(new LoggingBunyan({ name: 'turtle', resource }).stream('info'));
 }
 
 const logger = bunyan.createLogger({
   name: 'turtle',
   level: config.logger.level,
+  platform: config.platform,
+  environment: config.deploymentEnv,
   streams,
 });
 
-logger.withFields = (extraFields: any) => withFields(logger, extraFields);
-
 logger.init = async (job: IJob) => {
-  if (logglyStream) {
-    logglyStream.init(job);
-  }
   return await s3logger.init(job);
 };
 
 logger.cleanup = async () => {
   // a little hacky, but works
   logger.info({ lastBuildLog: true }, 'this is the last log from this build');
-  if (logglyStream) {
-    logglyStream.cleanup();
-  }
   await s3logger.waitForLogger();
 };
 
 export default logger;
-
-export function withFields(loggerObj: any, extraFields: any) {
-  return constants.LEVELS.reduce((obj, level) => {
-    obj[level] = (...args: any[]) => loggerObj[level](extraFields, ...args);
-    return obj;
-  }, {} as any);
-}

src/metrics/buildDuration.ts

@@ -1,11 +1,10 @@
 import * as cloudwatch from 'turtle/aws/cloudwatch';
 import { TYPE_DIMENSIONS } from 'turtle/aws/cloudwatch/dimensions';
-import logger from 'turtle/logger';
 
 const TURTLE_BUILD_DURATION_METRIC_NAME = 'turtleBuildDuration';
 const TOTAL_BUILD_DURATION_METRIC_NAME = 'totalBuildDuration';
 
-export function register() {
+export function register(logger: any) {
   for (const metricName of [TURTLE_BUILD_DURATION_METRIC_NAME, TOTAL_BUILD_DURATION_METRIC_NAME]) {
     logger.info(`Registered AWS Cloudwatch metric: ${metricName}`);
     cloudwatch.registerMetric({

src/metrics/buildStatus.ts

@@ -2,11 +2,10 @@ import * as cloudwatch from 'turtle/aws/cloudwatch';
 import { TYPE_DIMENSIONS } from 'turtle/aws/cloudwatch/dimensions';
 import * as reducers from 'turtle/aws/cloudwatch/reducers';
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 
 const BUILD_METRIC_NAME = 'build';
 
-export function register() {
+export function register(logger: any) {
   logger.info(`Registered AWS Cloudwatch metric: ${BUILD_METRIC_NAME}`);
   cloudwatch.registerMetric({
     metricName: BUILD_METRIC_NAME,

src/metrics/index.ts

@@ -1,12 +1,11 @@
 import * as cloudwatch from 'turtle/aws/cloudwatch';
-import logger from 'turtle/logger';
 import * as buildDuration from 'turtle/metrics/buildDuration';
 import * as buildStatus from 'turtle/metrics/buildStatus';
 
 const metricsToRegister = [buildStatus, buildDuration];
 
-export async function init() {
+export async function init(logger: any) {
   logger.info('Initializing Cloudwatch metrics');
-  await cloudwatch.init();
-  metricsToRegister.forEach((metric) => metric.register());
+  await cloudwatch.init(logger);
+  metricsToRegister.forEach((metric) => metric.register(logger));
 }

src/node_modules/bunyan-loggly/index.d.ts

@@ -1,13 +0,0 @@
-export = BunyanLoggly;
-
-declare class BunyanLoggly {
-  constructor(options: BunyanLoggly.IOptions, bufferLength?: number, bufferTimeout?: number, logglyCallback?: any);
-  public write(data: any): void;
-}
-
-declare namespace BunyanLoggly {
-  interface IOptions {
-    token: string;
-    subdomain: string;
-  }
-}

src/server.ts

@@ -4,9 +4,12 @@ import config from 'turtle/config';
 import { doJob } from 'turtle/jobManager';
 import logger from 'turtle/logger';
 import setup from 'turtle/setup';
-import { checkShouldExit, setShouldExit } from 'turtle/turtleContext';
+import { checkShouldExit, setShouldExit, turtleVersion } from 'turtle/turtleContext';
+import { getRedisClient, RedisClient } from 'turtle/utils/redis';
 
-process.on('unhandledRejection', (err) => logger.error('Unhandled promise rejection:', err));
+const REDIS_TURTLE_VERSION_KEY = 'turtle:version';
+
+process.on('unhandledRejection', (err) => logger.error({ err }, 'Unhandled promise rejection'));
 
 function handleExit() {
   if (checkShouldExit()) {
@@ -28,18 +31,27 @@ async function main() {
   );
   LoggerDetach.configure(logger);
   if (setup[config.platform]) {
-    await setup[config.platform]();
+    await setup[config.platform](logger);
   }
+
+  try {
+    const redis = await getRedisClient(logger, RedisClient.Configuration);
+    await redis.set(REDIS_TURTLE_VERSION_KEY, turtleVersion);
+    logger.info(`Register Turtle version ${turtleVersion} in Redis`);
+  } catch (err) {
+    logger.error({ err }, 'Failed to register Turtle version in Redis');
+  }
+
   while (true) {
     try {
-      await doJob();
+      await doJob(logger);
     } catch (err) {
-      logger.error('Failed to do a job', err);
+      logger.error({ err }, 'Failed to do a job');
     }
   }
 }
 
 main()
   .then(() => logger.error('This should never happen...'))
-  .catch((err) => logger.error('Something went terribly wrong :(', err))
+  .catch((err) => logger.error({ err }, 'Something went terribly wrong'))
   .then(() => process.exit(1));

src/setup/android.ts

@@ -1,8 +1,7 @@
-import logger from 'turtle/logger';
 import commonSetup from 'turtle/setup/common';
 
-export default async function setup() {
+export default async function setup(logger: any) {
   logger.info('Setting up environment...');
-  await commonSetup();
+  await commonSetup(logger);
   logger.info('Finished setting up environment');
 }

src/setup/common.ts

@@ -3,8 +3,8 @@ import fs from 'fs-extra';
 import config from 'turtle/config';
 import * as metrics from 'turtle/metrics';
 
-export default async function commonSetup() {
-  await metrics.init();
+export default async function commonSetup(logger: any) {
+  await metrics.init(logger);
   await fs.remove(config.directories.temporaryFilesRoot);
   await fs.remove(config.directories.tempS3LogsDir);
 }

src/setup/ios.ts

@@ -3,7 +3,6 @@ import os from 'os';
 import path from 'path';
 
 import * as keychain from 'turtle/builders/utils/ios/keychain';
-import logger from 'turtle/logger';
 import commonSetup from 'turtle/setup/common';
 
 async function deleteProvisioningProfilesFromHomedir() {
@@ -17,10 +16,10 @@ async function deleteProvisioningProfilesFromHomedir() {
   }
 }
 
-export default async function setup() {
+export default async function setup(logger: any) {
   logger.info('Setting up environment...');
   await deleteProvisioningProfilesFromHomedir();
   await keychain.cleanUp();
-  await commonSetup();
+  await commonSetup(logger);
   logger.info('Finished setting up environment');
 }

src/turtleContext.ts

@@ -1,5 +1,10 @@
+import fs from 'fs-extra';
+import path from 'path';
 import config from 'turtle/config';
 
+const { version: turtleVersion } = JSON.parse(fs.readFileSync(path.join(__dirname, '../package.json'), 'utf-8'));
+export { turtleVersion };
+
 let shouldExit = false;
 let currentJobId: string | null = null;
 

src/utils/priorities.ts

@@ -2,7 +2,6 @@ import Joi from 'joi';
 import os from 'os';
 
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 import { getRedisClient, RedisClient } from 'turtle/utils/redis';
 
 export const NORMAL = 'normalPriority';
@@ -40,7 +39,11 @@ export function createDefaultConfigurationKey(platform = config.platform) {
   return `${configPrefix(platform)}:default`;
 }
 
-export async function getPriorities() {
+export async function getPriorities(logger: any) {
+  if (config.env === 'test') {
+    return NORMAL_CONFIGURATION;
+  }
+
   try {
     const redis = await getRedisClient(RedisClient.Configuration);
     if (redis.getConfig === undefined) {
@@ -56,24 +59,23 @@ export async function getPriorities() {
       logger.debug(`Using configuration at index ${configurationsIndex} pulled from redis`);
       const { error, value: configValue } = configSchema.validate(configurations[configurationsIndex]);
       if (error) {
-        logger.warn(error);
+        logger.warn({ err: error });
         throw new Error('Received configuration is not valid');
       }
       return configValue;
     } catch (err) {
-      logger.warn(err);
+      logger.warn({ err });
       const defaultConfig = JSON.parse(await redis.get(createDefaultConfigurationKey()));
       const { error, value: defaultConfigValue } = configSchema.validate(defaultConfig);
       if (error) {
-        logger.warn(error);
+        logger.warn({ err: error });
         throw new Error('Received default configuration is not valid');
       }
       logger.warn('Using default configuration pulled from redis');
       return defaultConfigValue;
     }
   } catch (err) {
-    logger.warn(err);
-    logger.warn('Using configuration chosen locally');
+    logger.warn({ err }, 'Using configuration chosen locally');
     if (Math.random() < 0.7) {
       return HIGH_CONFIGURATION;
     } else {

src/utils/redis.ts

@@ -1,7 +1,6 @@
 import Redis from 'ioredis';
 
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 
 const MILLIS_TO_UPLOAD_LOGS = 3000;
 export const MILLIS_CONNECTION_TIMEOUT = 10000;
@@ -47,31 +46,31 @@ const redisClients: IRedisClients = {
   [RedisClient.Configuration]: null,
 };
 
-export async function getRedisClient(type = RedisClient.Default) {
+export async function getRedisClient(logger: any, type = RedisClient.Default) {
   if (!redisClients[type]) {
     try {
       redisClients[type] = await connect(MILLIS_CONNECTION_TIMEOUT, type);
     } catch (err) {
-      logger.error(err);
+      logger.error({ err });
     }
   }
   return redisClients[type];
 }
 
-export async function checkIfCancelled(jobId: string) {
+export async function checkIfCancelled(jobId: string, logger: any) {
   try {
-    const redis = await getRedisClient();
+    const redis = await getRedisClient(logger);
     return await redis.get(`jobs:cancelled:${jobId}`);
   } catch (err) {
     if (config.deploymentEnv === 'development') {
       logger.warn('Did you turn on redis server? Run `yarn start-docker` in server/www');
     }
-    logger.error(err);
+    logger.error({ err });
     return false;
   }
 }
 
-export async function registerListener(jobId: string, deleteMessage: any) {
+export async function registerListener(jobId: string, deleteMessage: any, logger: any) {
   try {
     const redis = await getRedisClient(RedisClient.Subscriber);
     redis.subscribe('jobs:cancelled');
@@ -88,9 +87,9 @@ export async function registerListener(jobId: string, deleteMessage: any) {
   }
 }
 
-export async function unregisterListeners() {
+export async function unregisterListeners(logger: any) {
   try {
-    const redis = await getRedisClient();
+    const redis = await getRedisClient(logger);
     redis.removeAllListeners('message');
   } catch (err) {
     logger.error(err);

tools/bin/aws

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws "$@"

tools/bin/aws_bash_completer

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws_bash_completer "$@"

tools/bin/aws_completer

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws_completer "$@"

tools/bin/git-crypt

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" git-crypt --command git-crypt "$@"

tools/bin/nix-binstubs

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+set -euo pipefail
+
+scriptDir=$(dirname "$0")
+nixDir="$scriptDir/../../nix"
+
+nix build -f $nixDir --no-link "$@"
+
+for package in "$@"; do
+  for b in $(ls $(nix path-info -f $nixDir $package)/bin); do
+    cat << EOF > "$scriptDir/$b"
+#! /usr/bin/env bash
+# This file was generated by $(basename "$0")
+exec nix run -f "\$(dirname "\$0")/../../nix" $package --command $b "\$@"
+EOF
+  chmod +x "$scriptDir/$b"
+  done
+done
+

tools/bin/node

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" nodejs-8_x --command node "$@"

tools/bin/npm

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" nodejs-8_x --command npm "$@"

tools/bin/npx

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" nodejs-8_x --command npx "$@"

tools/bin/yarn

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" yarn --command yarn "$@"