fix for breaking changes in xdl

.circleci/config.yml

@@ -1,212 +1,25 @@
-version: 2.1
-
-executors:
-  nix: &nix
-    docker:
-      - image: lnl7/nix:2.2
-    # Symlink to the readline-enabled bash which is the default command of the container
-    shell: /run/current-system/sw/bin/bash -leo pipefail
-    working_directory: ~/turtle
-  mac:
-    macos: # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions
-      xcode: '10.1.0'
-    working_directory: /Users/distiller/project
-    shell: /bin/bash -leo pipefail
-  android:
-    <<: *nix
-    resource_class: xlarge
-
-commands:
-  install_nix:
-    steps:
-      - run:
-          name: Install nix
-          command: curl https://nixos.org/releases/nix/nix-2.2.1/install | bash -s -- --no-daemon
-      - run:
-          name: Add nix-initializing line to .bashrc
-          command: echo ". ~/.nix-profile/etc/profile.d/nix.sh" >> ~/.bashrc
-
-  setup:
-    steps:
-      - run:
-          name: Install common packages
-          command: |
-            nix-env -iA \
-              nixpkgs.binutils \
-              nixpkgs.direnv \
-              nixpkgs.findutils \
-              nixpkgs.gawk \
-              nixpkgs.git \
-              nixpkgs.gnutar \
-              nixpkgs.gzip \
-              nixpkgs.openssh \
-              nixpkgs.python \
-              nixpkgs.xz
-      - run:
-          name: Configure direnv
-          command: |
-            mkdir -p ~/.config/direnv
-            echo -e "[whitelist]\nprefix = [ \"$HOME\" ]" > ~/.config/direnv/config.toml
-            echo 'eval "$(direnv export bash)"' >> ~/.bash_profile
-      - run:
-          name: Configure yarn
-          command: echo '--frozen-lockfile true' >> ~/.yarnrc
-      - checkout
-      - run:
-          name: Configure aws-cli
-          command: |
-              aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
-              aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
-
-  setup_nix:
-    steps:
-      - run:
-          name: Add cp and rm stubs
-          command: |
-            cat \<< 'EOF' >> /bin/cp
-            #! /usr/bin/env bash
-            /usr/bin/env cp "$@"
-            EOF
-            chmod +x /bin/cp
-
-            cat \<< 'EOF' >> /bin/rm
-            #! /usr/bin/env bash
-            /usr/bin/env rm "$@"
-            EOF
-            chmod +x /bin/rm
-
-  decrypt_secrets_if_possible:
-    steps:
-      - run:
-          name: Decrypt secrets if possible
-          command: |
-            if [[ -z "$TURTLE_GIT_CRYPT_KEY_BASE64" ]]; then
-              echo 'git-crypt key not present in environment'
-            else
-              git crypt unlock <(echo $TURTLE_GIT_CRYPT_KEY_BASE64 | base64 --decode)
-            fi
-
-  restore_yarn_cache:
-    parameters:
-      workingdir:
-        type: string
-        default: .
-    steps:
-      - restore_cache:
-          key: yarn-v1-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
-
-  save_yarn_cache:
-    parameters:
-      workingdir:
-        type: string
-        default: .
-    steps:
-      - save_cache:
-          key: yarn-v1-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
-          paths:
-            - ~/.cache/yarn # `yarn cache dir`
-
-  restore_workingdir_cache:
-    parameters:
-      platform:
-        type: string
-        default: ios
-      sdk_version:
-        type: string
-        default: sdk32
-    steps:
-      - restore_cache:
-          key: "workingdir-v1-<< parameters.platform >>-<< parameters.sdk_version >>"
-
-  save_workingdir_cache:
-    parameters:
-      workingdir:
-        type: string
-        default: /Users/distiller/project/workingdir
-      platform:
-        type: string
-        default: ios
-      sdk_version:
-        type: string
-        default: sdk32
-    steps:
-      - save_cache:
-          key: "workingdir-v1-<< parameters.platform >>-<< parameters.sdk_version >>"
-          paths:
-            - "<< parameters.workingdir >>/<< parameters.platform >>/<< parameters.sdk_version >>"
+version: 2
 
 jobs:
   build:
-    executor: nix
+    docker:
+      - image: circleci/node:8.12
     steps:
-      - setup
-      - restore_yarn_cache
+      - checkout
       - run: yarn
-      - save_yarn_cache
       - run: yarn lint
       - run: yarn build
-
-  unit_tests:
-    executor: nix
+  test:
+    docker:
+      - image: circleci/node:8.12
     steps:
-      - setup
-      - restore_yarn_cache
+      - checkout
       - run: yarn
-      - save_yarn_cache
-      - run:
-          name: Run unit tests
-          command: ./scripts/tests/runUnitTestsCI.sh
-
-  ios_smoke_tests:
-    executor: mac
-    steps:
-      - install_nix
-      - setup
-      - decrypt_secrets_if_possible
-      - restore_yarn_cache
-      - run: yarn
-      - save_yarn_cache
-      - restore_workingdir_cache:
-          sdk_version: sdk32
-          platform: ios
-      - run:
-          name: Run iOS smoke tests
-          command: ./scripts/tests/runIOSSmokeTestsCI.sh
-      - save_workingdir_cache:
-          workingdir: /Users/distiller/project/workingdir
-          sdk_version: sdk32
-          platform: ios
-
-  android_smoke_tests:
-    executor: android
-    steps:
-      - setup
-      - setup_nix
-      - restore_yarn_cache
-      - run: yarn
-      - save_yarn_cache
-      - restore_workingdir_cache:
-          sdk_version: sdk32
-          platform: android
-      - run:
-          name: Fetch Android shell app
-          command: SDK_VERSION=sdk32 ./scripts/android/fetchRemoteAndroidTarball.sh
-      - run:
-          name: Run Android smoke tests
-          command: |
-            nix-shell nix/android-shell.nix --pure \
-              --run "./scripts/tests/runAndroidSmokeTestsCI.sh"
-      - save_workingdir_cache:
-          workingdir: ~/turtle/workingdir
-          sdk_version: sdk32
-          platform: android
+      - run: ./scripts/testUnitCI.sh
 
 workflows:
-  checks:
+  version: 2
+  build_and_test:
     jobs:
       - build
-  tests:
-    jobs:
-      - unit_tests
-      - ios_smoke_tests
-      - android_smoke_tests
+      - test

.envrc

@@ -15,10 +15,4 @@ export NODE_ENV=development
 
 export EXPO_TURTLE_TOOLS_DIR="$(pwd)/tools"
 
-if [ -z "$EXPO_UNIVERSE_DIR" ]; then
-  PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
-fi
-
-if [[ $CI = "true" ]]; then
-  PATH_add $EXPO_TURTLE_TOOLS_DIR/ci/bin
-fi
+PATH_add $EXPO_TURTLE_TOOLS_DIR/bin

.gitignore

@@ -1,12 +1,10 @@
 !/src/node_modules
 
-node_modules
-yarn-error.log
-
 /build/
 /workingdir/*
 /artifacts/*
 shell.apk
+node_modules
 .secrets*
 
 *.swp

.nvmrc

@@ -1 +0,0 @@
-v8.15.1

.secrets.local-common

@@ -2,6 +2,7 @@ LOGGER_LEVEL="debug"
 API_PROTOCOL="http"
 API_HOSTNAME="localhost"
 API_PORT="3000"
+LOGGLY_TOKEN=""
 SENTRY_DSN=""
 ENVIRONMENT="development"
 REDIS_URL="redis://localhost:6379/0"

CHANGELOG.md

@@ -5,48 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
-### Changed
-- Updated iOS shell app for SDK 32 to fix AppAuth issue
-
-## [0.6.0] - 2019-04-09
-### Changed
-- Removed Branch module from Android shell apps for all SDK versions
-
-## [0.5.15] - 2019-04-03
-### Added
-- Turtle smoke tests are now being executed on CircleCI from the context of this repository.
-### Changed
-- Updated iOS shell app for SDK 32 (fixes [this loading issue for slow connections](https://github.com/expo/expo/issues/3574)).
-- Minor infra changes (doesn't affect users).
-- Error logs format - error objects are now passed in an object being passed as the first argument to logger function.
-### Removed
-- Sending logs to loggly.
-
-## [0.5.14] - 2019-03-22
-### Added
-- `--config` parameter for turtle-cli commands, which allows to specify a custom path to app.json file
-### Changed
-- Updated iOS shell app for SDK 32 (fixes [this notifications issue](https://github.com/expo/expo/issues/3663)).
-
-## [0.5.13] - 2019-03-12
-### Added
-- Android SDK version support validation.
-### Changed
-- iOS Enterprise builds don't compile bitcode now.
-
-## [0.5.12] - 2019-02-26
-### Added
-- Google Cloud logger (as a bunyan stream).
-### Changed
-- Android builder passes the manifest from a build job message if it's available.
-
-## [0.5.11] - 2019-02-22
-### Added
-- Builder for iOS client apps.
 
 ## [0.5.10] - 2019-02-19
 ### Added
-- Register Turtle version in Redis.
+- Register Turtle version in Redis
 ### Changed
 - Update Android shell app for SDK 32 to fix [update](https://github.com/expo/expo/issues/3504) and [back button](https://github.com/expo/expo/issues/1786) issues.
 

Dockerfile

@@ -102,6 +102,13 @@ ENV PATH ${GRADLE_HOME}/bin:$PATH
 # Install Gulp
 RUN npm install -g gulp-cli
 
+# Install Git
+RUN echo 'deb http://http.debian.net/debian wheezy-backports main' >> /etc/apt/sources.list && \
+  apt-get update && \
+  apt-get -t wheezy-backports install git-core && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
 ADD . /app
 
 RUN for SDK_VERSION in `ls /app/workingdir/android/`; do \

jest/integration-config.json

@@ -9,6 +9,6 @@
     "^.+\\.ts$": "ts-jest"
   },
   "testRegex": "(test|spec)\\.ts$",
-  "moduleFileExtensions": ["ts", "js", "json"],
+  "moduleFileExtensions": ["ts", "js"],
   "verbose": true
 }

jest/smoke-config.json

@@ -9,6 +9,6 @@
     "^.+\\.ts$": "ts-jest"
   },
   "testMatch": [ "**/__smoke__/*.test.ts" ],
-  "moduleFileExtensions": ["ts", "js", "json"],
+  "moduleFileExtensions": ["ts", "js"],
   "verbose": true
 }

jest/unit-config.json

@@ -9,6 +9,6 @@
     "^.+\\.ts$": "ts-jest"
   },
   "testMatch": [ "**/__tests__/*.ts" ],
-  "moduleFileExtensions": ["ts", "js", "json"],
+  "moduleFileExtensions": ["ts", "js"],
   "verbose": true
 }

k8s/android/turtle-deployment.template.yml

@@ -50,6 +50,8 @@ spec:
             value: exp-shell-app-assets
           - name: AWS_S3_REGION
             value: us-west-1
+          - name: LOGGLY_SUBDOMAIN
+            value: exponent
           - name: TURTLE_TEMPORARY_FILES_DIR
             value: /tmp/turtle
           # vars from k8s secrets
@@ -98,6 +100,11 @@ spec:
               secretKeyRef:
                 name: turtle-js-config
                 key: aws_cloudwatch_region
+          - name: LOGGLY_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: turtle-js-config
+                key: loggly_token
           - name: REDIS_URL
             valueFrom:
               secretKeyRef:

nix/all-packages.nix

@@ -1,5 +0,0 @@
-self: super:
-
-{
-  nodejs = super.nodejs-8_x;
-}

nix/android-shell.nix

@@ -1,50 +0,0 @@
-with (import ./default.nix {});
-
-let
-
-   ndk = androidenv.androidndk_17c.override { fullNDK = true; };
-   ndk_root = "${ndk}/libexec/${ndk.name}";
-   sdk_path = if stdenv.isDarwin
-     then "/Library/Android/sdk"
-     else "/Android/Sdk"; # intentional Capital S
-
-in
-
-mkShell rec {
-
-  LANG="en_US.UTF-8";
-  JAVA_HOME=openjdk8;
-
-  ANDROID_SDK_ROOT=builtins.getEnv("HOME") + sdk_path;
-  ANDROID_NDK_ROOT="${ANDROID_SDK_ROOT}/ndk-bundle";
-  ANDROID_HOME=ANDROID_SDK_ROOT;
-  ANDROID_NDK=ANDROID_NDK_ROOT;
-
-  nativeBuildInputs = [
-    awscli
-    curl
-    git
-    nodejs-8_x
-    openjdk8
-    yarn
-  ];
-
-  passthru = { inherit ndk; };
-
-  shellHook = ''
-    ${./install-ndk-17c.sh} ${ndk} ${ndk_root}
-    yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --licenses --sdk_root="$ANDROID_SDK_ROOT"
-    yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --sdk_root="$ANDROID_SDK_ROOT" "platforms;android-28" | grep -v =
-    yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --sdk_root="$ANDROID_SDK_ROOT" "build-tools;28.0.3" | grep -v =
-    ${lib.optionalString stdenv.isLinux ''
-      for dep in lib lib64; do
-        if [ -L /$dep ] || [ ! -e /$dep ]; then
-          echo "Creating /$dep"
-          ln -s ${stdenv.cc.libc}/$dep /$dep
-        else
-          echo "Using existing /$dep"
-        fi
-      done
-    ''}
-  '';
-}

nix/default.nix

@@ -1,19 +0,0 @@
-let
-  pin = builtins.fromJSON (builtins.readFile ./nixpkgs.json);
-
-  nixpkgs = builtins.fetchTarball {
-    url = "https://github.com/NixOS/nixpkgs-channels/archive/${pin.rev}.tar.gz";
-    inherit (pin) sha256;
-  };
-
-in
-
-{ ... } @ args:
-
-  import nixpkgs (args // {
-    config = {
-      allowUnfree = true;
-      android_sdk.accept_license = true;
-    };
-    overlays = [ (import ./all-packages.nix) ];
-  })

nix/install-ndk-17c.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-scriptdir="$(dirname ${BASH_SOURCE[0]})"
-shellnix="$scriptdir/android-shell.nix"
-
-ndk=${1:-$(nix-build "$shellnix" -A ndk --no-out-link)}
-ndkRoot=${2:-$(nix eval --file "$shellnix" --raw ANDROID_NDK_ROOT)}
-
-mkdir -p "$ANDROID_NDK_ROOT" # Create full path if necessary
-
-# Replace any existing link or directory with link
-rm -r "$ANDROID_NDK_ROOT"
-ln -sn "$ndkRoot" "$ANDROID_NDK_ROOT"

nix/nixpkgs.json

@@ -1,7 +0,0 @@
-{
-  "url": "https://github.com/NixOS/nixpkgs-channels.git",
-  "rev": "80738ed9dc0ce48d7796baed5364eef8072c794d",
-  "date": "2018-11-17T11:18:10+01:00",
-  "sha256": "0anmvr6b47gbbyl9v2fn86mfkcwgpbd5lf0yf3drgm8pbv57c1dc",
-  "fetchSubmodules": false
-}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "turtle-cli",
-  "version": "0.6.0",
+  "version": "0.5.10",
   "description": "The command-line tool for building standalone Expo apps",
   "preferGlobal": true,
   "scripts": {
@@ -10,7 +10,6 @@
     "watch": "EXPENV_OVERRIDES=local-common,local,local-queue expenv gulp watch:source",
     "bin:turtle": "./bin/turtle.js",
     "lint": "tslint -c tslint.json 'src/**/*.ts'",
-    "lint:fix": "yarn lint --fix",
     "prepare": "yarn build",
     "build": "gulp build:server",
     "build:production": "./scripts/build.sh",
@@ -27,7 +26,7 @@
     "test:integration": "yarn test --config jest/integration-config.json",
     "test:unit": "yarn test --config jest/unit-config.json",
     "test:smoke": "yarn test --config jest/smoke-config.json --forceExit",
-    "test:smoke:android": "PLATFORM=android yarn test:smoke --testNamePattern=android",
+    "test:smoke:android": "yarn test:smoke --testNamePattern=android",
     "test:smoke:ios": "PLATFORM=ios yarn test:smoke --testNamePattern=iOS",
     "release": "./scripts/release.sh",
     "release:beta": "./scripts/releaseBeta.sh",
@@ -39,17 +38,6 @@
     "set-priorities:staging": "expenv babel-node scripts/setConfigurations.js",
     "set-priorities:production": "EXPENV_OVERRIDES=prod expenv babel-node scripts/setConfigurations.js"
   },
-  "husky": {
-    "hooks": {
-      "pre-commit": "lint-staged"
-    }
-  },
-  "lint-staged": {
-    "src/**/*.ts": [
-      "yarn lint:fix",
-      "git add"
-    ]
-  },
   "bin": {
     "turtle": "./bin/turtle.js"
   },
@@ -100,9 +88,7 @@
     "gulp-plumber": "^1.2.0",
     "gulp-sourcemaps": "^2.6.4",
     "gulp-typescript": "^4.0.2",
-    "husky": "^1.3.1",
     "jest": "^23.6.0",
-    "lint-staged": "^8.1.5",
     "nock": "^10.0.1",
     "nodemon": "^1.18.7",
     "release-it": "^10.0.0",
@@ -110,16 +96,15 @@
     "source-map-loader": "^0.2.3",
     "ts-jest": "^23.10.4",
     "tslint": "^5.10.0",
-    "tslint-eslint-rules": "^5.4.0",
     "typescript": "^3.0.1"
   },
   "dependencies": {
     "@expo/spawn-async": "^1.4.0",
-    "@google-cloud/logging-bunyan": "^0.9.5",
     "async-retry": "^1.2.1",
     "aws-sdk": "^2.226.1",
     "bunyan": "^1.8.12",
     "bunyan-debug-stream": "^1.0.8",
+    "bunyan-loggly": "^1.3.4",
     "bunyan-sentry-stream": "^1.2.1",
     "commander": "^2.15.1",
     "copy": "^0.3.1",
@@ -138,9 +123,9 @@
     "tar": "^4.4.6",
     "uuid": "^3.2.1",
     "which": "^1.3.1",
-    "xdl": "53.1.6-alpha.2"
+    "xdl": "53.1.1"
   },
   "optionalDependencies": {
-    "@expo/traveling-fastlane-darwin": "^1.8.1"
+    "@expo/traveling-fastlane-darwin": "^1.8.0"
   }
 }

scripts/android/fetchRemoteAndroidTarball.sh

@@ -1,4 +1,24 @@
 #!/usr/bin/env bash
 
+set -xeo pipefail
+
 ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
-$ROOT_DIR/scripts/fetchRemoteTarballs.sh android
+ARTIFACTS_DIR="$ROOT_DIR/artifacts"
+WORKING_DIR="$ROOT_DIR/workingdir/android"
+ANDROID_SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/android"
+
+mkdir -p $ARTIFACTS_DIR
+
+for SDK_VERSION in `ls $ANDROID_SHELL_TARBALLS_DIRECTORY`; do
+  mkdir -p $WORKING_DIR/$SDK_VERSION
+  TARBALL_URI=$(head -n 1 $ANDROID_SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
+  if command -v run  >/dev/null 2>&1 ; then
+    run expo.s3env --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --command retry5 aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
+  else
+    # outside of universe aws credentials might not be set
+    if ! aws iam get-user >/dev/null 2>&1 ; then aws configure ; fi
+    aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
+  fi
+  tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
+  rm -rf $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
+done

scripts/fetchRemoteTarballs.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-set -xeo pipefail
-
-PLATFORM=$1
-
-ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
-ARTIFACTS_DIR="$ROOT_DIR/artifacts"
-WORKING_DIR="$ROOT_DIR/workingdir/$PLATFORM"
-SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/$PLATFORM"
-
-mkdir -p $ARTIFACTS_DIR
-
-convert_s3_to_https() {
-  BUCKET=`echo $1 | sed 's/s3:\/\/\([^/]*\).*/\1/'`
-  KEY=`echo $1 | sed 's/s3:\/\/[^/]*\/\(.*\)/\1/'`
-  echo "https://s3.amazonaws.com/$BUCKET/$KEY"
-}
-
-download_shell_app() {
-  SDK_VERSION=$1
-  mkdir -p $WORKING_DIR/$SDK_VERSION
-  TARBALL_S3_URL=$(head -n 1 $SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
-  TARBALL_HTTPS_URL=$(convert_s3_to_https $TARBALL_S3_URL)
-  curl -s $TARBALL_HTTPS_URL -o $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
-  tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
-  rm -rf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
-}
-
-if [[ -z "$SDK_VERSION" ]]; then
-  for SDK_VERSION in `ls $SHELL_TARBALLS_DIRECTORY`; do
-    download_shell_app $SDK_VERSION
-  done
-else
-  if [ ! -d "$WORKING_DIR/$SDK_VERSION" ]; then
-    download_shell_app $SDK_VERSION
-  fi
-fi

scripts/ios/fetchRemoteIosTarball.sh

@@ -1,4 +1,24 @@
 #!/usr/bin/env bash
 
+set -xeo pipefail
+
 ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
-$ROOT_DIR/scripts/fetchRemoteTarballs.sh ios
+ARTIFACTS_DIR="$ROOT_DIR/artifacts"
+WORKING_DIR="$ROOT_DIR/workingdir/ios"
+IOS_SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/ios"
+
+mkdir -p $ARTIFACTS_DIR
+
+for SDK_VERSION in `ls $IOS_SHELL_TARBALLS_DIRECTORY`; do
+  mkdir -p $WORKING_DIR/$SDK_VERSION
+  TARBALL_URI=$(head -n 1 $IOS_SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
+  if command -v run  >/dev/null 2>&1 ; then
+    run expo.s3env --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --command retry5 aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
+  else
+    # outside of universe aws credentials might not be set
+    if ! aws iam get-user >/dev/null 2>&1 ; then aws configure ; fi
+    aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
+  fi
+  tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
+  rm -rf $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
+done

scripts/testAndroidCI.sh

@@ -1,25 +1,37 @@
 #!/usr/bin/env bash
 
-set -ueo pipefail
-
 export NODE_ENV=test
 
 export ENVIRONMENT="development"
 export AWS_ACCESS_KEY_ID="aws_id"
 export AWS_SECRET_ACCESS_KEY="secret"
-export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in.fifo"
-export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in_priority.fifo"
-export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in.fifo"
-export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in_priority.fifo"
-export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_out.fifo"
+export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_ios_in.fifo"
+export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_ios_in.fifo"
+export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_android_in.fifo"
+export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_android_in.fifo"
+export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_out.fifo"
 export AWS_SQS_REGION="aws.west2"
 export AWS_S3_BUCKET="exp-shell-app-assets"
 export AWS_S3_REGION="us-west-1"
 export AWS_CLOUDWATCH_REGION="us-west-1"
 export API_PROTOCOL="http"
-export API_HOSTNAME="staging.expo.io"
-export API_PORT="443"
+export API_HOSTNAME="localhost"
+export API_PORT="3000"
 export REDIS_URL="noop"
 export REDIS_CONFIG_URL="noop"
 
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
+$ROOT_DIR/scripts/android/fetchRemoteAndroidTarball.sh
+
 yarn test:smoke:android
+smoke_result=$?
+
+yarn test:unit
+unit_result=$?
+
+[[ $smoke_result != 0 || $unit_result != 0 ]] && \
+	echo && \
+	echo "Tests failed." && \
+	exit -1
+
+exit 0

scripts/testIosCI.sh

@@ -1,15 +1,13 @@
 #!/usr/bin/env bash
 
-set -ueo pipefail
-
 export NODE_ENV=test
 
 export ENVIRONMENT="development"
-export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in.fifo"
-export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in_priority.fifo"
-export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in.fifo"
-export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in_priority.fifo"
-export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_out.fifo"
+export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_ios_in.fifo"
+export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_android_in.fifo"
+export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_ios_in_priority.fifo"
+export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_android_in_priority.fifo"
+export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_out.fifo"
 export AWS_SQS_REGION="aws.west2"
 export AWS_S3_BUCKET="exp-shell-app-assets"
 export AWS_S3_REGION="us-west-1"
@@ -20,7 +18,18 @@ export API_PORT="443"
 export REDIS_URL="noop"
 export REDIS_CONFIG_URL="noop"
 
-ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
-SDK_VERSION=sdk32 $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
+$ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
 
 yarn test:smoke:ios
+smoke_result=$?
+
+yarn test:unit
+unit_result=$?
+
+[[ $smoke_result != 0 || $unit_result != 0 ]] && \
+	echo && \
+	echo "Tests failed." && \
+	exit -1
+
+exit 0

scripts/testUnitCI.sh

@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
 
-set -ueo pipefail
-
 export NODE_ENV=test
 
 export ENVIRONMENT="development"
@@ -23,3 +21,6 @@ export REDIS_URL="noop"
 export REDIS_CONFIG_URL="noop"
 
 yarn test:unit
+unit_result=$?
+
+exit $unit_result

shellTarballs/android/sdk26

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
+s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz

shellTarballs/android/sdk27

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
+s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz

shellTarballs/android/sdk28

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
+s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz

shellTarballs/android/sdk29

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
+s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz

shellTarballs/android/sdk30

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
+s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz

shellTarballs/android/sdk31

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-89b2d24c86737d5a489c6a18467e31b2f9ce9005.tar.gz
+s3://exp-artifacts/android-shell-builder-5c339ff7bab6f2161d378cc70e1bf04d32ad98dd.tar.gz

shellTarballs/android/sdk32

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-0cab68670fc50ce1dc16b7d86ca7976870062c39.tar.gz
+s3://exp-artifacts/android-shell-builder-a4bd3bd26b1b31bdfa66c39b6b4d0d1d138c32fc.tar.gz

shellTarballs/ios/sdk32

@@ -1 +1 @@
-s3://exp-artifacts/ios-shell-builder-sdk-latest-dc981c7110b7d508de8e9c352064dc3d745c9f72.tar.gz
+s3://exp-artifacts/ios-shell-builder-sdk-latest-f6e1921542b2e26b4b9221dd2eddb74ea5cfdf74.tar.gz

src/aws/cloudwatch/index.ts

@@ -179,7 +179,7 @@ function _pushMetrics(metrics: IMetricsChunk[]) {
   metrics.forEach((metricsChunk) => {
     cloudWatch.putMetricData(metricsChunk, (err) => {
       if (err) {
-        logger.warn({ err }, '[cloudwatch]');
+        logger.warn('[cloudwatch]', err);
       }
     });
   });

src/aws/cloudwatch/reducers.ts

@@ -7,7 +7,7 @@ export function sum(metrics: IMetric[]) {
   const reduced = metrics.reduce((acc: IMetric | null, i: IMetric) => {
     let result = acc;
     if (!result) {
-      result = { ...i };
+      result = {...i};
     } else {
       result.Value += i.Value;
     }

src/aws/sqs.ts

@@ -36,7 +36,7 @@ export async function receiveMessage(priority: string) {
         }
       })
       .catch((err) => {
-        logger.error({ err }, 'Receive error');
+        logger.error('Receive error', err);
         throw err;
       });
   });
@@ -68,7 +68,7 @@ export function changeMessageVisibilityRecurring(priority: string, receiptHandle
   return setInterval(() => {
     if (getCurrentJobId() === jobId) {
       changeMessageVisibility(priority, receiptHandle).catch((err) => {
-        logger.warn({ err }, 'Error at change msg visibility');
+        logger.warn('Error at change msg visibility', err);
       });
     }
   }, (VISIBILITY_TIMEOUT_SEC * 1000) / 3);
@@ -86,6 +86,6 @@ export async function sendMessage(jobId: string, status: JOB_UPDATE_TYPE, data =
       return await sqs.sendMessage(params).promise();
     });
   } catch (err) {
-    logger.error({ err }, `Error sending SQS message: ${JSON.stringify(params)}}`);
+    logger.error(`Error sending SQS message: ${JSON.stringify(params)}}`, err);
   }
 }

src/bin/commands/build/index.ts

@@ -20,8 +20,7 @@ function setCommonCommandOptions(cmd: any) {
     .option(
       '--release-channel <channel-name>',
       'pull from specified release channel (default: default)',
-    )
-    .option('-c --config <config-file>', 'specify a path to app.json');
+    );
 }
 
 export default {

src/bin/commands/setup.ts

@@ -33,7 +33,8 @@ async function setupAction(program: any, cmd: any, platform: string, os?: string
     await setup(platform, sdkVersion);
     l.info('it\'s all set!');
   } catch (err) {
-    logger.error({ err }, `Failed to setup environment for ${platform} builds`);
+    logger.error(`Failed to setup environment for ${platform} builds`);
+    logger.error(err.stack);
     if (err instanceof ErrorWithCommandHelp) {
       cmd.help();
     } else if (err instanceof ErrorWithProgramHelp) {

src/bin/index.ts

@@ -17,16 +17,16 @@ Command.prototype.asyncAction = function asyncAction(asyncFn: (...asyncFnArgs: a
   return this.action(async (...args) => {
     try {
       await checkForUpdateAsync();
-    } catch (err) {
-      logger.warn({ err }, 'Failed to check for turtle-cli update.');
+    } catch (e) {
+      logger.warn('Failed to check for turtle-cli update.');
     }
     return await asyncFn(...args);
   });
 };
 
 export function run(programName: string) {
-  runAsync(programName).catch((err) => {
-    logger.error({ err }, 'Uncaught Error');
+  runAsync(programName).catch((e) => {
+    logger.error('Uncaught Error', e);
     process.exit(1);
   });
 }

src/bin/setup/utils/toolsDetector.ts

@@ -26,9 +26,9 @@ export async function ensureToolsAreInstalled(tools: IToolDefinition[]) {
     } catch (err) {
       isAnyToolMissing = true;
       if (!testFn) {
-        l.error({ err }, `${command} is missing in your $PATH`);
+        l.error(`${command} is missing in your $PATH`);
       }
-      l.error({ err }, missingDescription);
+      l.error(missingDescription);
     }
   }
   if (isAnyToolMissing) {

src/bin/utils/builder.ts

@@ -60,7 +60,7 @@ export function createBuilderAction({
         publicUrl: cmd.publicUrl,
       };
 
-      const appJSON = await ProjectUtils.loadAppJSON(projectDirArg, cmd.config);
+      const appJSON = await ProjectUtils.loadAppJSON(projectDirArg);
       const sdkVersion = _.get(appJSON, 'expo.sdkVersion');
       await setup(platform, sdkVersion);
       const credentials = await prepareCredentials(cmd);
@@ -72,7 +72,8 @@ export function createBuilderAction({
       const job = await sanitizeJob(rawJob);
       await builder(job);
     } catch (err) {
-      logger.error({ err }, 'Failed to build standalone app');
+      logger.error(`Failed to build standalone app: ${err.message}`);
+      logger.error(err.stack);
       if (err instanceof ErrorWithCommandHelp) {
         command.help();
       } else if (err instanceof ErrorWithProgramHelp) {

src/bin/utils/project.ts

@@ -1,17 +1,12 @@
 import fs from 'fs-extra';
 import path from 'path';
 
-export async function loadAppJSON(projectDirArg: string, config: string | null) {
-  let appJSONPath;
-  if (config) {
-    appJSONPath = resolveAbsoluteDir(config);
-  } else {
-    const projectDir = resolveAbsoluteDir(projectDirArg);
-    appJSONPath = path.join(projectDir, 'app.json');
-  }
+export async function loadAppJSON(projectDirArg: string) {
+  const projectDir = resolveAbsoluteDir(projectDirArg);
+  const appJSONPath = path.join(projectDir, 'app.json');
   const appJSONExists = await fs.pathExists(appJSONPath);
   if (!appJSONExists) {
-    throw new Error(`Couldn't find app.json.`);
+    throw new Error(`Couldn't find app.json in ${projectDir} directory.`);
   } else {
     return require(appJSONPath);
   }

src/builders/android.ts

@@ -10,13 +10,11 @@ import { formatShellAppDirectory } from 'turtle/builders/utils/android/workingdi
 import * as commonUtils from 'turtle/builders/utils/common';
 import * as imageHelpers from 'turtle/builders/utils/image';
 import { uploadBuildToS3 } from 'turtle/builders/utils/uploader';
-import { ensureCanBuildSdkVersion } from 'turtle/builders/utils/version';
 import config from 'turtle/config';
 import { IAndroidCredentials, IJob, IJobResult } from 'turtle/job';
 import logger from 'turtle/logger';
 
 export default async function buildAndroid(jobData: IJob): Promise<IJobResult> {
-  await ensureCanBuildSdkVersion(jobData);
   const credentials = await getOrCreateCredentials(jobData);
   const apkFilePath = await runShellAppBuilder(jobData, credentials);
   const randomHex = uuidv4().replace(/-/g, '');
@@ -62,21 +60,20 @@ async function runShellAppBuilder(
 
   const outputFilePath = path.join(temporaryFilesRoot, `shell-signed-${jobData.id}.apk`);
 
-  const { config: jobConfig, manifest, sdkVersion: sdkVersionFromJob } = jobData;
-  const sdkVersion = _.get(manifest, 'sdkVersion', sdkVersionFromJob);
+  const { manifest: { sdkVersion: sdkVersionFromManifest = null } = {}, sdkVersion: sdkVersionFromJob } = jobData;
+  const sdkVersion = sdkVersionFromJob || sdkVersionFromManifest;
   const workingDir = formatShellAppDirectory({ sdkVersion });
 
   try {
     await AndroidShellApp.createAndroidShellAppAsync({
       url: commonUtils.getExperienceUrl(jobData),
-      sdkVersion,
+      sdkVersion: _.get(jobData, 'manifest.sdkVersion') || jobData.sdkVersion,
       keystore: tempKeystorePath,
-      manifest,
       alias: credentials.keystoreAlias,
       keystorePassword: credentials.keystorePassword,
       keyPassword: credentials.keyPassword,
       privateConfigFile: tempShellAppConfigPath,
-      releaseChannel: jobConfig.releaseChannel,
+      releaseChannel: jobData.config.releaseChannel,
       workingDir,
       outputFile: outputFilePath,
     });

src/builders/ios/index.ts

@@ -1,5 +1,8 @@
+import path from 'path';
+
 import fs from 'fs-extra';
 import pick from 'lodash/pick';
+import { ExponentTools } from 'xdl';
 
 import buildArchive from 'turtle/builders/ios/archive';
 import { createBuilderContext, IContext } from 'turtle/builders/ios/context';
@@ -7,13 +10,14 @@ import buildSimulator from 'turtle/builders/ios/simulator';
 import { logErrorOnce } from 'turtle/builders/utils/common';
 import prepareAdHocBuildCredentials from 'turtle/builders/utils/ios/adhocBuild';
 import { uploadBuildToS3 } from 'turtle/builders/utils/uploader';
-import { ensureCanBuildSdkVersion } from 'turtle/builders/utils/version';
 import config from 'turtle/config';
-import { IOS } from 'turtle/constants/index';
+import { IOS, PLATFORMS } from 'turtle/constants/index';
 import { IJob, IJobResult } from 'turtle/job';
 
 const { BUILD_TYPES } = IOS;
 
+let SUPPORTED_SDK_VERSIONS: number[];
+
 export default async function iosBuilder(job: IJob): Promise<IJobResult> {
   if (job.config.buildType !== BUILD_TYPES.CLIENT) {
     await ensureCanBuildSdkVersion(job);
@@ -64,3 +68,25 @@ async function cleanup(ctx: IContext) {
 
 const getTemporaryDirs = (ctx: IContext) =>
   Object.values(pick(ctx, ['appDir', 'provisioningProfileDir']));
+
+async function ensureCanBuildSdkVersion(job: IJob) {
+  if (config.builder.useLocalWorkingDir) {
+    return;
+  }
+
+  if (!SUPPORTED_SDK_VERSIONS) {
+    SUPPORTED_SDK_VERSIONS = await findSupportedSdkVersions();
+  }
+
+  const targetMajorSdkVersion = ExponentTools.parseSdkMajorVersion(job.sdkVersion || job.manifest.sdkVersion);
+  if (!SUPPORTED_SDK_VERSIONS.includes(targetMajorSdkVersion)) {
+    throw new Error(`Unsupported SDK Version!`);
+  }
+}
+
+async function findSupportedSdkVersions(): Promise<number[]> {
+  const SDK_DIR_PREFIX = 'sdk';
+  const files = await fs.readdir(path.join(config.directories.workingDir, PLATFORMS.IOS));
+  const sdks = files.filter((file) => file.startsWith(SDK_DIR_PREFIX));
+  return sdks.map((sdk) => parseInt(sdk.substr(SDK_DIR_PREFIX.length), 10));
+}

src/builders/utils/ios/keychain.ts

@@ -16,7 +16,7 @@ export async function create(ctx: IContext): Promise<IKeychain> {
     l.info('done creating keychain');
     return keychainInfo;
   } catch (err) {
-    l.error({ err }, 'unable to create keychain');
+    l.error('unable to create keychain');
     throw err;
   }
 }
@@ -32,7 +32,7 @@ export async function remove(ctx: IContext, keychainPath: string) {
     l.info('done deleting keychain');
     return keychainInfo;
   } catch (err) {
-    l.error({ err }, 'unable to delete keychain');
+    l.error('unable to delete keychain');
     throw err;
   }
 }
@@ -57,7 +57,7 @@ export async function importCert(
     await IosKeychain.importIntoKeychain({ keychainPath, certPath, certPassword });
     l.info('done importing distribution certificate into keychain');
   } catch (err) {
-    l.error({ err }, 'unable to import distribution certificate into keychain');
+    l.error('unable to import distribution certificate into keychain');
     throw err;
   }
 }

src/builders/utils/version.ts

@@ -1,30 +0,0 @@
-import fs from 'fs-extra';
-import path from 'path';
-import { ExponentTools } from 'xdl';
-
-import config from 'turtle/config';
-import { IJob } from 'turtle/job';
-
-let SUPPORTED_SDK_VERSIONS: number[];
-
-export async function ensureCanBuildSdkVersion(job: IJob) {
-  if (config.builder.useLocalWorkingDir) {
-    return;
-  }
-
-  if (!SUPPORTED_SDK_VERSIONS) {
-    SUPPORTED_SDK_VERSIONS = await findSupportedSdkVersions(job.platform);
-  }
-
-  const targetMajorSdkVersion = ExponentTools.parseSdkMajorVersion(job.sdkVersion || job.manifest.sdkVersion);
-  if (!SUPPORTED_SDK_VERSIONS.includes(targetMajorSdkVersion)) {
-    throw new Error(`Unsupported SDK Version!`);
-  }
-}
-
-async function findSupportedSdkVersions(platform: 'android' | 'ios'): Promise<number[]> {
-  const SDK_DIR_PREFIX = 'sdk';
-  const files = await fs.readdir(path.join(config.directories.workingDir, platform));
-  const sdks = files.filter((file) => file.startsWith(SDK_DIR_PREFIX));
-  return sdks.map((sdk) => parseInt(sdk.substr(SDK_DIR_PREFIX.length), 10));
-}

src/config.ts

@@ -47,9 +47,11 @@ export default {
     client: {
       level: env('CLIENT_LOGGER_LEVEL', 'info'),
     },
-  },
-  google: {
-    credentials: env('GOOGLE_APPLICATION_CREDENTIALS', ''),
+    loggly: {
+      token: env('LOGGLY_TOKEN', ''),
+      subdomain: env('LOGGLY_SUBDOMAIN', ''),
+      buffer: envNum('LOGGLY_BUFFER', 100),
+    },
   },
   sentry: {
     dsn: env('SENTRY_DSN', ''),

src/jobManager.ts

@@ -47,7 +47,7 @@ export async function getJob() {
         return job;
       }
     } catch (err) {
-      logger.error({ err }, 'Error at receiving messages');
+      logger.error('Error at receiving messages', err);
     }
   }
 }
@@ -160,6 +160,6 @@ async function deleteMessage(priority: string, receiptHandle: string) {
     setCurrentJobId(null);
     await sqs.deleteMessage(priority, receiptHandle);
   } catch (err) {
-    logger.error({ err }, 'Error at deleting msg');
+    logger.error('Error at deleting msg', err);
   }
 }

src/logger/hackyLogglyStream.ts

@@ -0,0 +1,48 @@
+import EventEmitter from 'events';
+
+import BunyanLoggly from 'bunyan-loggly';
+
+import config from 'turtle/config';
+import * as constants from 'turtle/constants/logger';
+import { IJob } from 'turtle/job';
+
+export interface ILogglyConfig {
+  token: string;
+  subdomain: string;
+  tags: string[];
+}
+
+interface ILogglyExtraFields {
+  jobID: string;
+  experienceName: string;
+}
+
+export default class HackyLogglyStream extends EventEmitter {
+  private logglyStream: BunyanLoggly;
+  private extraFields: ILogglyExtraFields | null;
+
+  constructor(logglyConfig: ILogglyConfig) {
+    super();
+    this.logglyStream = new BunyanLoggly(logglyConfig, config.logger.loggly.buffer);
+    this.extraFields = null;
+  }
+  public init(job: IJob) {
+    this.extraFields = {
+      jobID: job.id,
+      experienceName: job.experienceName,
+    };
+  }
+  public cleanup() {
+    this.extraFields = null;
+  }
+  public write(rec: any) {
+    const { msg: message, hostname: host, level, ...other } = rec;
+    this.logglyStream.write({
+      message,
+      level: constants.LEVELS_MAPPING[level],
+      host,
+      ...other,
+      data: this.extraFields || {},
+    });
+  }
+}

src/logger/index.ts

@@ -6,14 +6,10 @@ import { Client as RavenClient } from 'raven';
 import config from 'turtle/config';
 import * as constants from 'turtle/constants/logger';
 import { IJob } from 'turtle/job';
+import HackyLogglyStream from 'turtle/logger/hackyLogglyStream';
 import S3Stream from 'turtle/logger/s3Stream';
 import { isOffline } from 'turtle/turtleContext';
 
-// type error when using import
-// https://github.com/googleapis/nodejs-logging-bunyan/issues/241
-// tslint:disable-next-line:no-var-requires
-const { LoggingBunyan } = require('@google-cloud/logging-bunyan');
-
 interface IStream {
   stream: any;
   type?: string;
@@ -22,6 +18,7 @@ interface IStream {
 }
 
 export const s3logger = new S3Stream();
+let logglyStream: HackyLogglyStream;
 
 const streams: IStream[] = [];
 
@@ -50,35 +47,45 @@ if (config.sentry.dsn && config.deploymentEnv !== 'development') {
   });
 }
 
-if (config.google.credentials) {
-  const resource = {
-    type: 'generic_node',
-    labels: {
-      node_id: config.hostname,
-      location: '', // default value
-      namespace: '', // default value
-    },
+if (config.logger.loggly.token) {
+  const logglyConfig = {
+    token: config.logger.loggly.token,
+    subdomain: config.logger.loggly.subdomain,
+    // prettier-ignore
+    tags: [
+      'app-shell-apps',
+      `platform.${config.platform}`,
+      `environment.${config.deploymentEnv}`,
+    ],
   };
-  streams.push(new LoggingBunyan({ name: 'turtle', resource }).stream('info'));
+  logglyStream = new HackyLogglyStream(logglyConfig);
+  streams.push({
+    type: 'raw',
+    stream: logglyStream,
+  });
 }
 
 const logger = bunyan.createLogger({
   name: 'turtle',
   level: config.logger.level,
-  platform: config.platform,
-  environment: config.deploymentEnv,
   streams,
 });
 
 logger.withFields = (extraFields: any) => withFields(logger, extraFields);
 
 logger.init = async (job: IJob) => {
+  if (logglyStream) {
+    logglyStream.init(job);
+  }
   return await s3logger.init(job);
 };
 
 logger.cleanup = async () => {
   // a little hacky, but works
   logger.info({ lastBuildLog: true }, 'this is the last log from this build');
+  if (logglyStream) {
+    logglyStream.cleanup();
+  }
   await s3logger.waitForLogger();
 };
 

src/node_modules/bunyan-loggly/index.d.ts

@@ -0,0 +1,13 @@
+export = BunyanLoggly;
+
+declare class BunyanLoggly {
+  constructor(options: BunyanLoggly.IOptions, bufferLength?: number, bufferTimeout?: number, logglyCallback?: any);
+  public write(data: any): void;
+}
+
+declare namespace BunyanLoggly {
+  interface IOptions {
+    token: string;
+    subdomain: string;
+  }
+}

src/server.ts

@@ -9,7 +9,7 @@ import { getRedisClient, RedisClient } from 'turtle/utils/redis';
 
 const REDIS_TURTLE_VERSION_KEY = 'turtle:version';
 
-process.on('unhandledRejection', (err) => logger.error({ err }, 'Unhandled promise rejection'));
+process.on('unhandledRejection', (err) => logger.error('Unhandled promise rejection:', err));
 
 function handleExit() {
   if (checkShouldExit()) {
@@ -39,19 +39,19 @@ async function main() {
     await redis.set(REDIS_TURTLE_VERSION_KEY, turtleVersion);
     logger.info(`Register Turtle version ${turtleVersion} in Redis`);
   } catch (err) {
-    logger.error({ err }, 'Failed to register Turtle version in Redis');
+    logger.error('Failed to register Turtle version in Redis', err);
   }
 
   while (true) {
     try {
       await doJob();
     } catch (err) {
-      logger.error({ err }, 'Failed to do a job');
+      logger.error('Failed to do a job', err);
     }
   }
 }
 
 main()
   .then(() => logger.error('This should never happen...'))
-  .catch((err) => logger.error({ err }, 'Something went terribly wrong'))
+  .catch((err) => logger.error('Something went terribly wrong :(', err))
   .then(() => process.exit(1));

src/utils/priorities.ts

@@ -41,10 +41,6 @@ export function createDefaultConfigurationKey(platform = config.platform) {
 }
 
 export async function getPriorities() {
-  if (config.env === 'test') {
-    return NORMAL_CONFIGURATION;
-  }
-
   try {
     const redis = await getRedisClient(RedisClient.Configuration);
     if (redis.getConfig === undefined) {
@@ -60,23 +56,24 @@ export async function getPriorities() {
       logger.debug(`Using configuration at index ${configurationsIndex} pulled from redis`);
       const { error, value: configValue } = configSchema.validate(configurations[configurationsIndex]);
       if (error) {
-        logger.warn({ err: error });
+        logger.warn(error);
         throw new Error('Received configuration is not valid');
       }
       return configValue;
     } catch (err) {
-      logger.warn({ err });
+      logger.warn(err);
       const defaultConfig = JSON.parse(await redis.get(createDefaultConfigurationKey()));
       const { error, value: defaultConfigValue } = configSchema.validate(defaultConfig);
       if (error) {
-        logger.warn({ err: error });
+        logger.warn(error);
         throw new Error('Received default configuration is not valid');
       }
       logger.warn('Using default configuration pulled from redis');
       return defaultConfigValue;
     }
   } catch (err) {
-    logger.warn({ err }, 'Using configuration chosen locally');
+    logger.warn(err);
+    logger.warn('Using configuration chosen locally');
     if (Math.random() < 0.7) {
       return HIGH_CONFIGURATION;
     } else {

src/utils/redis.ts

@@ -52,7 +52,7 @@ export async function getRedisClient(type = RedisClient.Default) {
     try {
       redisClients[type] = await connect(MILLIS_CONNECTION_TIMEOUT, type);
     } catch (err) {
-      logger.error({ err });
+      logger.error(err);
     }
   }
   return redisClients[type];
@@ -66,7 +66,7 @@ export async function checkIfCancelled(jobId: string) {
     if (config.deploymentEnv === 'development') {
       logger.warn('Did you turn on redis server? Run `yarn start-docker` in server/www');
     }
-    logger.error({ err });
+    logger.error(err);
     return false;
   }
 }

tools/bin/aws

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws "$@"

tools/bin/aws_bash_completer

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws_bash_completer "$@"

tools/bin/aws_completer

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws_completer "$@"

tools/bin/git-crypt

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../nix" git-crypt --command git-crypt "$@"

tools/bin/nix-binstubs

@@ -1,19 +0,0 @@
-#! /usr/bin/env bash
-set -euo pipefail
-
-scriptDir=$(dirname "$0")
-nixDir="$scriptDir/../../nix"
-
-nix build -f $nixDir --no-link "$@"
-
-for package in "$@"; do
-  for b in $(ls $(nix path-info -f $nixDir $package)/bin); do
-    cat << EOF > "$scriptDir/$b"
-#! /usr/bin/env bash
-# This file was generated by $(basename "$0")
-exec nix run -f "\$(dirname "\$0")/../../nix" $package --command $b "\$@"
-EOF
-  chmod +x "$scriptDir/$b"
-  done
-done
-

tools/ci/bin/node

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../../nix" nodejs-8_x --command node "$@"

tools/ci/bin/npm

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../../nix" nodejs-8_x --command npm "$@"

tools/ci/bin/npx

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../../nix" nodejs-8_x --command npx "$@"

tools/ci/bin/yarn

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../../nix" yarn --command yarn "$@"

tools/ci/bin/yarnpkg

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../../nix" yarn --command yarnpkg "$@"

tslint.json

@@ -5,10 +5,7 @@
   "rules": {
     "quotemark": [true, "single", "jsx-double"],
     "object-literal-sort-keys": false,
-    "object-curly-spacing": true,
     "no-namespace": false
   },
-  "rulesDirectory": [
-    "node_modules/tslint-eslint-rules/dist/rules"
-  ]
+  "rulesDirectory": []
 }