Release 0.6.1

# Why

To make it easier to find logs in GCloud Console.

# How

Basically, I copy-pasted `HackyLogglyStream` that we previously used in the project - https://github.com/expo/turtle/blob/de4837f36105c03be9afb4028241cff7459da974/src/logger/hackyLogglyStream.ts

# Test Plan

Tested locally.
<img width="487" alt="Screenshot 2019-04-15 at 15 09 37" src="https://user-images.githubusercontent.com/5256730/56135267-a3d3b200-5f90-11e9-9c96-505380b9f10c.png">

.babelrc

@@ -1,16 +0,0 @@
-{
-  "presets": [
-    ["env", {
-      "targets": {
-        "node": "8.9"
-      }
-    }]
-  ],
-  "plugins": [
-    ["module-resolver", {
-      "alias": {
-        "turtle": "./build"
-      }
-    }]
-  ]
-}

.circleci/config.yml

@@ -1,25 +1,212 @@
-version: 2
+version: 2.1
+
+executors:
+  nix: &nix
+    docker:
+      - image: lnl7/nix:2.2
+    # Symlink to the readline-enabled bash which is the default command of the container
+    shell: /run/current-system/sw/bin/bash -leo pipefail
+    working_directory: ~/turtle
+  mac:
+    macos: # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions
+      xcode: '10.1.0'
+    working_directory: /Users/distiller/project
+    shell: /bin/bash -leo pipefail
+  android:
+    <<: *nix
+    resource_class: xlarge
+
+commands:
+  install_nix:
+    steps:
+      - run:
+          name: Install nix
+          command: curl https://nixos.org/releases/nix/nix-2.2.1/install | bash -s -- --no-daemon
+      - run:
+          name: Add nix-initializing line to .bashrc
+          command: echo ". ~/.nix-profile/etc/profile.d/nix.sh" >> ~/.bashrc
+
+  setup:
+    steps:
+      - run:
+          name: Install common packages
+          command: |
+            nix-env -iA \
+              nixpkgs.binutils \
+              nixpkgs.direnv \
+              nixpkgs.findutils \
+              nixpkgs.gawk \
+              nixpkgs.git \
+              nixpkgs.gnutar \
+              nixpkgs.gzip \
+              nixpkgs.openssh \
+              nixpkgs.python \
+              nixpkgs.xz
+      - run:
+          name: Configure direnv
+          command: |
+            mkdir -p ~/.config/direnv
+            echo -e "[whitelist]\nprefix = [ \"$HOME\" ]" > ~/.config/direnv/config.toml
+            echo 'eval "$(direnv export bash)"' >> ~/.bash_profile
+      - run:
+          name: Configure yarn
+          command: echo '--frozen-lockfile true' >> ~/.yarnrc
+      - checkout
+      - run:
+          name: Configure aws-cli
+          command: |
+              aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
+              aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+
+  setup_nix:
+    steps:
+      - run:
+          name: Add cp and rm stubs
+          command: |
+            cat \<< 'EOF' >> /bin/cp
+            #! /usr/bin/env bash
+            /usr/bin/env cp "$@"
+            EOF
+            chmod +x /bin/cp
+
+            cat \<< 'EOF' >> /bin/rm
+            #! /usr/bin/env bash
+            /usr/bin/env rm "$@"
+            EOF
+            chmod +x /bin/rm
+
+  decrypt_secrets_if_possible:
+    steps:
+      - run:
+          name: Decrypt secrets if possible
+          command: |
+            if [[ -z "$TURTLE_GIT_CRYPT_KEY_BASE64" ]]; then
+              echo 'git-crypt key not present in environment'
+            else
+              git crypt unlock <(echo $TURTLE_GIT_CRYPT_KEY_BASE64 | base64 --decode)
+            fi
+
+  restore_yarn_cache:
+    parameters:
+      workingdir:
+        type: string
+        default: .
+    steps:
+      - restore_cache:
+          key: yarn-v1-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
+
+  save_yarn_cache:
+    parameters:
+      workingdir:
+        type: string
+        default: .
+    steps:
+      - save_cache:
+          key: yarn-v1-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
+          paths:
+            - ~/.cache/yarn # `yarn cache dir`
+
+  restore_workingdir_cache:
+    parameters:
+      platform:
+        type: string
+        default: ios
+      sdk_version:
+        type: string
+        default: sdk32
+    steps:
+      - restore_cache:
+          key: "workingdir-v1-<< parameters.platform >>-<< parameters.sdk_version >>"
+
+  save_workingdir_cache:
+    parameters:
+      workingdir:
+        type: string
+        default: /Users/distiller/project/workingdir
+      platform:
+        type: string
+        default: ios
+      sdk_version:
+        type: string
+        default: sdk32
+    steps:
+      - save_cache:
+          key: "workingdir-v1-<< parameters.platform >>-<< parameters.sdk_version >>"
+          paths:
+            - "<< parameters.workingdir >>/<< parameters.platform >>/<< parameters.sdk_version >>"
 
 jobs:
   build:
-    docker:
-      - image: circleci/node:8.12
+    executor: nix
     steps:
-      - checkout
+      - setup
+      - restore_yarn_cache
       - run: yarn
+      - save_yarn_cache
       - run: yarn lint
       - run: yarn build
-  test:
-    docker:
-      - image: circleci/node:8.12
+
+  unit_tests:
+    executor: nix
     steps:
-      - checkout
+      - setup
+      - restore_yarn_cache
       - run: yarn
-      - run: ./scripts/testUnitCI.sh
+      - save_yarn_cache
+      - run:
+          name: Run unit tests
+          command: ./scripts/tests/runUnitTestsCI.sh
+
+  ios_smoke_tests:
+    executor: mac
+    steps:
+      - install_nix
+      - setup
+      - decrypt_secrets_if_possible
+      - restore_yarn_cache
+      - run: yarn
+      - save_yarn_cache
+      - restore_workingdir_cache:
+          sdk_version: sdk32
+          platform: ios
+      - run:
+          name: Run iOS smoke tests
+          command: ./scripts/tests/runIOSSmokeTestsCI.sh
+      - save_workingdir_cache:
+          workingdir: /Users/distiller/project/workingdir
+          sdk_version: sdk32
+          platform: ios
+
+  android_smoke_tests:
+    executor: android
+    steps:
+      - setup
+      - setup_nix
+      - restore_yarn_cache
+      - run: yarn
+      - save_yarn_cache
+      - restore_workingdir_cache:
+          sdk_version: sdk32
+          platform: android
+      - run:
+          name: Fetch Android shell app
+          command: SDK_VERSION=sdk32 ./scripts/android/fetchRemoteAndroidTarball.sh
+      - run:
+          name: Run Android smoke tests
+          command: |
+            nix-shell nix/android-shell.nix --pure \
+              --run "./scripts/tests/runAndroidSmokeTestsCI.sh"
+      - save_workingdir_cache:
+          workingdir: ~/turtle/workingdir
+          sdk_version: sdk32
+          platform: android
 
 workflows:
-  version: 2
-  build_and_test:
+  checks:
     jobs:
       - build
-      - test
+  tests:
+    jobs:
+      - unit_tests
+      - ios_smoke_tests
+      - android_smoke_tests

.envrc

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 source_local() {
   file=./.envrc.local
@@ -15,4 +15,10 @@ export NODE_ENV=development
 
 export EXPO_TURTLE_TOOLS_DIR="$(pwd)/tools"
 
-PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
+if [ -z "$EXPO_UNIVERSE_DIR" ]; then
+  PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
+fi
+
+if [[ $CI = "true" ]]; then
+  PATH_add $EXPO_TURTLE_TOOLS_DIR/ci/bin
+fi

.gitignore

@@ -1,10 +1,13 @@
 !/src/node_modules
 
+node_modules
+yarn-error.log
+google-cloud-credentials.json
+
 /build/
 /workingdir/*
 /artifacts/*
 shell.apk
-node_modules
 .secrets*
 
 *.swp

.ignore

@@ -0,0 +1 @@
+fixtures/

.nvmrc

@@ -0,0 +1 @@
+v8.15.1

.secrets.local-common

@@ -2,7 +2,6 @@ LOGGER_LEVEL="debug"
 API_PROTOCOL="http"
 API_HOSTNAME="localhost"
 API_PORT="3000"
-LOGGLY_TOKEN=""
 SENTRY_DSN=""
 ENVIRONMENT="development"
 REDIS_URL="redis://localhost:6379/0"

CHANGELOG.md

@@ -5,6 +5,30 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Checking fastlane version before running build (turtle-cli).
+### Changed
+- Updated iOS shell app for SDK 32 to fix AppAuth issue.
+- Babel replaced by ttypescript.
+- Updated xdl to 53.5.3 and @expo/spawn-async to 1.4.2, so that error stack traces should be more informative now.
+### Fixed
+- Brought back `jobID` and `experienceName` fields to GCloud logs.
+### Removed
+- `buidAndroidTarballLocally.sh` script (it has been moved to expo repository - [expo/expo#4022](https://github.com/expo/expo/pull/4022)).
+
+## [0.6.0] - 2019-04-09
+### Changed
+- Removed Branch module from Android shell apps for all SDK versions.
+
+## [0.5.15] - 2019-04-03
+### Added
+- Turtle smoke tests are now being executed on CircleCI from the context of this repository.
+### Changed
+- Updated iOS shell app for SDK 32 (fixes [this loading issue for slow connections](https://github.com/expo/expo/issues/3574)).
+- Minor infra changes (doesn't affect users).
+- Error logs format - error objects are now passed in an object being passed as the first argument to logger function.
+### Removed
+- Sending logs to loggly.
 
 ## [0.5.14] - 2019-03-22
 ### Added

Dockerfile

@@ -99,16 +99,6 @@ RUN rm gradle-4.4-all.zip
 ENV GRADLE_HOME /usr/local/gradle-4.4
 ENV PATH ${GRADLE_HOME}/bin:$PATH
 
-# Install Gulp
-RUN npm install -g gulp-cli
-
-# Install Git
-RUN echo 'deb http://http.debian.net/debian wheezy-backports main' >> /etc/apt/sources.list && \
-  apt-get update && \
-  apt-get -t wheezy-backports install git-core && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
-
 ADD . /app
 
 RUN for SDK_VERSION in `ls /app/workingdir/android/`; do \

gulpfile.js

@@ -1,91 +0,0 @@
-'use strict';
-
-const childProcess = require('child_process');
-const util = require('util');
-const path = require('path');
-
-const gulp = require('gulp');
-const babel = require('gulp-babel');
-const changed = require('gulp-changed');
-const nodemon = require('gulp-nodemon');
-const plumber = require('gulp-plumber');
-const sourcemaps = require('gulp-sourcemaps');
-const ts = require('gulp-typescript');
-const rimraf = require('rimraf');
-
-const rimrafAsync = util.promisify(rimraf);
-
-const tsProject = ts.createProject('tsconfig.json');
-
-gulp.task('clean', async () => {
-  console.log(`Removing build directories...`);
-  await rimrafAsync('build');
-});
-
-gulp.task('build:server', () => {
-  console.log(`Compiling the Node.js server JavaScript...`);
-  return tsProject
-    .src()
-    .pipe(plumber())
-    .pipe(changed('build'))
-    .pipe(sourcemaps.init())
-    .pipe(tsProject())
-    .pipe(babel())
-    .pipe(
-      sourcemaps.write('.', {
-        includeContent: false,
-        sourceRoot: '../src',
-      })
-    )
-    .pipe(gulp.dest('build'));
-});
-
-gulp.task('watch:source', async () => {
-  console.log(`Watching for changes to the Node.js server JavaScript...`);
-  gulp.watch('src/**/*.(t|j)s', gulp.series('build:server'));
-});
-
-gulp.task('watch:local', createWatchTask('development'));
-gulp.task('watch:server', createWatchTask('production'));
-
-function createWatchTask(nodeEnv) {
-  return async () => {
-    console.log(`Starting the Node.js server and watching for changes (NODE_ENV=${nodeEnv})...`);
-    // Run nodemon in a separate process to isolate its side effects on stdio streams
-    let nodeCmd = 'node';
-    if (nodeEnv === 'development') {
-      nodeCmd += ' --inspect=:9315';
-    }
-    nodemon({
-      script: './build/server.js',
-      execMap: {
-        js: nodeCmd,
-      },
-      env: {
-        NODE_ENV: nodeEnv,
-      },
-
-      watch: ['./build'],
-      stdout: false,
-      readable: false,
-    }).on('readable', function() {
-      const bunyan = childProcess.fork(
-        path.join('.', 'node_modules', '.bin', 'bunyan'),
-        ['-o', 'short', '-l', 'debug', '--color'],
-        { silent: true }
-      );
-
-      bunyan.stdout.pipe(process.stdout);
-      bunyan.stderr.pipe(process.stderr);
-      this.stdout.pipe(bunyan.stdin);
-      this.stderr.pipe(bunyan.stdin);
-    });
-  };
-}
-
-gulp.task(
-  'watch',
-  gulp.series('clean', 'build:server', gulp.parallel('watch:source', 'watch:local'))
-);
-
-gulp.task('default', gulp.series('watch'));

k8s/android/turtle-deployment.template.yml

@@ -50,8 +50,6 @@ spec:
             value: exp-shell-app-assets
           - name: AWS_S3_REGION
             value: us-west-1
-          - name: LOGGLY_SUBDOMAIN
-            value: exponent
           - name: TURTLE_TEMPORARY_FILES_DIR
             value: /tmp/turtle
           # vars from k8s secrets
@@ -100,11 +98,6 @@ spec:
               secretKeyRef:
                 name: turtle-js-config
                 key: aws_cloudwatch_region
-          - name: LOGGLY_TOKEN
-            valueFrom:
-              secretKeyRef:
-                name: turtle-js-config
-                key: loggly_token
           - name: REDIS_URL
             valueFrom:
               secretKeyRef:

nix/all-packages.nix

@@ -0,0 +1,5 @@
+self: super:
+
+{
+  nodejs = super.nodejs-8_x;
+}

nix/android-shell.nix

@@ -0,0 +1,50 @@
+with (import ./default.nix {});
+
+let
+
+   ndk = androidenv.androidndk_17c.override { fullNDK = true; };
+   ndk_root = "${ndk}/libexec/${ndk.name}";
+   sdk_path = if stdenv.isDarwin
+     then "/Library/Android/sdk"
+     else "/Android/Sdk"; # intentional Capital S
+
+in
+
+mkShell rec {
+
+  LANG="en_US.UTF-8";
+  JAVA_HOME=openjdk8;
+
+  ANDROID_SDK_ROOT=builtins.getEnv("HOME") + sdk_path;
+  ANDROID_NDK_ROOT="${ANDROID_SDK_ROOT}/ndk-bundle";
+  ANDROID_HOME=ANDROID_SDK_ROOT;
+  ANDROID_NDK=ANDROID_NDK_ROOT;
+
+  nativeBuildInputs = [
+    awscli
+    curl
+    git
+    nodejs-8_x
+    openjdk8
+    yarn
+  ];
+
+  passthru = { inherit ndk; };
+
+  shellHook = ''
+    ${./install-ndk-17c.sh} ${ndk} ${ndk_root}
+    yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --licenses --sdk_root="$ANDROID_SDK_ROOT"
+    yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --sdk_root="$ANDROID_SDK_ROOT" "platforms;android-28" | grep -v =
+    yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --sdk_root="$ANDROID_SDK_ROOT" "build-tools;28.0.3" | grep -v =
+    ${lib.optionalString stdenv.isLinux ''
+      for dep in lib lib64; do
+        if [ -L /$dep ] || [ ! -e /$dep ]; then
+          echo "Creating /$dep"
+          ln -s ${stdenv.cc.libc}/$dep /$dep
+        else
+          echo "Using existing /$dep"
+        fi
+      done
+    ''}
+  '';
+}

nix/default.nix

@@ -0,0 +1,19 @@
+let
+  pin = builtins.fromJSON (builtins.readFile ./nixpkgs.json);
+
+  nixpkgs = builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs-channels/archive/${pin.rev}.tar.gz";
+    inherit (pin) sha256;
+  };
+
+in
+
+{ ... } @ args:
+
+  import nixpkgs (args // {
+    config = {
+      allowUnfree = true;
+      android_sdk.accept_license = true;
+    };
+    overlays = [ (import ./all-packages.nix) ];
+  })

nix/install-ndk-17c.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+scriptdir="$(dirname ${BASH_SOURCE[0]})"
+shellnix="$scriptdir/android-shell.nix"
+
+ndk=${1:-$(nix-build "$shellnix" -A ndk --no-out-link)}
+ndkRoot=${2:-$(nix eval --file "$shellnix" --raw ANDROID_NDK_ROOT)}
+
+mkdir -p "$ANDROID_NDK_ROOT" # Create full path if necessary
+
+# Replace any existing link or directory with link
+rm -r "$ANDROID_NDK_ROOT"
+ln -sn "$ndkRoot" "$ANDROID_NDK_ROOT"

nix/nixpkgs.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/NixOS/nixpkgs-channels.git",
+  "rev": "80738ed9dc0ce48d7796baed5364eef8072c794d",
+  "date": "2018-11-17T11:18:10+01:00",
+  "sha256": "0anmvr6b47gbbyl9v2fn86mfkcwgpbd5lf0yf3drgm8pbv57c1dc",
+  "fetchSubmodules": false
+}

nodemon.json

@@ -0,0 +1,6 @@
+{
+  "watch": ["src"],
+  "ext": "ts",
+  "ignore": ["src/**/*.test.ts"],
+  "exec": "ts-node -r tsconfig-paths/register --files src/server.ts"
+}

package.json

@@ -1,19 +1,18 @@
 {
   "name": "turtle-cli",
-  "version": "0.5.14",
+  "version": "0.6.1",
   "description": "The command-line tool for building standalone Expo apps",
   "preferGlobal": true,
   "scripts": {
-    "start": "EXPENV_OVERRIDES=local-common,local,local-queue expenv gulp",
+    "start": "EXPENV_OVERRIDES=local-common,local,local-queue expenv nodemon | bunyan -o short -l debug --color",
     "start:ios": "PLATFORM=ios yarn start",
     "start:android": "PLATFORM=android yarn start",
-    "watch": "EXPENV_OVERRIDES=local-common,local,local-queue expenv gulp watch:source",
     "bin:turtle": "./bin/turtle.js",
     "lint": "tslint -c tslint.json 'src/**/*.ts'",
-    "prepare": "yarn build",
-    "build": "gulp build:server",
+    "lint:fix": "yarn lint --fix",
+    "prepare": "rimraf ./build && yarn build",
+    "build": "ttsc",
     "build:production": "./scripts/build.sh",
-    "build:android-tarball:local": "./scripts/android/buildAndroidTarballLocally.sh",
     "secrets:init-private": "./scripts/initPrivateSecrets.sh",
     "set-workingdir:remote": "./scripts/setWorkingdirMode.sh remote",
     "set-workingdir:local": "./scripts/setWorkingdirMode.sh local",
@@ -26,7 +25,7 @@
     "test:integration": "yarn test --config jest/integration-config.json",
     "test:unit": "yarn test --config jest/unit-config.json",
     "test:smoke": "yarn test --config jest/smoke-config.json --forceExit",
-    "test:smoke:android": "yarn test:smoke --testNamePattern=android",
+    "test:smoke:android": "PLATFORM=android yarn test:smoke --testNamePattern=android",
     "test:smoke:ios": "PLATFORM=ios yarn test:smoke --testNamePattern=iOS",
     "release": "./scripts/release.sh",
     "release:beta": "./scripts/releaseBeta.sh",
@@ -34,9 +33,21 @@
     "ssh:android:connect": "./scripts/ssh/connectToAndroidPod.sh",
     "ssh:ios:ips": "./scripts/ssh/findIosIps.sh",
     "set-priorities": "yarn set-priorities:local",
-    "set-priorities:local": "EXPENV_OVERRIDES=local-common,local,local-queue expenv babel-node scripts/setConfigurations.js",
-    "set-priorities:staging": "expenv babel-node scripts/setConfigurations.js",
-    "set-priorities:production": "EXPENV_OVERRIDES=prod expenv babel-node scripts/setConfigurations.js"
+    "set-priorities:default": "expenv ts-node -r tsconfig-paths/register --files src/scripts/setConfigurations.ts",
+    "set-priorities:local": "EXPENV_OVERRIDES=local-common,local,local-queue yarn set-priorities:default",
+    "set-priorities:staging": "yarn set-priorities:default",
+    "set-priorities:production": "EXPENV_OVERRIDES=prod yarn set-priorities:default"
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "lint-staged"
+    }
+  },
+  "lint-staged": {
+    "src/**/*.ts": [
+      "yarn lint:fix",
+      "git add"
+    ]
   },
   "bin": {
     "turtle": "./bin/turtle.js"
@@ -72,40 +83,34 @@
     "@types/progress": "^2.0.1",
     "@types/raven": "^2.5.1",
     "@types/request": "^2.47.1",
+    "@types/semver": "^6.0.0",
     "@types/sharp": "^0.17.8",
     "@types/tar": "^4.0.0",
     "@types/uuid": "^3.4.3",
     "@types/which": "^1.3.1",
-    "babel-cli": "^6.26.0",
-    "babel-core": "^6.26.3",
-    "babel-jest": "^23.6.0",
-    "babel-plugin-module-resolver": "^3.1.1",
-    "babel-preset-env": "^1.6.1",
-    "gulp": "github:gulpjs/gulp#4.0",
-    "gulp-babel": "^7.0.1",
-    "gulp-changed": "^3.2.0",
-    "gulp-nodemon": "^2.4.1",
-    "gulp-plumber": "^1.2.0",
-    "gulp-sourcemaps": "^2.6.4",
-    "gulp-typescript": "^4.0.2",
+    "husky": "^1.3.1",
     "jest": "^23.6.0",
+    "lint-staged": "^8.1.5",
     "nock": "^10.0.1",
-    "nodemon": "^1.18.7",
+    "nodemon": "^1.18.11",
     "release-it": "^10.0.0",
     "rimraf": "^2.6.2",
-    "source-map-loader": "^0.2.3",
     "ts-jest": "^23.10.4",
+    "ts-node": "^8.0.3",
+    "ts-transformer-imports": "^0.4.1",
+    "tsconfig-paths": "^3.8.0",
     "tslint": "^5.10.0",
-    "typescript": "^3.0.1"
+    "tslint-eslint-rules": "^5.4.0",
+    "ttypescript": "^1.5.6",
+    "typescript": "^3.4.2"
   },
   "dependencies": {
-    "@expo/spawn-async": "^1.4.0",
+    "@expo/spawn-async": "^1.4.2",
     "@google-cloud/logging-bunyan": "^0.9.5",
     "async-retry": "^1.2.1",
     "aws-sdk": "^2.226.1",
     "bunyan": "^1.8.12",
     "bunyan-debug-stream": "^1.0.8",
-    "bunyan-loggly": "^1.3.4",
     "bunyan-sentry-stream": "^1.2.1",
     "commander": "^2.15.1",
     "copy": "^0.3.1",
@@ -119,12 +124,13 @@
     "raven": "^2.6.2",
     "request": "^2.88.0",
     "request-progress": "^3.0.0",
+    "semver": "^6.0.0",
     "sharp": "^0.21.0",
     "source-map-support": "^0.5.6",
     "tar": "^4.4.6",
     "uuid": "^3.2.1",
     "which": "^1.3.1",
-    "xdl": "53.1.6-alpha.2"
+    "xdl": "53.5.3"
   },
   "optionalDependencies": {
     "@expo/traveling-fastlane-darwin": "^1.8.1"

scripts/android/buildAndroidTarballLocally.sh

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-
-set -xeo pipefail
-
-ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
-ARTIFACTS_DIR="$ROOT_DIR/artifacts"
-TEMP_DIR="/tmp/android-shell-app"
-
-mkdir -p $ARTIFACTS_DIR
-rm -rf $TEMP_DIR
-mkdir -p $TEMP_DIR
-ln -s ${EXPO_UNIVERSE_DIR}/exponent/tools-public $TEMP_DIR/tools-public
-ln -s ${EXPO_UNIVERSE_DIR}/exponent/secrets $TEMP_DIR/secrets
-ln -s ${EXPO_UNIVERSE_DIR}/exponent/template-files $TEMP_DIR/template-files
-ln -s ${EXPO_UNIVERSE_DIR}/exponent/android $TEMP_DIR/android
-ln -s ${EXPO_UNIVERSE_DIR}/exponent/package.json $TEMP_DIR/package.json
-ln -s ${EXPO_UNIVERSE_DIR}/exponent/expokit-npm-package $TEMP_DIR/expokit-npm-package
-ln -s ${EXPO_UNIVERSE_DIR}/exponent/packages $TEMP_DIR/packages
-ln -s ${EXPO_UNIVERSE_DIR}/libraries/eslint-config-universe $TEMP_DIR/eslint-config-universe
-ln -s ${EXPO_UNIVERSE_DIR}/libraries/json-file $TEMP_DIR/json-file
-ln -s ${EXPO_UNIVERSE_DIR}/libraries/osascript $TEMP_DIR/osascript
-ln -s ${EXPO_UNIVERSE_DIR}/libraries/schemer $TEMP_DIR/schemer
-ln -s ${EXPO_UNIVERSE_DIR}/package.json $TEMP_DIR/universe-package.json
-
-# generate dynamic macros
-mkdir -p $TEMP_DIR/android/expoview/src/main/java/host/exp/exponent/generated/
-cd $TEMP_DIR/tools-public
-./generate-dynamic-macros-android.sh
-rm -rf $TEMP_DIR/secrets
-
-cd $TEMP_DIR; tar -czhf $ARTIFACTS_DIR/android-shell-builder.tar.gz .
-rm -rf $TEMP_DIR

scripts/android/fetchRemoteAndroidTarball.sh

@@ -1,24 +1,4 @@
 #!/usr/bin/env bash
 
-set -xeo pipefail
-
 ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
-ARTIFACTS_DIR="$ROOT_DIR/artifacts"
-WORKING_DIR="$ROOT_DIR/workingdir/android"
-ANDROID_SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/android"
-
-mkdir -p $ARTIFACTS_DIR
-
-for SDK_VERSION in `ls $ANDROID_SHELL_TARBALLS_DIRECTORY`; do
-  mkdir -p $WORKING_DIR/$SDK_VERSION
-  TARBALL_URI=$(head -n 1 $ANDROID_SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
-  if command -v run  >/dev/null 2>&1 ; then
-    run expo.s3env --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --command retry5 aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
-  else
-    # outside of universe aws credentials might not be set
-    if ! aws iam get-user >/dev/null 2>&1 ; then aws configure ; fi
-    aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
-  fi
-  tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
-  rm -rf $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
-done
+$ROOT_DIR/scripts/fetchRemoteTarballs.sh android

scripts/fetchRemoteTarballs.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -xeo pipefail
+
+PLATFORM=$1
+
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
+ARTIFACTS_DIR="$ROOT_DIR/artifacts"
+WORKING_DIR="$ROOT_DIR/workingdir/$PLATFORM"
+SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/$PLATFORM"
+
+mkdir -p $ARTIFACTS_DIR
+
+convert_s3_to_https() {
+  BUCKET=`echo $1 | sed 's/s3:\/\/\([^/]*\).*/\1/'`
+  KEY=`echo $1 | sed 's/s3:\/\/[^/]*\/\(.*\)/\1/'`
+  echo "https://s3.amazonaws.com/$BUCKET/$KEY"
+}
+
+download_shell_app() {
+  SDK_VERSION=$1
+  mkdir -p $WORKING_DIR/$SDK_VERSION
+  TARBALL_S3_URL=$(head -n 1 $SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
+  TARBALL_HTTPS_URL=$(convert_s3_to_https $TARBALL_S3_URL)
+  curl -s $TARBALL_HTTPS_URL -o $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
+  tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
+  rm -rf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
+}
+
+if [[ -z "$SDK_VERSION" ]]; then
+  for SDK_VERSION in `ls $SHELL_TARBALLS_DIRECTORY`; do
+    download_shell_app $SDK_VERSION
+  done
+else
+  if [ ! -d "$WORKING_DIR/$SDK_VERSION" ]; then
+    download_shell_app $SDK_VERSION
+  fi
+fi

scripts/ios/fetchRemoteIosTarball.sh

@@ -1,24 +1,4 @@
 #!/usr/bin/env bash
 
-set -xeo pipefail
-
 ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
-ARTIFACTS_DIR="$ROOT_DIR/artifacts"
-WORKING_DIR="$ROOT_DIR/workingdir/ios"
-IOS_SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/ios"
-
-mkdir -p $ARTIFACTS_DIR
-
-for SDK_VERSION in `ls $IOS_SHELL_TARBALLS_DIRECTORY`; do
-  mkdir -p $WORKING_DIR/$SDK_VERSION
-  TARBALL_URI=$(head -n 1 $IOS_SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
-  if command -v run  >/dev/null 2>&1 ; then
-    run expo.s3env --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --command retry5 aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
-  else
-    # outside of universe aws credentials might not be set
-    if ! aws iam get-user >/dev/null 2>&1 ; then aws configure ; fi
-    aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
-  fi
-  tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
-  rm -rf $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
-done
+$ROOT_DIR/scripts/fetchRemoteTarballs.sh ios

scripts/tests/runAndroidSmokeTestsCI.sh

@@ -1,34 +1,25 @@
 #!/usr/bin/env bash
 
+set -ueo pipefail
+
 export NODE_ENV=test
 
 export ENVIRONMENT="development"
 export AWS_ACCESS_KEY_ID="aws_id"
 export AWS_SECRET_ACCESS_KEY="secret"
-export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_ios_in.fifo"
-export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_ios_in.fifo"
-export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_android_in.fifo"
-export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_android_in.fifo"
-export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_out.fifo"
+export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in.fifo"
+export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in_priority.fifo"
+export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in.fifo"
+export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in_priority.fifo"
+export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_out.fifo"
 export AWS_SQS_REGION="aws.west2"
 export AWS_S3_BUCKET="exp-shell-app-assets"
 export AWS_S3_REGION="us-west-1"
 export AWS_CLOUDWATCH_REGION="us-west-1"
 export API_PROTOCOL="http"
-export API_HOSTNAME="localhost"
-export API_PORT="3000"
+export API_HOSTNAME="staging.expo.io"
+export API_PORT="443"
 export REDIS_URL="noop"
 export REDIS_CONFIG_URL="noop"
 
 yarn test:smoke:android
-smoke_result=$?
-
-yarn test:unit
-unit_result=$?
-
-[[ $smoke_result != 0 || $unit_result != 0 ]] && \
-	echo && \
-	echo "Tests failed." && \
-	exit -1
-
-exit 0

scripts/tests/runIOSSmokeTestsCI.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -ueo pipefail
+
 export NODE_ENV=test
 
 export ENVIRONMENT="development"
@@ -18,18 +20,7 @@ export API_PORT="443"
 export REDIS_URL="noop"
 export REDIS_CONFIG_URL="noop"
 
-ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
-$ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
+SDK_VERSION=sdk32 $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
 
 yarn test:smoke:ios
-smoke_result=$?
-
-yarn test:unit
-unit_result=$?
-
-[[ $smoke_result != 0 || $unit_result != 0 ]] && \
-	echo && \
-	echo "Tests failed." && \
-	exit -1
-
-exit 0

scripts/tests/runUnitTestsCI.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -ueo pipefail
+
 export NODE_ENV=test
 
 export ENVIRONMENT="development"
@@ -21,6 +23,3 @@ export REDIS_URL="noop"
 export REDIS_CONFIG_URL="noop"
 
 yarn test:unit
-unit_result=$?
-
-exit $unit_result

shellTarballs/android/sdk26

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz

shellTarballs/android/sdk27

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz

shellTarballs/android/sdk28

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz

shellTarballs/android/sdk29

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz

shellTarballs/android/sdk30

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz

shellTarballs/android/sdk31

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-5c339ff7bab6f2161d378cc70e1bf04d32ad98dd.tar.gz
+s3://exp-artifacts/android-shell-builder-89b2d24c86737d5a489c6a18467e31b2f9ce9005.tar.gz

shellTarballs/android/sdk32

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-a4bd3bd26b1b31bdfa66c39b6b4d0d1d138c32fc.tar.gz
+s3://exp-artifacts/android-shell-builder-0cab68670fc50ce1dc16b7d86ca7976870062c39.tar.gz

shellTarballs/ios/sdk32

@@ -1 +1 @@
-s3://exp-artifacts/ios-shell-builder-sdk-latest-d9cabffbcba20e074e91e4d7f8d862c98acfc680.tar.gz
+s3://exp-artifacts/ios-shell-builder-sdk-latest-dc981c7110b7d508de8e9c352064dc3d745c9f72.tar.gz

src/aws/cloudwatch/index.ts

@@ -179,7 +179,7 @@ function _pushMetrics(metrics: IMetricsChunk[]) {
   metrics.forEach((metricsChunk) => {
     cloudWatch.putMetricData(metricsChunk, (err) => {
       if (err) {
-        logger.warn('[cloudwatch]', err);
+        logger.warn({ err }, '[cloudwatch]');
       }
     });
   });

src/aws/cloudwatch/reducers.ts

@@ -7,7 +7,7 @@ export function sum(metrics: IMetric[]) {
   const reduced = metrics.reduce((acc: IMetric | null, i: IMetric) => {
     let result = acc;
     if (!result) {
-      result = {...i};
+      result = { ...i };
     } else {
       result.Value += i.Value;
     }

src/aws/sqs.ts

@@ -36,7 +36,7 @@ export async function receiveMessage(priority: string) {
         }
       })
       .catch((err) => {
-        logger.error('Receive error', err);
+        logger.error({ err }, 'Receive error');
         throw err;
       });
   });
@@ -68,7 +68,7 @@ export function changeMessageVisibilityRecurring(priority: string, receiptHandle
   return setInterval(() => {
     if (getCurrentJobId() === jobId) {
       changeMessageVisibility(priority, receiptHandle).catch((err) => {
-        logger.warn('Error at change msg visibility', err);
+        logger.warn({ err }, 'Error at change msg visibility');
       });
     }
   }, (VISIBILITY_TIMEOUT_SEC * 1000) / 3);
@@ -86,6 +86,6 @@ export async function sendMessage(jobId: string, status: JOB_UPDATE_TYPE, data =
       return await sqs.sendMessage(params).promise();
     });
   } catch (err) {
-    logger.error(`Error sending SQS message: ${JSON.stringify(params)}}`, err);
+    logger.error({ err }, `Error sending SQS message: ${JSON.stringify(params)}}`);
   }
 }

src/bin/commands/setup.ts

@@ -33,8 +33,7 @@ async function setupAction(program: any, cmd: any, platform: string, os?: string
     await setup(platform, sdkVersion);
     l.info('it\'s all set!');
   } catch (err) {
-    logger.error(`Failed to setup environment for ${platform} builds`);
-    logger.error(err.stack);
+    logger.error({ err }, `Failed to setup environment for ${platform} builds`);
     if (err instanceof ErrorWithCommandHelp) {
       cmd.help();
     } else if (err instanceof ErrorWithProgramHelp) {

src/bin/index.ts

@@ -17,16 +17,16 @@ Command.prototype.asyncAction = function asyncAction(asyncFn: (...asyncFnArgs: a
   return this.action(async (...args) => {
     try {
       await checkForUpdateAsync();
-    } catch (e) {
-      logger.warn('Failed to check for turtle-cli update.');
+    } catch (err) {
+      logger.warn({ err }, 'Failed to check for turtle-cli update.');
     }
     return await asyncFn(...args);
   });
 };
 
 export function run(programName: string) {
-  runAsync(programName).catch((e) => {
-    logger.error('Uncaught Error', e);
+  runAsync(programName).catch((err) => {
+    logger.error({ err }, 'Uncaught Error');
     process.exit(1);
   });
 }

src/bin/setup/ios.ts

@@ -1,4 +1,7 @@
+import last from 'lodash/last';
+import trim from 'lodash/trim';
 import path from 'path';
+import semver from 'semver';
 
 import { ExponentTools } from 'xdl';
 
@@ -16,6 +19,19 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
   {
     command: 'fastlane',
     missingDescription: 'Please check https://docs.fastlane.tools/getting-started/ios/setup/',
+    versionCheckFn: async () => {
+      const MINIMAL_VERSION = '2.99.0';
+
+      const { stdout } = await ExponentTools.spawnAsyncThrowError(
+        'fastlane',
+        ['--version'],
+        { stdio: 'pipe', env: { ...process.env, FASTLANE_SKIP_UPDATE_CHECK: '1' } },
+      );
+      const fastlaneVersion = parseFastlaneVersion(stdout);
+      if (fastlaneVersion && !semver.satisfies(fastlaneVersion, `>= ${MINIMAL_VERSION}`)) {
+        throw new Error(`Please install newer version of fastlane (>= ${MINIMAL_VERSION})`);
+      }
+    },
   },
   {
     command: 'xcode',
@@ -47,3 +63,11 @@ export default async function setup(sdkVersion?: string) {
 function formatShellAppTarballUriPath(sdkMajorVersion: string) {
   return path.join(config.directories.shellTarballsDir, PLATFORMS.IOS, `sdk${sdkMajorVersion}`);
 }
+
+function parseFastlaneVersion(stdout: string) {
+  const lastLine = last(trim(stdout).split('\n'));
+  if (!lastLine) {
+    return null;
+  }
+  return last(lastLine.split(' ')) || null;
+}

src/bin/setup/utils/toolsDetector.ts

@@ -10,11 +10,12 @@ export interface IToolDefinition {
   command: string;
   missingDescription: string;
   testFn?: () => Promise<boolean>;
+  versionCheckFn?: () => void;
 }
 
 export async function ensureToolsAreInstalled(tools: IToolDefinition[]) {
   let isAnyToolMissing = false;
-  for (const { command, missingDescription, testFn } of tools) {
+  for (const { command, missingDescription, testFn, versionCheckFn } of tools) {
     try {
       if (testFn) {
         if (!await testFn()) {
@@ -26,9 +27,17 @@ export async function ensureToolsAreInstalled(tools: IToolDefinition[]) {
     } catch (err) {
       isAnyToolMissing = true;
       if (!testFn) {
-        l.error(`${command} is missing in your $PATH`);
+        l.error({ err }, `${command} is missing in your $PATH`);
+      }
+      l.error({ err }, missingDescription);
+    }
+    if (versionCheckFn) {
+      try {
+        await versionCheckFn();
+      } catch (err) {
+        isAnyToolMissing = true;
+        l.error({ err }, `wrong version of ${command} installed`);
       }
-      l.error(missingDescription);
     }
   }
   if (isAnyToolMissing) {

src/bin/utils/builder.ts

@@ -72,8 +72,7 @@ export function createBuilderAction({
       const job = await sanitizeJob(rawJob);
       await builder(job);
     } catch (err) {
-      logger.error(`Failed to build standalone app: ${err.message}`);
-      logger.error(err.stack);
+      logger.error({ err }, 'Failed to build standalone app');
       if (err instanceof ErrorWithCommandHelp) {
         command.help();
       } else if (err instanceof ErrorWithProgramHelp) {

src/builders/utils/ios/keychain.ts

@@ -16,7 +16,7 @@ export async function create(ctx: IContext): Promise<IKeychain> {
     l.info('done creating keychain');
     return keychainInfo;
   } catch (err) {
-    l.error('unable to create keychain');
+    l.error({ err }, 'unable to create keychain');
     throw err;
   }
 }
@@ -32,7 +32,7 @@ export async function remove(ctx: IContext, keychainPath: string) {
     l.info('done deleting keychain');
     return keychainInfo;
   } catch (err) {
-    l.error('unable to delete keychain');
+    l.error({ err }, 'unable to delete keychain');
     throw err;
   }
 }
@@ -57,7 +57,7 @@ export async function importCert(
     await IosKeychain.importIntoKeychain({ keychainPath, certPath, certPassword });
     l.info('done importing distribution certificate into keychain');
   } catch (err) {
-    l.error('unable to import distribution certificate into keychain');
+    l.error({ err }, 'unable to import distribution certificate into keychain');
     throw err;
   }
 }

src/config.ts

@@ -47,11 +47,6 @@ export default {
     client: {
       level: env('CLIENT_LOGGER_LEVEL', 'info'),
     },
-    loggly: {
-      token: env('LOGGLY_TOKEN', ''),
-      subdomain: env('LOGGLY_SUBDOMAIN', ''),
-      buffer: envNum('LOGGLY_BUFFER', 100),
-    },
   },
   google: {
     credentials: env('GOOGLE_APPLICATION_CREDENTIALS', ''),

src/jobManager.ts

@@ -47,7 +47,7 @@ export async function getJob() {
         return job;
       }
     } catch (err) {
-      logger.error('Error at receiving messages', err);
+      logger.error({ err }, 'Error at receiving messages');
     }
   }
 }
@@ -160,6 +160,6 @@ async function deleteMessage(priority: string, receiptHandle: string) {
     setCurrentJobId(null);
     await sqs.deleteMessage(priority, receiptHandle);
   } catch (err) {
-    logger.error('Error at deleting msg', err);
+    logger.error({ err }, 'Error at deleting msg');
   }
 }

src/logger/gcloudStream.ts

@@ -0,0 +1,58 @@
+import EventEmitter from 'events';
+
+import { IJob } from 'turtle/job';
+
+// type error when using import
+// https://github.com/googleapis/nodejs-logging-bunyan/issues/241
+// tslint:disable-next-line:no-var-requires
+const { LoggingBunyan } = require('@google-cloud/logging-bunyan');
+
+export interface IGCloudConfig {
+  name: string;
+  resource: {
+    type: string;
+    labels: {
+      node_id: string;
+      location: string;
+      namespace: string;
+    };
+  };
+}
+
+interface IGCloudExtraFields {
+  jobID: string;
+  experienceName: string;
+}
+
+export default class GCloudStream extends EventEmitter {
+  private logsStream: any;
+  private extraFields: IGCloudExtraFields | null;
+
+   constructor(gcloudConfig: IGCloudConfig) {
+    super();
+
+    const { name, resource } = gcloudConfig;
+    this.logsStream = new LoggingBunyan({ name, resource });
+    this.extraFields = null;
+  }
+
+  public init(job: IJob) {
+    this.extraFields = {
+      jobID: job.id,
+      experienceName: job.experienceName,
+    };
+  }
+
+  public cleanup() {
+    this.extraFields = null;
+  }
+
+  public write(rec: any) {
+    const { msg: message, ...other } = rec;
+    this.logsStream.write({
+      message,
+      ...other,
+      data: this.extraFields || {},
+    });
+  }
+}

src/logger/hackyLogglyStream.ts

@@ -1,48 +0,0 @@
-import EventEmitter from 'events';
-
-import BunyanLoggly from 'bunyan-loggly';
-
-import config from 'turtle/config';
-import * as constants from 'turtle/constants/logger';
-import { IJob } from 'turtle/job';
-
-export interface ILogglyConfig {
-  token: string;
-  subdomain: string;
-  tags: string[];
-}
-
-interface ILogglyExtraFields {
-  jobID: string;
-  experienceName: string;
-}
-
-export default class HackyLogglyStream extends EventEmitter {
-  private logglyStream: BunyanLoggly;
-  private extraFields: ILogglyExtraFields | null;
-
-  constructor(logglyConfig: ILogglyConfig) {
-    super();
-    this.logglyStream = new BunyanLoggly(logglyConfig, config.logger.loggly.buffer);
-    this.extraFields = null;
-  }
-  public init(job: IJob) {
-    this.extraFields = {
-      jobID: job.id,
-      experienceName: job.experienceName,
-    };
-  }
-  public cleanup() {
-    this.extraFields = null;
-  }
-  public write(rec: any) {
-    const { msg: message, hostname: host, level, ...other } = rec;
-    this.logglyStream.write({
-      message,
-      level: constants.LEVELS_MAPPING[level],
-      host,
-      ...other,
-      data: this.extraFields || {},
-    });
-  }
-}

src/logger/index.ts

@@ -6,15 +6,10 @@ import { Client as RavenClient } from 'raven';
 import config from 'turtle/config';
 import * as constants from 'turtle/constants/logger';
 import { IJob } from 'turtle/job';
-import HackyLogglyStream from 'turtle/logger/hackyLogglyStream';
+import GCloudStream from 'turtle/logger/gcloudStream';
 import S3Stream from 'turtle/logger/s3Stream';
 import { isOffline } from 'turtle/turtleContext';
 
-// type error when using import
-// https://github.com/googleapis/nodejs-logging-bunyan/issues/241
-// tslint:disable-next-line:no-var-requires
-const { LoggingBunyan } = require('@google-cloud/logging-bunyan');
-
 interface IStream {
   stream: any;
   type?: string;
@@ -23,7 +18,7 @@ interface IStream {
 }
 
 export const s3logger = new S3Stream();
-let logglyStream: HackyLogglyStream;
+let gcloudStream: GCloudStream;
 
 const streams: IStream[] = [];
 
@@ -52,32 +47,24 @@ if (config.sentry.dsn && config.deploymentEnv !== 'development') {
   });
 }
 
-if (config.logger.loggly.token) {
-  const logglyConfig = {
-    token: config.logger.loggly.token,
-    subdomain: config.logger.loggly.subdomain,
-    // prettier-ignore
-    tags: [
-      'app-shell-apps',
-    ],
-  };
-  logglyStream = new HackyLogglyStream(logglyConfig);
-  streams.push({
-    type: 'raw',
-    stream: logglyStream,
-  });
-}
-
 if (config.google.credentials) {
-  const resource = {
-    type: 'generic_node',
-    labels: {
-      node_id: config.hostname,
-      location: '', // default value
-      namespace: '', // default value
+  const gcloudConfig = {
+    name: 'turtle',
+    resource: {
+      type: 'generic_node',
+      labels: {
+        node_id: config.hostname,
+        location: '', // default value
+        namespace: '', // default value
+      },
     },
   };
-  streams.push(new LoggingBunyan({ name: 'turtle', resource }).stream('info'));
+  gcloudStream = new GCloudStream(gcloudConfig);
+  streams.push({
+    level: 'info',
+    type: 'raw',
+    stream: gcloudStream,
+  });
 }
 
 const logger = bunyan.createLogger({
@@ -91,8 +78,8 @@ const logger = bunyan.createLogger({
 logger.withFields = (extraFields: any) => withFields(logger, extraFields);
 
 logger.init = async (job: IJob) => {
-  if (logglyStream) {
-    logglyStream.init(job);
+  if (gcloudStream) {
+    gcloudStream.init(job);
   }
   return await s3logger.init(job);
 };
@@ -100,8 +87,8 @@ logger.init = async (job: IJob) => {
 logger.cleanup = async () => {
   // a little hacky, but works
   logger.info({ lastBuildLog: true }, 'this is the last log from this build');
-  if (logglyStream) {
-    logglyStream.cleanup();
+  if (gcloudStream) {
+    gcloudStream.cleanup();
   }
   await s3logger.waitForLogger();
 };

src/node_modules/bunyan-loggly/index.d.ts

@@ -1,13 +0,0 @@
-export = BunyanLoggly;
-
-declare class BunyanLoggly {
-  constructor(options: BunyanLoggly.IOptions, bufferLength?: number, bufferTimeout?: number, logglyCallback?: any);
-  public write(data: any): void;
-}
-
-declare namespace BunyanLoggly {
-  interface IOptions {
-    token: string;
-    subdomain: string;
-  }
-}

src/scripts/setConfigurations.ts

@@ -1,12 +1,13 @@
-import { env } from 'turtle/utils/env';
-import { connect, RedisClient } from 'turtle/utils/redis';
+
 import { PLATFORMS } from 'turtle/constants';
+import logger from 'turtle/logger';
 import {
   createConfigurationsKey,
   createDefaultConfigurationKey,
   HIGH_CONFIGURATION,
   NORMAL_CONFIGURATION,
 } from 'turtle/utils/priorities';
+import { connect, RedisClient } from 'turtle/utils/redis';
 
 const TIMEOUT = 3000;
 
@@ -25,7 +26,6 @@ const TIMEOUT = 3000;
 // they should come back to using configurations no. 1 and 2.
 // If there are no more available configurations from the table, the default configuration is used.
 
-
 const AVAILABLE_CONFIGURATIONS_IOS = [
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
@@ -54,7 +54,6 @@ const AVAILABLE_CONFIGURATIONS_ANDROID = [
   NORMAL_CONFIGURATION,
 ];
 
-
 const DEFAULT_CONFIGURATION = HIGH_CONFIGURATION;
 
 async function run() {
@@ -68,6 +67,8 @@ async function run() {
   redis.disconnect();
 }
 
-run()
-  .then(() => console.log('All done'))
-  .catch(err => console.error('Error:', err));
+if (require.main === module) {
+  run()
+    .then(() => logger.info('All done'))
+    .catch((err) => logger.error('Error:', err));
+}

src/server.ts

@@ -9,7 +9,7 @@ import { getRedisClient, RedisClient } from 'turtle/utils/redis';
 
 const REDIS_TURTLE_VERSION_KEY = 'turtle:version';
 
-process.on('unhandledRejection', (err) => logger.error('Unhandled promise rejection:', err));
+process.on('unhandledRejection', (err) => logger.error({ err }, 'Unhandled promise rejection'));
 
 function handleExit() {
   if (checkShouldExit()) {
@@ -39,19 +39,19 @@ async function main() {
     await redis.set(REDIS_TURTLE_VERSION_KEY, turtleVersion);
     logger.info(`Register Turtle version ${turtleVersion} in Redis`);
   } catch (err) {
-    logger.error('Failed to register Turtle version in Redis', err);
+    logger.error({ err }, 'Failed to register Turtle version in Redis');
   }
 
   while (true) {
     try {
       await doJob();
     } catch (err) {
-      logger.error('Failed to do a job', err);
+      logger.error({ err }, 'Failed to do a job');
     }
   }
 }
 
 main()
   .then(() => logger.error('This should never happen...'))
-  .catch((err) => logger.error('Something went terribly wrong :(', err))
+  .catch((err) => logger.error({ err }, 'Something went terribly wrong'))
   .then(() => process.exit(1));

src/utils/priorities.ts

@@ -60,24 +60,23 @@ export async function getPriorities() {
       logger.debug(`Using configuration at index ${configurationsIndex} pulled from redis`);
       const { error, value: configValue } = configSchema.validate(configurations[configurationsIndex]);
       if (error) {
-        logger.warn(error);
+        logger.warn({ err: error });
         throw new Error('Received configuration is not valid');
       }
       return configValue;
     } catch (err) {
-      logger.warn(err);
+      logger.warn({ err });
       const defaultConfig = JSON.parse(await redis.get(createDefaultConfigurationKey()));
       const { error, value: defaultConfigValue } = configSchema.validate(defaultConfig);
       if (error) {
-        logger.warn(error);
+        logger.warn({ err: error });
         throw new Error('Received default configuration is not valid');
       }
       logger.warn('Using default configuration pulled from redis');
       return defaultConfigValue;
     }
   } catch (err) {
-    logger.warn(err);
-    logger.warn('Using configuration chosen locally');
+    logger.warn({ err }, 'Using configuration chosen locally');
     if (Math.random() < 0.7) {
       return HIGH_CONFIGURATION;
     } else {

src/utils/redis.ts

@@ -20,7 +20,7 @@ const clientToUrl: IClientConfig = {
   [RedisClient.Configuration]: config.redis.configUrl,
 };
 
-export function connect(timeoutMs: number, type: string) {
+export function connect(timeoutMs: number, type: string): Promise<any> {
   return new Promise((resolve, reject) => {
     const redisClient = new Redis(clientToUrl[type], {
       maxRetriesPerRequest: 2,
@@ -52,7 +52,7 @@ export async function getRedisClient(type = RedisClient.Default) {
     try {
       redisClients[type] = await connect(MILLIS_CONNECTION_TIMEOUT, type);
     } catch (err) {
-      logger.error(err);
+      logger.error({ err });
     }
   }
   return redisClients[type];
@@ -66,7 +66,7 @@ export async function checkIfCancelled(jobId: string) {
     if (config.deploymentEnv === 'development') {
       logger.warn('Did you turn on redis server? Run `yarn start-docker` in server/www');
     }
-    logger.error(err);
+    logger.error({ err });
     return false;
   }
 }

tools/bin/aws

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws "$@"

tools/bin/aws_bash_completer

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws_bash_completer "$@"

tools/bin/aws_completer

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws_completer "$@"

tools/bin/git-crypt

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" git-crypt --command git-crypt "$@"

tools/bin/nix-binstubs

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+set -euo pipefail
+
+scriptDir=$(dirname "$0")
+nixDir="$scriptDir/../../nix"
+
+nix build -f $nixDir --no-link "$@"
+
+for package in "$@"; do
+  for b in $(ls $(nix path-info -f $nixDir $package)/bin); do
+    cat << EOF > "$scriptDir/$b"
+#! /usr/bin/env bash
+# This file was generated by $(basename "$0")
+exec nix run -f "\$(dirname "\$0")/../../nix" $package --command $b "\$@"
+EOF
+  chmod +x "$scriptDir/$b"
+  done
+done
+

tools/ci/bin/node

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../../nix" nodejs-8_x --command node "$@"

tools/ci/bin/npm

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../../nix" nodejs-8_x --command npm "$@"

tools/ci/bin/npx

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../../nix" nodejs-8_x --command npx "$@"

tools/ci/bin/yarn

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../../nix" yarn --command yarn "$@"

tools/ci/bin/yarnpkg

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../../nix" yarn --command yarnpkg "$@"

tsconfig.json

@@ -6,17 +6,20 @@
     "sourceMap": true,
     "inlineSources": true,
     "outDir": "./build",
-    "rootDir": "./src",
     "strict": true,
     "noUnusedLocals": true,
     "noUnusedParameters": true,
     "noFallthroughCasesInSwitch": true,
     "esModuleInterop": true,
-    "baseUrl": "./",
+    "baseUrl": ".",
     "paths": {
       "turtle/*": ["src/*"],
     },
-    "allowJs": true
+    "allowJs": true,
+    "plugins": [
+      { "transform": "ts-transformer-imports" }
+    ]
   },
+  "include": ["src"],
   "exclude": ["node_modules", "build", "workingdir"]
 }

tslint.json

@@ -5,7 +5,10 @@
   "rules": {
     "quotemark": [true, "single", "jsx-double"],
     "object-literal-sort-keys": false,
+    "object-curly-spacing": true,
     "no-namespace": false
   },
-  "rulesDirectory": []
+  "rulesDirectory": [
+    "node_modules/tslint-eslint-rules/dist/rules"
+  ]
 }