Instantiate logger only at entrypoints

This ensures buildPhase, jobID, and experienceName labels are propagated when appropriate.

.babelrc

@@ -0,0 +1,16 @@
+{
+  "presets": [
+    ["env", {
+      "targets": {
+        "node": "8.9"
+      }
+    }]
+  ],
+  "plugins": [
+    ["module-resolver", {
+      "alias": {
+        "turtle": "./build"
+      }
+    }]
+  ]
+}

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 executors:
   nix: &nix
     docker:
-      - image: lnl7/nix:2.2
+      - image: lnl7/nix:2.1.2
     # Symlink to the readline-enabled bash which is the default command of the container
     shell: /run/current-system/sw/bin/bash -leo pipefail
     working_directory: ~/turtle
@@ -33,12 +33,10 @@ commands:
           command: |
             nix-env -iA \
               nixpkgs.binutils \
-              nixpkgs.curl \
               nixpkgs.direnv \
               nixpkgs.findutils \
               nixpkgs.gawk \
               nixpkgs.git \
-              nixpkgs.gnused \
               nixpkgs.gnutar \
               nixpkgs.gzip \
               nixpkgs.openssh \
@@ -139,12 +137,11 @@ commands:
 
 jobs:
   build:
-    executor: nix
+    docker:
+      - image: circleci/node:8.12
     steps:
-      - setup
-      - restore_yarn_cache
+      - checkout
       - run: yarn
-      - save_yarn_cache
       - run: yarn lint
       - run: yarn build
 
@@ -156,7 +153,7 @@ jobs:
       - run: yarn
       - save_yarn_cache
       - run:
-          name: Run unit tests
+          name: Run iOS smoke tests
           command: ./scripts/tests/runUnitTestsCI.sh
 
   ios_smoke_tests:

.envrc

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 source_local() {
   file=./.envrc.local
@@ -17,7 +17,4 @@ export EXPO_TURTLE_TOOLS_DIR="$(pwd)/tools"
 
 if [ -z "$EXPO_UNIVERSE_DIR" ]; then
   PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
-  if [ "$CI" = "true" ]; then
-    PATH_add $EXPO_TURTLE_TOOLS_DIR/ci/bin
-  fi
 fi

.gitignore

@@ -2,7 +2,6 @@
 
 node_modules
 yarn-error.log
-google-cloud-credentials.json
 
 /build/
 /workingdir/*

.nvmrc

@@ -1 +0,0 @@
-v8.15.1

CHANGELOG.md

@@ -5,43 +5,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
-
-## [0.7.2] - 2019-05-09
-### Removed
-- Support for `CLIENT_LOGGER_LEVEL` env variable.
-### Added
-- Support for Android tarballs without `universe-package.json`.
 ### Changed
-- Upgraded `@expo/traveling-fastlane-darwin` to 1.9.4 (fixes issues with generating provisioning profiles for ad-hoc builds).
-
-## [0.7.1] - 2019-05-06
-### Fixed
-- Resolving `fastlane` version when `LC_ALL` is not set.
-
-## [0.7.0] - 2019-05-06
-### Added
-- Support for custom Android builds.
-### Changed
-- Refactored logger.
-### Fixed
-- `platform` log field wrongly set to `ios` for Android builds (when using `turtle-cli`).
-- Brought back `jobID` and `experienceName` fields to Android logs.
-
-## [0.6.1] - 2019-04-23
-### Added
-- Checking fastlane version before running build (turtle-cli).
-### Changed
-- Updated iOS shell app for SDK 32 to fix AppAuth issue.
-- Babel replaced by ttypescript.
-- Updated xdl to 53.5.3 and @expo/spawn-async to 1.4.2, so that error stack traces should be more informative now.
-### Fixed
-- Brought back `jobID` and `experienceName` fields to GCloud logs.
-### Removed
-- `buidAndroidTarballLocally.sh` script (it has been moved to expo repository - [expo/expo#4022](https://github.com/expo/expo/pull/4022)).
+- Updated iOS shell app for SDK 32 to fix AppAuth issue
 
 ## [0.6.0] - 2019-04-09
 ### Changed
-- Removed Branch module from Android shell apps for all SDK versions.
+- Removed Branch module from Android shell apps for all SDK versions
 
 ## [0.5.15] - 2019-04-03
 ### Added

Dockerfile

@@ -99,19 +99,18 @@ RUN rm gradle-4.4-all.zip
 ENV GRADLE_HOME /usr/local/gradle-4.4
 ENV PATH ${GRADLE_HOME}/bin:$PATH
 
+# Install Gulp
+RUN npm install -g gulp-cli
+
 ADD . /app
 
 RUN for SDK_VERSION in `ls /app/workingdir/android/`; do \
       cd /app/workingdir/android/$SDK_VERSION && \
-      if [ -f universe-package.json ]; then \
       mv package.json exponent-package.json && \
       mv universe-package.json package.json && \
       yarn install && \
       mv package.json universe-package.json && \
-      mv exponent-package.json package.json; \
-      else \
-      yarn install; \
-      fi \
+      mv exponent-package.json package.json \
     ; done
 
 WORKDIR /app

bin/turtle.js

@@ -1,15 +1,6 @@
 #!/usr/bin/env node
 'use strict';
 
-const subcommand = process.argv[2];
-if (subcommand) {
-  const [, platform] = subcommand.split(':');
-  if (platform === 'ios' || platform === 'android') {
-    // so that logs have correct platform set
-    process.env.PLATFORM = platform;
-  }
-}
-
 require('source-map-support').install();
 require('../build/bin/env').initOfflineEnv();
 require('../build/bin/index').run('turtle');

gulpfile.js

@@ -0,0 +1,91 @@
+'use strict';
+
+const childProcess = require('child_process');
+const util = require('util');
+const path = require('path');
+
+const gulp = require('gulp');
+const babel = require('gulp-babel');
+const changed = require('gulp-changed');
+const nodemon = require('gulp-nodemon');
+const plumber = require('gulp-plumber');
+const sourcemaps = require('gulp-sourcemaps');
+const ts = require('gulp-typescript');
+const rimraf = require('rimraf');
+
+const rimrafAsync = util.promisify(rimraf);
+
+const tsProject = ts.createProject('tsconfig.json');
+
+gulp.task('clean', async () => {
+  console.log(`Removing build directories...`);
+  await rimrafAsync('build');
+});
+
+gulp.task('build:server', () => {
+  console.log(`Compiling the Node.js server JavaScript...`);
+  return tsProject
+    .src()
+    .pipe(plumber())
+    .pipe(changed('build'))
+    .pipe(sourcemaps.init())
+    .pipe(tsProject())
+    .pipe(babel())
+    .pipe(
+      sourcemaps.write('.', {
+        includeContent: false,
+        sourceRoot: '../src',
+      })
+    )
+    .pipe(gulp.dest('build'));
+});
+
+gulp.task('watch:source', async () => {
+  console.log(`Watching for changes to the Node.js server JavaScript...`);
+  gulp.watch('src/**/*.(t|j)s', gulp.series('build:server'));
+});
+
+gulp.task('watch:local', createWatchTask('development'));
+gulp.task('watch:server', createWatchTask('production'));
+
+function createWatchTask(nodeEnv) {
+  return async () => {
+    console.log(`Starting the Node.js server and watching for changes (NODE_ENV=${nodeEnv})...`);
+    // Run nodemon in a separate process to isolate its side effects on stdio streams
+    let nodeCmd = 'node';
+    if (nodeEnv === 'development') {
+      nodeCmd += ' --inspect=:9315';
+    }
+    nodemon({
+      script: './build/server.js',
+      execMap: {
+        js: nodeCmd,
+      },
+      env: {
+        NODE_ENV: nodeEnv,
+      },
+
+      watch: ['./build'],
+      stdout: false,
+      readable: false,
+    }).on('readable', function() {
+      const bunyan = childProcess.fork(
+        path.join('.', 'node_modules', '.bin', 'bunyan'),
+        ['-o', 'short', '-l', 'debug', '--color'],
+        { silent: true }
+      );
+
+      bunyan.stdout.pipe(process.stdout);
+      bunyan.stderr.pipe(process.stderr);
+      this.stdout.pipe(bunyan.stdin);
+      this.stderr.pipe(bunyan.stdin);
+    });
+  };
+}
+
+gulp.task(
+  'watch',
+  gulp.series('clean', 'build:server', gulp.parallel('watch:source', 'watch:local'))
+);
+
+gulp.task('default', gulp.series('watch'));

nodemon.json

@@ -1,6 +0,0 @@
-{
-  "watch": ["src"],
-  "ext": "ts",
-  "ignore": ["src/**/*.test.ts"],
-  "exec": "ts-node -r tsconfig-paths/register --files src/server.ts"
-}

package.json

@@ -1,18 +1,20 @@
 {
   "name": "turtle-cli",
-  "version": "0.7.2",
+  "version": "0.6.0",
   "description": "The command-line tool for building standalone Expo apps",
   "preferGlobal": true,
   "scripts": {
-    "start": "EXPENV_OVERRIDES=local-common,local,local-queue expenv nodemon | bunyan -o short -l debug --color",
+    "start": "EXPENV_OVERRIDES=local-common,local,local-queue expenv gulp",
     "start:ios": "PLATFORM=ios yarn start",
     "start:android": "PLATFORM=android yarn start",
+    "watch": "EXPENV_OVERRIDES=local-common,local,local-queue expenv gulp watch:source",
     "bin:turtle": "./bin/turtle.js",
     "lint": "tslint -c tslint.json 'src/**/*.ts'",
     "lint:fix": "yarn lint --fix",
-    "prepare": "rimraf ./build && yarn build",
-    "build": "ttsc",
+    "prepare": "yarn build",
+    "build": "gulp build:server",
     "build:production": "./scripts/build.sh",
+    "build:android-tarball:local": "./scripts/android/buildAndroidTarballLocally.sh",
     "secrets:init-private": "./scripts/initPrivateSecrets.sh",
     "set-workingdir:remote": "./scripts/setWorkingdirMode.sh remote",
     "set-workingdir:local": "./scripts/setWorkingdirMode.sh local",
@@ -33,10 +35,9 @@
     "ssh:android:connect": "./scripts/ssh/connectToAndroidPod.sh",
     "ssh:ios:ips": "./scripts/ssh/findIosIps.sh",
     "set-priorities": "yarn set-priorities:local",
-    "set-priorities:default": "expenv ts-node -r tsconfig-paths/register --files src/scripts/setConfigurations.ts",
-    "set-priorities:local": "EXPENV_OVERRIDES=local-common,local,local-queue yarn set-priorities:default",
-    "set-priorities:staging": "yarn set-priorities:default",
-    "set-priorities:production": "EXPENV_OVERRIDES=prod yarn set-priorities:default"
+    "set-priorities:local": "EXPENV_OVERRIDES=local-common,local,local-queue expenv babel-node scripts/setConfigurations.js",
+    "set-priorities:staging": "expenv babel-node scripts/setConfigurations.js",
+    "set-priorities:production": "EXPENV_OVERRIDES=prod expenv babel-node scripts/setConfigurations.js"
   },
   "husky": {
     "hooks": {
@@ -83,30 +84,38 @@
     "@types/progress": "^2.0.1",
     "@types/raven": "^2.5.1",
     "@types/request": "^2.47.1",
-    "@types/semver": "^6.0.0",
     "@types/sharp": "^0.17.8",
     "@types/tar": "^4.0.0",
     "@types/uuid": "^3.4.3",
     "@types/which": "^1.3.1",
+    "babel-cli": "^6.26.0",
+    "babel-core": "^6.26.3",
+    "babel-jest": "^23.6.0",
+    "babel-plugin-module-resolver": "^3.1.1",
+    "babel-preset-env": "^1.6.1",
+    "gulp": "github:gulpjs/gulp#4.0",
+    "gulp-babel": "^7.0.1",
+    "gulp-changed": "^3.2.0",
+    "gulp-nodemon": "^2.4.1",
+    "gulp-plumber": "^1.2.0",
+    "gulp-sourcemaps": "^2.6.4",
+    "gulp-typescript": "^4.0.2",
     "husky": "^1.3.1",
     "jest": "^23.6.0",
     "lint-staged": "^8.1.5",
     "nock": "^10.0.1",
-    "nodemon": "^1.18.11",
+    "nodemon": "^1.18.7",
     "release-it": "^10.0.0",
     "rimraf": "^2.6.2",
+    "source-map-loader": "^0.2.3",
     "ts-jest": "^23.10.4",
-    "ts-node": "^8.0.3",
-    "ts-transformer-imports": "^0.4.1",
-    "tsconfig-paths": "^3.8.0",
     "tslint": "^5.10.0",
     "tslint-eslint-rules": "^5.4.0",
-    "ttypescript": "^1.5.6",
-    "typescript": "^3.4.2"
+    "typescript": "^3.0.1"
   },
   "dependencies": {
-    "@expo/spawn-async": "^1.4.2",
-    "@google-cloud/logging-bunyan": "^0.9.5",
+    "@expo/spawn-async": "^1.4.0",
+    "@google-cloud/logging-bunyan": "^0.10.1",
     "async-retry": "^1.2.1",
     "aws-sdk": "^2.226.1",
     "bunyan": "^1.8.12",
@@ -124,15 +133,14 @@
     "raven": "^2.6.2",
     "request": "^2.88.0",
     "request-progress": "^3.0.0",
-    "semver": "^6.0.0",
     "sharp": "^0.21.0",
     "source-map-support": "^0.5.6",
     "tar": "^4.4.6",
     "uuid": "^3.2.1",
     "which": "^1.3.1",
-    "xdl": "53.5.4"
+    "xdl": "53.1.6-alpha.2"
   },
   "optionalDependencies": {
-    "@expo/traveling-fastlane-darwin": "1.9.4"
+    "@expo/traveling-fastlane-darwin": "^1.8.1"
   }
 }

scripts/android/buildAndroidTarballLocally.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -xeo pipefail
+
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
+ARTIFACTS_DIR="$ROOT_DIR/artifacts"
+TEMP_DIR="/tmp/android-shell-app"
+
+mkdir -p $ARTIFACTS_DIR
+rm -rf $TEMP_DIR
+mkdir -p $TEMP_DIR
+ln -s ${EXPO_UNIVERSE_DIR}/exponent/tools-public $TEMP_DIR/tools-public
+ln -s ${EXPO_UNIVERSE_DIR}/exponent/secrets $TEMP_DIR/secrets
+ln -s ${EXPO_UNIVERSE_DIR}/exponent/template-files $TEMP_DIR/template-files
+ln -s ${EXPO_UNIVERSE_DIR}/exponent/android $TEMP_DIR/android
+ln -s ${EXPO_UNIVERSE_DIR}/exponent/package.json $TEMP_DIR/package.json
+ln -s ${EXPO_UNIVERSE_DIR}/exponent/expokit-npm-package $TEMP_DIR/expokit-npm-package
+ln -s ${EXPO_UNIVERSE_DIR}/exponent/packages $TEMP_DIR/packages
+ln -s ${EXPO_UNIVERSE_DIR}/libraries/eslint-config-universe $TEMP_DIR/eslint-config-universe
+ln -s ${EXPO_UNIVERSE_DIR}/libraries/json-file $TEMP_DIR/json-file
+ln -s ${EXPO_UNIVERSE_DIR}/libraries/osascript $TEMP_DIR/osascript
+ln -s ${EXPO_UNIVERSE_DIR}/libraries/schemer $TEMP_DIR/schemer
+ln -s ${EXPO_UNIVERSE_DIR}/package.json $TEMP_DIR/universe-package.json
+
+# generate dynamic macros
+mkdir -p $TEMP_DIR/android/expoview/src/main/java/host/exp/exponent/generated/
+cd $TEMP_DIR/tools-public
+./generate-dynamic-macros-android.sh
+rm -rf $TEMP_DIR/secrets
+
+cd $TEMP_DIR; tar -czhf $ARTIFACTS_DIR/android-shell-builder.tar.gz .
+rm -rf $TEMP_DIR

scripts/setConfigurations.js

@@ -1,13 +1,12 @@
-
+import { env } from 'turtle/utils/env';
+import { connect, RedisClient } from 'turtle/utils/redis';
 import { PLATFORMS } from 'turtle/constants';
-import logger from 'turtle/logger';
 import {
   createConfigurationsKey,
   createDefaultConfigurationKey,
   HIGH_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
+  NORMAL_CONFIGURATION,
 } from 'turtle/utils/priorities';
-import { connect, RedisClient } from 'turtle/utils/redis';
 
 const TIMEOUT = 3000;
 
@@ -26,34 +25,36 @@ const TIMEOUT = 3000;
 // they should come back to using configurations no. 1 and 2.
 // If there are no more available configurations from the table, the default configuration is used.
 
+
 const AVAILABLE_CONFIGURATIONS_IOS = [
-  HIGH_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
+  NORMAL_CONFIGURATION,
+  NORMAL_CONFIGURATION,
   HIGH_CONFIGURATION,
+  NORMAL_CONFIGURATION,
 ];
 
 const AVAILABLE_CONFIGURATIONS_ANDROID = [
-  HIGH_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
+  NORMAL_CONFIGURATION,
+  NORMAL_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
+  NORMAL_CONFIGURATION,
+  NORMAL_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
+  NORMAL_CONFIGURATION,
+  NORMAL_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
+  NORMAL_CONFIGURATION,
+  NORMAL_CONFIGURATION,
 ];
 
+
 const DEFAULT_CONFIGURATION = HIGH_CONFIGURATION;
 
 async function run() {
@@ -67,8 +68,6 @@ async function run() {
   redis.disconnect();
 }
 
-if (require.main === module) {
-  run()
-    .then(() => logger.info('All done'))
-    .catch((err) => logger.error('Error:', err));
-}
+run()
+  .then(() => console.log('All done'))
+  .catch(err => console.error('Error:', err));

scripts/setWorkingdirMode.sh

@@ -36,7 +36,7 @@ setup_local() {
   mkdir -p $WORKINGDIR
   echo "$WORKINGDIR directory created"
 
-  LINKS=("template-files" "package.json" "android" "expokit-npm-package" "packages" "client-builds" "cpp" "exponent-view-template" "ios" "shellAppBase-builds" "shellAppWorkspaces" "node_modules")
+  LINKS=("template-files" "package.json" "android" "expokit-npm-package" "packages" "client-builds" "cpp" "exponent-view-template" "ios" "shellAppBase-builds" "shellAppWorkspaces")
 
   for LINKNAME in ${LINKS[@]}; do
     ensure_link_exists "$EXPO_UNIVERSE_DIR/exponent/$LINKNAME" "$WORKINGDIR/$LINKNAME"
@@ -55,14 +55,6 @@ setup_remote() {
       $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
     fi
   fi
-  
-  for sdk in $ROOT_DIR/workingdir/android/* ; do
-    if [[ -f "$sdk/universe-package.json" ]]; then
-      mv $sdk/package.json $sdk/exponent-package.json
-      mv $sdk/universe-package.json $sdk/package.json
-      cd $sdk && yarn
-    fi
-  done
 }
 
 switch_local() {

src/__smoke__/smoke.test.ts

@@ -34,7 +34,7 @@ async function performSmokeTest(recording: string) {
     });
   }
 
-  await jobManager.doJob();
+  await jobManager.doJob(logger);
   if (!fixtureExists) {
     await fs.writeFile(recording, JSON.stringify(nock.recorder.play()));
   }

src/aws/cloudwatch/index.ts

@@ -11,7 +11,6 @@ import {
   MetricRegistrationObject,
 } from 'turtle/aws/cloudwatch/types';
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 
 interface IState {
   metrics: IMetric[];
@@ -38,12 +37,12 @@ const cloudWatch = new aws.CloudWatch({
   secretAccessKey: config.aws.secretAccessKey,
 });
 
-export function init() {
+export function init(logger: any) {
   if (config.cloudwatch.disabled) {
     return;
   }
 
-  state.intervalId = setInterval(_dumpMetrics, config.cloudwatch.intervalMs);
+  state.intervalId = setInterval(_dumpMetrics(logger), config.cloudwatch.intervalMs);
 }
 
 export function addMetricData({
@@ -111,23 +110,25 @@ function _calculateGroupHash(metricName: string, extraDimensions: any[]) {
   return `${metricName}${dimString}`;
 }
 
-function _dumpMetrics() {
-  if (!state.metrics.length && Object.keys(state.registeredMetrics).length < 0) {
-    // empty metrics list, skipping
-    return;
-  }
+function _dumpMetrics(logger: any) {
+  return () => {
+    if (!state.metrics.length && Object.keys(state.registeredMetrics).length < 0) {
+      // empty metrics list, skipping
+      return;
+    }
 
-  logger.trace(
-    '[cloudwatch] lenghts for state.metrics=%d, state.registeredMetrics=%d',
-    state.metrics.length,
-    Object.keys(state.registeredMetrics).length,
-  );
+    logger.trace(
+      '[cloudwatch] lengths for state.metrics=%d, state.registeredMetrics=%d',
+      state.metrics.length,
+      Object.keys(state.registeredMetrics).length,
+    );
 
-  _addEmptyMetrics(state.metrics);
-  const formattedMetrics = _formatMetrics(state.metrics);
-  _pushMetrics(formattedMetrics);
+    _addEmptyMetrics(state.metrics);
+    const formattedMetrics = _formatMetrics(state.metrics);
+    _pushMetrics(formattedMetrics, logger);
 
-  state.metrics = [];
+    state.metrics = [];
+  };
 }
 
 function _addEmptyMetrics(metrics: IMetric[]) {
@@ -174,7 +175,7 @@ function _reduceMetrics(metrics: IMetric[]) {
   );
 }
 
-function _pushMetrics(metrics: IMetricsChunk[]) {
+function _pushMetrics(metrics: IMetricsChunk[], logger: any) {
   logger.trace('[cloudwatch] pushing metrics', JSON.stringify(metrics, null, '\t'));
   metrics.forEach((metricsChunk) => {
     cloudWatch.putMetricData(metricsChunk, (err) => {

src/aws/sqs.ts

@@ -5,7 +5,6 @@ import _ from 'lodash';
 import { OUTPUT_QUEUE_URL, QUEUE_URL } from 'turtle/aws/utils';
 import config from 'turtle/config';
 import { JOB_UPDATE_TYPE } from 'turtle/constants/build';
-import logger from 'turtle/logger';
 import { getCurrentJobId } from 'turtle/turtleContext';
 
 const sqs = new AWS.SQS({
@@ -16,7 +15,7 @@ const sqs = new AWS.SQS({
 
 const VISIBILITY_TIMEOUT_SEC = 30;
 
-export async function receiveMessage(priority: string) {
+export async function receiveMessage(priority: string, logger: any) {
   const params = {
     MaxNumberOfMessages: 1,
     QueueUrl: QUEUE_URL(priority),
@@ -64,7 +63,7 @@ export async function changeMessageVisibility(priority: string, receiptHandle: s
 // Every VISIBILITY_TIMEOUT_SEC / 3 seconds we are telling AWS SQS
 // that we're still processing the message, so it does not
 // send the build job to another turtle agent
-export function changeMessageVisibilityRecurring(priority: string, receiptHandle: string, jobId: string) {
+export function changeMessageVisibilityRecurring(priority: string, receiptHandle: string, jobId: string, logger: any) {
   return setInterval(() => {
     if (getCurrentJobId() === jobId) {
       changeMessageVisibility(priority, receiptHandle).catch((err) => {
@@ -74,7 +73,7 @@ export function changeMessageVisibilityRecurring(priority: string, receiptHandle
   }, (VISIBILITY_TIMEOUT_SEC * 1000) / 3);
 }
 
-export async function sendMessage(jobId: string, status: JOB_UPDATE_TYPE, data = {}) {
+export async function sendMessage(jobId: string, status: JOB_UPDATE_TYPE, data = {}, logger: any) {
   const params = {
     MessageBody: JSON.stringify({ jobId, status, ...data }),
     QueueUrl: OUTPUT_QUEUE_URL(),

src/bin/commands/build/index.ts

@@ -2,8 +2,8 @@ import { default as buildAndroid } from 'turtle/bin/commands/build/android';
 import { default as buildIOS } from 'turtle/bin/commands/build/ios';
 
 function createCommand(builder: any) {
-  return (program: any) => {
-    return builder(program, setCommonCommandOptions);
+  return (logger: any) => (program: any) => {
+    return builder(program, setCommonCommandOptions, logger);
   };
 }
 

src/bin/commands/index.ts

@@ -1,8 +1,9 @@
 import builders from 'turtle/bin/commands/build';
 import setup from 'turtle/bin/commands/setup';
 
-export const setupIOS = setup.ios;
-export const setupAndroid = setup.android;
-
-export const buildIOS = builders.ios;
-export const buildAndroid = builders.android;
+export default {
+  setupIOS: setup.ios,
+  setupAndroid: setup.android,
+  buildIOS: builders.ios,
+  buildAndroid: builders.android,
+};

src/bin/commands/setup.ts

@@ -4,9 +4,6 @@ import { ErrorWithCommandHelp } from 'turtle/bin/commands/ErrorWithCommandHelp';
 import { ErrorWithProgramHelp } from 'turtle/bin/commands/ErrorWithProgramHelp';
 import setup from 'turtle/bin/setup/setup';
 import { PLATFORMS } from 'turtle/constants';
-import logger from 'turtle/logger';
-
-const l = logger.child({ buildPhase: 'setting up environment' });
 
 function createSetupCommand(platform: string, os?: string) {
   return (program: any) => {
@@ -23,14 +20,16 @@ function createSetupCommand(platform: string, os?: string) {
   };
 }
 
-async function setupAction(program: any, cmd: any, platform: string, os?: string) {
+async function setupAction(program: any, cmd: any, platform: string, logger: any, os?: string) {
   try {
     if (os && process.platform !== os) {
       throw new Error('We don\'t support running standalone app builds for this platform on your operating system');
     }
 
     const { sdkVersion } = cmd;
-    await setup(platform, sdkVersion);
+    const l = logger.child({ buildPhase: 'setting up environment' });
+
+    await setup(platform, l, sdkVersion);
     l.info('it\'s all set!');
   } catch (err) {
     logger.error({ err }, `Failed to setup environment for ${platform} builds`);

src/bin/index.ts

@@ -4,7 +4,7 @@ import program, { Command } from 'commander';
 import fs from 'fs-extra';
 import { LoggerDetach, ModuleVersion } from 'xdl';
 
-import * as commands from 'turtle/bin/commands';
+import commands from 'turtle/bin/commands';
 import logger from 'turtle/logger';
 
 const { name, version } = JSON.parse(fs.readFileSync(path.join(__dirname, '../../package.json'), 'utf-8'));
@@ -33,7 +33,7 @@ export function run(programName: string) {
 
 async function runAsync(programName: string) {
   program.version(version);
-  Object.values(commands).forEach((command) => registerCommand(program, command));
+  Object.values(commands).map((command) => command(logger)).forEach((command) => registerCommand(program, command));
   program.parse(process.argv);
 
   const subCommand = process.argv[2];

src/bin/setup/android/index.ts

@@ -12,7 +12,6 @@ import { IToolDefinition } from 'turtle/bin/setup/utils/toolsDetector';
 import { formatShellAppDirectory } from 'turtle/builders/utils/android/workingdir';
 import config from 'turtle/config';
 import { PLATFORMS } from 'turtle/constants';
-import logger from 'turtle/logger';
 
 const which = util.promisify(_which);
 const REQUIRED_TOOLS: IToolDefinition[] = [
@@ -40,25 +39,24 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
     },
   },
 ];
-const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
-const l = logger.child(LOGGER_FIELDS);
 
-export default async function setup(sdkVersion?: string) {
-  await checkSystem(REQUIRED_TOOLS);
-  await prepareAndroidEnv();
+export default async function setup(logger: any, sdkVersion?: string) {
+  await checkSystem(REQUIRED_TOOLS, logger);
+  await prepareAndroidEnv(logger);
   if (sdkVersion) {
     await ensureShellAppIsPresent(
       sdkVersion,
       { formatShellAppDirectory, formatShellAppTarballUriPath },
+      logger,
       _shellAppPostDownloadAction,
     );
   }
 }
 
-async function prepareAndroidEnv() {
+async function prepareAndroidEnv(logger: any) {
   await fs.ensureDir(config.directories.androidDependenciesDir);
-  const sdkConfig = await ensureAndroidSDKIsPresent();
-  const ndkConfig = await ensureAndroidNDKIsPresent();
+  const sdkConfig = await ensureAndroidSDKIsPresent(logger);
+  const ndkConfig = await ensureAndroidNDKIsPresent(logger);
   _setEnvVars({ ...sdkConfig.envVars, ...ndkConfig.envVars });
   _alterPath([...sdkConfig.path, ...sdkConfig.path]);
 }
@@ -67,21 +65,23 @@ function formatShellAppTarballUriPath(sdkMajorVersion: string) {
   return path.join(config.directories.shellTarballsDir, PLATFORMS.ANDROID, `sdk${sdkMajorVersion}`);
 }
 
-async function _shellAppPostDownloadAction(workingdir: string) {
+async function _shellAppPostDownloadAction(workingdir: string, logger: any) {
   const inWorkingdir = (filename: string) => path.join(workingdir, filename);
 
   await fs.move(inWorkingdir('package.json'), inWorkingdir('exponent-package.json'));
   await fs.move(inWorkingdir('universe-package.json'), inWorkingdir('package.json'));
-  await _installNodeModules(workingdir);
+  await _installNodeModules(workingdir, logger);
   await fs.move(inWorkingdir('package.json'), inWorkingdir('universe-package.json'));
   await fs.move(inWorkingdir('exponent-package.json'), inWorkingdir('package.json'));
 
   // TODO: remove following lines after upgrading android shell app
   const toolsPublicDir = path.join(workingdir, 'tools-public');
-  await _installNodeModules(toolsPublicDir);
+  await _installNodeModules(toolsPublicDir, logger);
 }
 
-async function _installNodeModules(cwd: string) {
+async function _installNodeModules(cwd: string, logger: any) {
+  const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
+  const l = logger.child(LOGGER_FIELDS);
   l.info(`installing dependencies in ${cwd} directory...`);
   const command = await _shouldUseYarnOrNpm();
   await ExponentTools.spawnAsyncThrowError(command, ['install'], {

src/bin/setup/android/ndk.ts

@@ -7,20 +7,17 @@ import fs from 'fs-extra';
 import { formatArtifactDownloadPath } from 'turtle/bin/setup/utils/common';
 import download from 'turtle/bin/setup/utils/downloader';
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 
 const ANDROID_NDK_URL = `https://dl.google.com/android/repository/android-ndk-r10e-${os.platform()}-x86_64.zip`;
 
-const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
-const l = logger.child(LOGGER_FIELDS);
-
-export default async function ensureAndroidNDKIsPresent() {
+export default async function ensureAndroidNDKIsPresent(logger: any) {
   const androidNdkDir = path.join(config.directories.androidDependenciesDir, 'ndk');
   if (!(await fs.pathExists(androidNdkDir))) {
     await fs.ensureDir(androidNdkDir);
     const androidNdkDownloadPath = formatArtifactDownloadPath(ANDROID_NDK_URL);
 
     try {
+      const l = logger.child({ buildPhase: 'setting up environment' });
       l.info('Downloading Android NDK');
       await fs.ensureDir(config.directories.artifactsDir);
       await download(ANDROID_NDK_URL, androidNdkDownloadPath);

src/bin/setup/android/sdk.ts

@@ -8,20 +8,17 @@ import { ExponentTools } from 'xdl';
 import { formatArtifactDownloadPath } from 'turtle/bin/setup/utils/common';
 import download from 'turtle/bin/setup/utils/downloader';
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 
 const ANDROID_SDK_URL = 'https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip';
-
 const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
-const l = logger.child(LOGGER_FIELDS);
-
-export default async function ensureAndroidSDKIsPresent() {
+export default async function ensureAndroidSDKIsPresent(logger: any) {
   const androidSdkDir = path.join(config.directories.androidDependenciesDir, 'sdk');
   if (!(await fs.pathExists(androidSdkDir))) {
     await fs.ensureDir(androidSdkDir);
     const androidSdkDownloadPath = formatArtifactDownloadPath(ANDROID_SDK_URL);
 
     try {
+      const l = logger.child(LOGGER_FIELDS);
       l.info('Downloading Android SDK');
       await fs.ensureDir(config.directories.artifactsDir);
       await download(ANDROID_SDK_URL, androidSdkDownloadPath);

src/bin/setup/ios.ts

@@ -1,7 +1,4 @@
-import last from 'lodash/last';
-import trim from 'lodash/trim';
 import path from 'path';
-import semver from 'semver';
 
 import { ExponentTools } from 'xdl';
 
@@ -19,21 +16,6 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
   {
     command: 'fastlane',
     missingDescription: 'Please check https://docs.fastlane.tools/getting-started/ios/setup/',
-    versionCheckFn: async () => {
-      const MINIMAL_VERSION = '2.99.0';
-
-      const { stdout } = await ExponentTools.spawnAsyncThrowError(
-        'fastlane',
-        ['--version'],
-        { stdio: 'pipe', env: { ...process.env, FASTLANE_SKIP_UPDATE_CHECK: '1', LC_ALL: 'en_US.UTF-8' } },
-      );
-      const fastlaneVersion = parseFastlaneVersion(stdout);
-      if (fastlaneVersion && !semver.satisfies(fastlaneVersion, `>= ${MINIMAL_VERSION}`)) {
-        throw new Error(
-          `Your fastlane on version ${fastlaneVersion}. Please upgrade it to at least ${MINIMAL_VERSION}.`,
-        );
-      }
-    },
   },
   {
     command: 'xcode',
@@ -52,24 +34,16 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
   },
 ];
 
-export default async function setup(sdkVersion?: string) {
-  await checkSystem(REQUIRED_TOOLS);
+export default async function setup(logger: any, sdkVersion?: string) {
+  await checkSystem(REQUIRED_TOOLS, logger);
   if (sdkVersion) {
     await ensureShellAppIsPresent(sdkVersion, {
       formatShellAppDirectory,
       formatShellAppTarballUriPath,
-    });
+    }, logger);
   }
 }
 
 function formatShellAppTarballUriPath(sdkMajorVersion: string) {
   return path.join(config.directories.shellTarballsDir, PLATFORMS.IOS, `sdk${sdkMajorVersion}`);
 }
-
-function parseFastlaneVersion(stdout: string) {
-  const lastLine = last(trim(stdout).split('\n'));
-  if (!lastLine) {
-    return null;
-  }
-  return last(lastLine.split(' ')) || null;
-}

src/bin/setup/setup.ts

@@ -4,15 +4,15 @@ import setupAndroid from 'turtle/bin/setup/android';
 import setupIos from 'turtle/bin/setup/ios';
 import { PLATFORMS } from 'turtle/constants';
 
-export default async function setup(platform: string, sdkVersion?: string) {
+export default async function setup(platform: string, logger: any, sdkVersion?: string) {
   if (sdkVersion && !isSemver(sdkVersion)) {
     throw new Error('SDK version is not valid.');
   }
 
   if (platform === PLATFORMS.IOS) {
-    return await setupIos(sdkVersion);
+    return await setupIos(logger, sdkVersion);
   } else if (platform === PLATFORMS.ANDROID) {
-    return await setupAndroid(sdkVersion);
+    return await setupAndroid(logger, sdkVersion);
   } else {
     throw new Error('This should never happen :(');
   }

src/bin/setup/utils/common.ts

@@ -8,50 +8,54 @@ import download from 'turtle/bin/setup/utils/downloader';
 import { ensureToolsAreInstalled, IToolDefinition } from 'turtle/bin/setup/utils/toolsDetector';
 import { IShellAppDirectoryConfig } from 'turtle/builders/utils/workingdir';
 import config from 'turtle/config';
-import logger from 'turtle/logger';
-
-const l = logger.child({ buildPhase: 'setting up environment' });
 
 interface IShellAppFormaters {
   formatShellAppDirectory: (config: IShellAppDirectoryConfig) => string;
   formatShellAppTarballUriPath: (sdkMajorVersion: string) => string;
 }
 
-type PostDownloadAction = (workingdir: string) => void;
+type PostDownloadAction = (workingdir: string, logger: any) => void;
 
-export async function checkSystem(requiredTools: IToolDefinition[]) {
-  await ensureToolsAreInstalled(requiredTools);
+export async function checkSystem(requiredTools: IToolDefinition[], logger: any) {
+  await ensureToolsAreInstalled(requiredTools, logger);
 }
 
 export async function ensureShellAppIsPresent(
   sdkVersion: string,
   formatters: IShellAppFormaters,
+  logger: any,
   postDownloadAction?: PostDownloadAction,
 ) {
   const workingdir = formatters.formatShellAppDirectory({ sdkVersion });
+  const l = logger.child({ buildPhase: 'setting up environment' });
   if (await fs.pathExists(workingdir)) {
     return;
   }
   l.info(`shell app for SDK ${sdkVersion} doesn't exist, downloading...`);
-  await _downloadShellApp(sdkVersion, workingdir, formatters);
+  await _downloadShellApp(sdkVersion, workingdir, formatters, l);
   if (postDownloadAction) {
-    await postDownloadAction(workingdir);
+    await postDownloadAction(workingdir, logger);
   }
 }
 
-async function _downloadShellApp(sdkVersion: string, targetDirectory: string, formatters: IShellAppFormaters) {
+async function _downloadShellApp(
+  sdkVersion: string,
+  targetDirectory: string,
+  formatters: IShellAppFormaters,
+  logger: any,
+) {
   const shellAppTarballS3Uri = await _readShellAppTarballS3Uri(sdkVersion, formatters);
   const tarballDownloadTargetPath = formatArtifactDownloadPath(shellAppTarballS3Uri);
   await fs.ensureDir(config.directories.artifactsDir);
   await download(shellAppTarballS3Uri, tarballDownloadTargetPath);
-  l.info('shell app has been downloaded');
+  logger.info('shell app has been downloaded');
   await fs.ensureDir(targetDirectory);
-  l.info('extracting shell app (this may take a while)...');
+  logger.info('extracting shell app (this may take a while)...');
   await tar.x({
     file: tarballDownloadTargetPath,
     C: targetDirectory,
   });
-  l.info('shell app extracted');
+  logger.info('shell app extracted');
 }
 
 export function formatArtifactDownloadPath(uri: string) {

src/bin/setup/utils/toolsDetector.ts

@@ -1,21 +1,17 @@
 import util from 'util';
 import _which from 'which';
 
-import logger from 'turtle/logger';
-
 const which = util.promisify(_which);
-const l = logger.child({ buildPhase: 'setting up environment' });
 
 export interface IToolDefinition {
   command: string;
   missingDescription: string;
   testFn?: () => Promise<boolean>;
-  versionCheckFn?: () => void;
 }
 
-export async function ensureToolsAreInstalled(tools: IToolDefinition[]) {
+export async function ensureToolsAreInstalled(tools: IToolDefinition[], logger: any) {
   let isAnyToolMissing = false;
-  for (const { command, missingDescription, testFn, versionCheckFn } of tools) {
+  for (const { command, missingDescription, testFn } of tools) {
     try {
       if (testFn) {
         if (!await testFn()) {
@@ -25,20 +21,13 @@ export async function ensureToolsAreInstalled(tools: IToolDefinition[]) {
         await which(command);
       }
     } catch (err) {
+      const l = logger.child({ buildPhase: 'setting up environment' });
       isAnyToolMissing = true;
       if (!testFn) {
         l.error({ err }, `${command} is missing in your $PATH`);
       }
       l.error({ err }, missingDescription);
     }
-    if (versionCheckFn) {
-      try {
-        await versionCheckFn();
-      } catch (err) {
-        isAnyToolMissing = true;
-        l.error({ err }, `wrong version of ${command} installed`);
-      }
-    }
   }
   if (isAnyToolMissing) {
     throw new Error('Tools required to perform standalone app build are missing. Please install them.');

src/bin/utils/builder.ts

@@ -5,10 +5,10 @@ import { ErrorWithProgramHelp } from 'turtle/bin/commands/ErrorWithProgramHelp';
 import setup from 'turtle/bin/setup/setup';
 import * as ProjectUtils from 'turtle/bin/utils/project';
 import * as UserUtils from 'turtle/bin/utils/user';
-import logger from 'turtle/logger';
 import { sanitizeJob } from 'turtle/validator';
 
 export function createBuilderAction({
+  logger,
   program,
   command,
   prepareCredentials,
@@ -62,7 +62,7 @@ export function createBuilderAction({
 
       const appJSON = await ProjectUtils.loadAppJSON(projectDirArg, cmd.config);
       const sdkVersion = _.get(appJSON, 'expo.sdkVersion');
-      await setup(platform, sdkVersion);
+      await setup(platform, logger, sdkVersion);
       const credentials = await prepareCredentials(cmd);
       const rawJob = {
         ...buildJobObject(appJSON, args, credentials),

src/builders/android.ts

@@ -2,7 +2,6 @@ import path from 'path';
 
 import fs from 'fs-extra';
 import _ from 'lodash';
-import semver from 'semver';
 import uuidv4 from 'uuid/v4';
 import { AndroidShellApp, ImageUtils } from 'xdl';
 
@@ -10,22 +9,21 @@ import getOrCreateCredentials from 'turtle/builders/utils/android/credentials';
 import { formatShellAppDirectory } from 'turtle/builders/utils/android/workingdir';
 import * as commonUtils from 'turtle/builders/utils/common';
 import * as imageHelpers from 'turtle/builders/utils/image';
-import { resolveNativeModules } from 'turtle/builders/utils/unimodules';
 import { uploadBuildToS3 } from 'turtle/builders/utils/uploader';
 import { ensureCanBuildSdkVersion } from 'turtle/builders/utils/version';
 import config from 'turtle/config';
 import { IAndroidCredentials, IJob, IJobResult } from 'turtle/job';
-import logger from 'turtle/logger';
 
-export default async function buildAndroid(jobData: IJob): Promise<IJobResult> {
+export default async function buildAndroid(jobData: IJob, logger: any): Promise<IJobResult> {
   await ensureCanBuildSdkVersion(jobData);
-  const credentials = await getOrCreateCredentials(jobData);
-  const apkFilePath = await runShellAppBuilder(jobData, credentials);
+  const credentials = await getOrCreateCredentials(jobData, logger);
+  const apkFilePath = await runShellAppBuilder(jobData, credentials, logger);
   const randomHex = uuidv4().replace(/-/g, '');
   const s3Filename = `${jobData.experienceName}-${randomHex}-signed.apk`;
   const s3FileKey = `android/${s3Filename}`;
   const fakeUploadFilename = s3Filename.replace('/', '\\');
   const artifactUrl = await uploadBuildToS3({
+    logger,
     uploadPath: apkFilePath,
     s3FileKey,
     ...config.builder.fakeUpload && {
@@ -42,6 +40,7 @@ export default async function buildAndroid(jobData: IJob): Promise<IJobResult> {
 async function runShellAppBuilder(
   jobData: IJob,
   credentials: IAndroidCredentials,
+  logger: any,
 ): Promise<string> {
   const { temporaryFilesRoot } = config.directories;
   await fs.ensureDir(temporaryFilesRoot);
@@ -68,11 +67,6 @@ async function runShellAppBuilder(
   const sdkVersion = _.get(manifest, 'sdkVersion', sdkVersionFromJob);
   const workingDir = formatShellAppDirectory({ sdkVersion });
 
-  logger.info({ buildPhase: 'resolve native modules' }, 'Resolving universal modules dependencies');
-  const enabledModules = semver.satisfies(sdkVersion, '>= 33.0.0')
-    ? await resolveNativeModules(workingDir, manifest.dependencies)
-    : null;
-
   try {
     await AndroidShellApp.createAndroidShellAppAsync({
       url: commonUtils.getExperienceUrl(jobData),
@@ -86,10 +80,9 @@ async function runShellAppBuilder(
       releaseChannel: jobConfig.releaseChannel,
       workingDir,
       outputFile: outputFilePath,
-      modules: enabledModules,
     });
   } catch (err) {
-    commonUtils.logErrorOnce(err);
+    commonUtils.logErrorOnce(err, logger);
     throw err;
   } finally {
     if (!config.builder.skipCleanup) {

src/builders/index.ts

@@ -1,7 +1,10 @@
+import { IJob, IJobResult } from 'turtle/job';
 import buildAndroid from './android';
 import buildIOS from './ios';
 
-export default {
+const builders: { [index: string]: (job: IJob, logger: any) => Promise<IJobResult>} = {
   ios: buildIOS,
   android: buildAndroid,
 };
+
+export default builders;

src/builders/ios/archive.ts

@@ -7,7 +7,6 @@ import * as fileUtils from 'turtle/builders/utils/file';
 import * as keychain from 'turtle/builders/utils/ios/keychain';
 import runShellAppBuilder from 'turtle/builders/utils/ios/shellAppBuilder';
 import { IJob } from 'turtle/job';
-import logger from 'turtle/logger/index';
 
 export default async function buildArchive(ctx: IContext, job: IJob) {
   let keychainInfo;
@@ -19,7 +18,7 @@ export default async function buildArchive(ctx: IContext, job: IJob) {
     const manifest = await runShellAppBuilder(ctx, job);
     await buildAndSignIPA(ctx, job, keychainInfo.path, manifest);
   } catch (err) {
-    logErrorOnce(err);
+    logErrorOnce(err, ctx.logger);
     throw err;
   } finally {
     if (keychainInfo) {
@@ -29,8 +28,8 @@ export default async function buildArchive(ctx: IContext, job: IJob) {
 }
 
 async function buildAndSignIPA(ctx: IContext, job: IJob, keychainPath: string, manifest: any) {
-  const l = logger.child({ buildPhase: 'building and signing IPA' });
-  l.info('building and signing IPA');
+  ctx.logger = ctx.logger.child({ buildPhase: 'building and signing IPA' });
+  ctx.logger.info('building and signing IPA');
 
   const {
     credentials: { provisioningProfile, certPassword, teamId },
@@ -42,7 +41,7 @@ async function buildAndSignIPA(ctx: IContext, job: IJob, keychainPath: string, m
 
   const { provisioningProfilePath } = ctx;
   await fileUtils.writeBase64ToBinaryFile(provisioningProfilePath, provisioningProfile as string);
-  l.info('saved provisioning profile to temporary path');
+  ctx.logger.info('saved provisioning profile to temporary path');
 
   const ipaBuilder = createIPABuilder({
     keychainPath,
@@ -57,5 +56,5 @@ async function buildAndSignIPA(ctx: IContext, job: IJob, keychainPath: string, m
   });
   await ipaBuilder.build();
 
-  l.info(`done building and signing IPA`);
+  ctx.logger.info(`done building and signing IPA`);
 }

src/builders/ios/context.ts

@@ -15,6 +15,7 @@ export interface IContext {
   baseArchiveDir: string;
   buildDir: string;
   fakeUploadBuildPath?: string;
+  logger: any;
   outputPath: string;
   provisioningProfilePath: string;
   provisioningProfileDir: string;
@@ -29,7 +30,7 @@ export interface IContext {
 const { EXPOKIT_APP, EXPONENT_APP } = IosShellApp;
 const { BUILD_TYPES } = IOS;
 
-export function createBuilderContext(job: IJob): IContext {
+export function createBuilderContext(job: IJob, logger: any): IContext {
   const { join } = path;
   const appUUID = uuidv4();
   const {
@@ -47,6 +48,7 @@ export function createBuilderContext(job: IJob): IContext {
     appDir: join(config.directories.temporaryFilesRoot, appUUID),
     appUUID,
     workingDir,
+    logger,
   };
   context.buildDir = join(context.appDir, 'build');
   context.provisioningProfileDir = join(context.appDir, 'provisioning');

src/builders/ios/index.ts

@@ -14,12 +14,12 @@ import { IJob, IJobResult } from 'turtle/job';
 
 const { BUILD_TYPES } = IOS;
 
-export default async function iosBuilder(job: IJob): Promise<IJobResult> {
+export default async function iosBuilder(job: IJob, logger: any): Promise<IJobResult> {
   if (job.config.buildType !== BUILD_TYPES.CLIENT) {
     await ensureCanBuildSdkVersion(job);
   }
 
-  const ctx = createBuilderContext(job);
+  const ctx = createBuilderContext(job, logger);
 
   try {
     await initBuilder(ctx);
@@ -27,7 +27,7 @@ export default async function iosBuilder(job: IJob): Promise<IJobResult> {
     const { buildType } = job.config;
 
     if (buildType === BUILD_TYPES.CLIENT) {
-      await prepareAdHocBuildCredentials(job);
+      await prepareAdHocBuildCredentials(job, ctx.logger);
     }
 
     if ([BUILD_TYPES.ARCHIVE, BUILD_TYPES.CLIENT].includes(buildType!)) {
@@ -41,7 +41,7 @@ export default async function iosBuilder(job: IJob): Promise<IJobResult> {
     const artifactUrl = await uploadBuildToS3(ctx);
     return { artifactUrl };
   } catch (err) {
-    logErrorOnce(err);
+    logErrorOnce(err, ctx.logger);
     throw err;
   } finally {
     await cleanup(ctx);

src/builders/ios/simulator.ts

@@ -1,11 +1,10 @@
 import { IContext } from 'turtle/builders/ios/context';
 import runShellAppBuilder from 'turtle/builders/utils/ios/shellAppBuilder';
 import { IJob } from 'turtle/job';
-import logger from 'turtle/logger';
 
 export default async function buildSimulator(ctx: IContext, job: IJob) {
-  const l = logger.child({ buildPhase: 'running simulator builder' });
-  l.info('running simulator build');
+  ctx.logger = ctx.logger.child({ buildPhase: 'running simulator builder' });
+  ctx.logger.info('running simulator build');
   await runShellAppBuilder(ctx, job);
-  l.info('build complete');
+  ctx.logger.info('build complete');
 }

src/builders/utils/android/credentials.ts

@@ -7,10 +7,9 @@ import { Credentials } from 'xdl';
 import * as sqs from 'turtle/aws/sqs';
 import { UPDATE_CREDENTIALS } from 'turtle/constants/build';
 import { IAndroidCredentials, IJob } from 'turtle/job';
-import logger from 'turtle/logger';
 import { isOffline } from 'turtle/turtleContext';
 
-async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredentials> {
+async function getOrCreateCredentials(jobData: IJob, logger: any): Promise<IAndroidCredentials> {
   if (!jobData.credentials) {
     const l = logger.child({ buildPhase: 'generating keystore' });
     const credentials: any = {};
@@ -44,7 +43,7 @@ async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredential
     } else {
       await sqs.sendMessage(jobData.id, UPDATE_CREDENTIALS, {
         ...credentials,
-      });
+      }, l);
       l.info('Keystore sent to storage successfully');
     }
     await fs.unlink(keystoreFilename);

src/builders/utils/common.ts

@@ -2,7 +2,6 @@ import * as url from 'url';
 
 import config from 'turtle/config';
 import { IJob } from 'turtle/job';
-import logger from 'turtle/logger';
 
 export function getExperienceUrl(job: IJob) {
   const { experienceName, config: jobConfig } = job;
@@ -18,7 +17,7 @@ export function getExperienceUrl(job: IJob) {
 
 const alreadyLoggedError = Symbol('alreadyLoggedError');
 
-export function logErrorOnce(err: any) {
+export function logErrorOnce(err: any, logger: any) {
   if (!err[alreadyLoggedError]) {
     logger.error(err.stack);
     err[alreadyLoggedError] = true;

src/builders/utils/ios/adhocBuild.ts

@@ -9,13 +9,12 @@ import * as sqs from 'turtle/aws/sqs';
 import BuildError, { BuildErrorReason } from 'turtle/builders/BuildError';
 import { UPDATE_CREDENTIALS } from 'turtle/constants/build';
 import { IJob } from 'turtle/job';
-import logger from 'turtle/logger';
 import { isOffline } from 'turtle/turtleContext';
 
 // tslint:disable-next-line:no-var-requires
 const travelingFastlane = process.platform === 'darwin' ? require('@expo/traveling-fastlane-darwin')() : null;
 
-async function prepareAdHocBuildCredentials(job: IJob) {
+async function prepareAdHocBuildCredentials(job: IJob, logger: any) {
   if (process.platform !== 'darwin') {
     throw new Error('This function should be called only on macOS!');
   }
@@ -59,7 +58,7 @@ async function prepareAdHocBuildCredentials(job: IJob) {
       await sqs.sendMessage(job.id, UPDATE_CREDENTIALS, {
         provisioningProfileId: credentials.provisioningProfileId,
         provisioningProfile: credentials.provisioningProfile,
-      });
+      }, logger);
       logger.info('Ad Hoc provisioning profile sent to storage successfully');
     }
   } catch (e) {

src/builders/utils/ios/keychain.ts

@@ -2,14 +2,13 @@ import { IosKeychain } from 'xdl';
 
 import { IContext } from 'turtle/builders/ios/context';
 import * as fileUtils from 'turtle/builders/utils/file';
-import logger from 'turtle/logger';
 
 interface IKeychain {
   path: string;
 }
 
 export async function create(ctx: IContext): Promise<IKeychain> {
-  const l = logger.child({ buildPhase: 'creating keychain' });
+  const l = ctx.logger.child({ buildPhase: 'creating keychain' });
   try {
     l.info('creating keychain...');
     const keychainInfo = await IosKeychain.createKeychain(ctx.appUUID, false);
@@ -22,7 +21,7 @@ export async function create(ctx: IContext): Promise<IKeychain> {
 }
 
 export async function remove(ctx: IContext, keychainPath: string) {
-  const l = logger.child({ buildPhase: 'deleting keychain' });
+  const l = ctx.logger.child({ buildPhase: 'deleting keychain' });
   try {
     l.info('delete keychain...');
     const keychainInfo = await IosKeychain.deleteKeychain({
@@ -49,7 +48,7 @@ export async function importCert(
     certPassword,
   }: { keychainPath: string; certP12: string; certPassword: string },
 ) {
-  const l = logger.child({ buildPhase: 'importing certificate into keychain' });
+  const l = ctx.logger.child({ buildPhase: 'importing certificate into keychain' });
   try {
     l.info('importing distribution certificate into keychain...');
     const { tempCertPath: certPath } = ctx;

src/builders/utils/ios/shellAppBuilder.ts

@@ -10,7 +10,6 @@ import * as commonUtils from 'turtle/builders/utils/common';
 import * as imageHelpers from 'turtle/builders/utils/image';
 import { IOS } from 'turtle/constants/index';
 import { IJob } from 'turtle/job';
-import logger from 'turtle/logger/index';
 
 const copyAsync = util.promisify(copy);
 
@@ -21,7 +20,7 @@ export default async function runShellAppBuilder(ctx: IContext, job: IJob): Prom
 
   await copyAsync(ctx.applicationFilesSrc, ctx.baseArchiveDir);
 
-  logger.info(
+  ctx.logger.info(
     { buildPhase: 'icons setup' },
     'ImageUtils: setting image functions to alternative sharp implementations',
   );
@@ -49,6 +48,6 @@ export default async function runShellAppBuilder(ctx: IContext, job: IJob): Prom
     });
   }
 
-  logger.info({ buildPhase: 'configuring NSBundle' }, 'configuring NSBundle...');
+  ctx.logger.info({ buildPhase: 'configuring NSBundle' }, 'configuring NSBundle...');
   return await IosShellApp.configureAndCopyArchiveAsync(shellAppParams);
 }

src/builders/utils/unimodules.ts

@@ -1,167 +0,0 @@
-import fs from 'fs-extra';
-import filter from 'lodash/filter';
-import keys from 'lodash/keys';
-import merge from 'lodash/merge';
-import values from 'lodash/values';
-import path from 'path';
-import logger from 'turtle/logger';
-
-export interface IUnimoduleEntry {
-  name: string;
-  version: string;
-  dirname: string;
-}
-
-interface IPackageInfo {
-  name: string;
-  version: string;
-  dirname: string; // could be different than name
-  dependencies: string[];
-}
-
-export async function resolveNativeModules(workingdir: string, packages?: string[]): Promise<IUnimoduleEntry[]> {
-  const resolver = new Resolver(workingdir);
-  await resolver.init();
-
-  if (!packages) {
-    resolver.addAll();
-  } else {
-    const corePackages = await readCorePackages(workingdir);
-    for (const pkg of corePackages) {
-      if (resolver.isUnimodule(pkg)) {
-        resolver.addModule(pkg);
-      }
-    }
-    for (const pkg of packages) {
-      if (resolver.isUnimodule(pkg)) {
-        resolver.addModule(pkg);
-      }
-    }
-  }
-
-  return resolver.getModules();
-}
-
-async function readCorePackages(workingdir: string) {
-  const pkgJsonPath = path.join(workingdir, 'node_modules', 'react-native-unimodules', 'package.json');
-  if (await fs.pathExists(pkgJsonPath)) {
-    const packageJson = JSON.parse(await fs.readFile(pkgJsonPath, 'utf-8'));
-    return keys(packageJson.dependencies);
-  } else {
-    logger.warn('No core packages detected');
-    return [];
-  }
-}
-
-interface IDependencyInfo {
-  [key: string]: IPackageInfo;
-}
-
-class Resolver {
-  private workingdir: string;
-  private modulesMap: {[key: string]: IUnimoduleEntry};
-  private dependencyMap: IDependencyInfo;
-
-  constructor(workingdir: string) {
-    this.workingdir = workingdir;
-    this.modulesMap = {};
-    this.dependencyMap = {};
-  }
-
-  public async init() {
-    this.dependencyMap = await generateDependenciesInfo(this.workingdir);
-  }
-
-  public isUnimodule(moduleName: string): boolean {
-    return !!this.dependencyMap[moduleName];
-  }
-
-  public addAll() {
-    for (const moduleName of Object.keys(this.dependencyMap)) {
-      this.addModule(moduleName);
-    }
-  }
-
-  public addModule(moduleName: string) {
-    if (this.dependencyMap[moduleName]) {
-      this.resolveModule(moduleName);
-    } else {
-      throw new Error('Attempting to add unknown module');
-    }
-  }
-
-  public getModules(): IUnimoduleEntry[] {
-     return values(this.modulesMap);
-  }
-
-  private async resolveModule(moduleName: string) {
-    if (this.modulesMap[moduleName]) {
-      return;
-    }
-    const pkg = this.dependencyMap[moduleName];
-    logger.info(`Adding ${moduleName}:${pkg.version}`);
-    this.modulesMap[moduleName] = {
-      name: pkg.name,
-      dirname: pkg.dirname,
-      version: pkg.version,
-    };
-    for (const dep of pkg.dependencies) {
-      this.resolveModule(dep);
-    }
-  }
-}
-
-async function generateDependenciesInfo(workingdir: string): Promise<IDependencyInfo> {
-  const dir = await fs.readdir(path.join(workingdir, 'packages'));
-  const map: IDependencyInfo = {};
-
-  // scrap directory for package metadata
-  for (const pkgName of dir) {
-    const pkg = await getPackage(workingdir, pkgName);
-    if (pkg) {
-      map[pkg.name] = pkg;
-    }
-  }
-
-  // handle @unimodules directory
-  const unimodulesDir = await fs.readdir(path.join(workingdir, 'packages', '@unimodules'));
-  for (const pkgName of unimodulesDir) {
-    const pkg = await getPackage(workingdir, path.join('@unimodules', pkgName));
-    if (pkg) {
-      map[pkg.name] = pkg;
-    }
-  }
-
-  // remove non unimodule dependencies
-  for (const pkg of Object.keys(map)) {
-    map[pkg].dependencies = filter(map[pkg].dependencies, (dep) => !!map[dep]);
-  }
-  return map;
-}
-
-async function getPackage(workingdir: string, pkgName: string): Promise<IPackageInfo | null> {
-  const pkgPath = path.join(workingdir, 'packages', pkgName);
-  const isAndroid = await fs.pathExists(path.join(pkgPath, 'android'));
-  const isIos = await fs.pathExists(path.join(pkgPath, 'ios'));
-  const isUnimodule = await fs.pathExists(path.join(pkgPath, 'unimodule.json'));
-  const hasPackageJson = await fs.pathExists(path.join(pkgPath, 'package.json'));
-  if (!isUnimodule || !hasPackageJson || (!isAndroid && !isIos)) {
-    return null;
-  }
-
-  const packageJson = JSON.parse(await fs.readFile(path.join(pkgPath, 'package.json'), 'utf-8'));
-
-  const pkgDeps = merge({}, // reading everything to make sure
-    packageJson.dependencies,
-    packageJson.peerDependencies,
-    packageJson.unimodulePeerDependencies,
-  );
-  const info: IPackageInfo = {
-    name: packageJson.name,
-    version: packageJson.version,
-    dirname: pkgName,
-    dependencies: Object.keys(pkgDeps),
-  };
-
-  return info;
-}

src/builders/utils/uploader.ts

@@ -2,24 +2,24 @@ import * as fs from 'fs-extra';
 
 import { uploadFile } from 'turtle/aws/s3';
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 
 interface IUploadCtx {
   fakeUploadBuildPath?: string;
   s3FileKey?: string;
   uploadPath: string;
+  logger: any;
 }
 
 export async function uploadBuildToS3(ctx: IUploadCtx) {
   if (config.builder.fakeUpload) {
-    const l = logger.child({ buildPhase: 'copying build artifact' });
+    const l = ctx.logger.child({ buildPhase: 'copying build artifact' });
     const { fakeUploadBuildPath, uploadPath } = ctx;
     l.info('copying build to fake upload directory');
     await fs.copy(uploadPath, fakeUploadBuildPath as string);
     l.info(`copied build to ${fakeUploadBuildPath}`);
     return fakeUploadBuildPath as string;
   } else {
-    const l = logger.child({ buildPhase: 'uploading to S3' });
+    const l = ctx.logger.child({ buildPhase: 'uploading to S3' });
     l.info('uploading build artifact to S3');
     const { Location: fileLocation } = await uploadFile({
       key: ctx.s3FileKey as string,

src/config.ts

@@ -43,7 +43,10 @@ export default {
   },
   logger: {
     level: env('LOGGER_LEVEL', 'info'),
-    intervalMs: envNum('LOGGER_INTERVAL_MS', 5000),
+    interval: envNum('LOGGER_INTERVAL', 5),
+    client: {
+      level: env('CLIENT_LOGGER_LEVEL', 'info'),
+    },
   },
   google: {
     credentials: env('GOOGLE_APPLICATION_CREDENTIALS', ''),

src/jobManager.ts

@@ -6,7 +6,6 @@ import BuildError from 'turtle/builders/BuildError';
 import { logErrorOnce } from 'turtle/builders/utils/common';
 import config from 'turtle/config';
 import { BUILD } from 'turtle/constants/index';
-import logger from 'turtle/logger';
 import * as buildDurationMetric from 'turtle/metrics/buildDuration';
 import * as buildStatusMetric from 'turtle/metrics/buildStatus';
 import { checkShouldExit, setCurrentJobId, turtleVersion } from 'turtle/turtleContext';
@@ -14,35 +13,35 @@ import { getPriorities } from 'turtle/utils/priorities';
 import * as redis from 'turtle/utils/redis';
 import { sanitizeJob } from 'turtle/validator';
 
-function _maybeExit() {
+function _maybeExit(logger: any) {
   if (checkShouldExit()) {
     logger.warn('Exiting due to previously received termination signal.');
     process.exit();
   }
 }
 
-export async function doJob() {
-  const jobData = await getJob();
-  await processJob(jobData);
-  _maybeExit();
+export async function doJob(logger: any) {
+  const jobData = await getJob(logger);
+  await processJob(jobData, logger);
+  _maybeExit(logger);
 }
 
-export async function getJob() {
-  _maybeExit();
+export async function getJob(logger: any) {
+  _maybeExit(logger);
   logger.info('Fetching job');
   while (true) {
     try {
       let job = null;
-      const priorities = await getPriorities();
+      const priorities = await getPriorities(logger);
       for (const priority of priorities) {
-        job = await sqs.receiveMessage(priority);
-        _maybeExit();
+        job = await sqs.receiveMessage(priority, logger);
+        _maybeExit(logger);
         if (job != null) {
           job.priority = priority;
           break;
         }
       }
-      _maybeExit();
+      _maybeExit(logger);
       if (job !== null) {
         return job;
       }
@@ -52,30 +51,30 @@ export async function getJob() {
   }
 }
 
-async function processJob(jobData: any) {
+async function processJob(jobData: any, logger: any) {
   const receiptHandle = jobData.ReceiptHandle;
   const { priority } = jobData;
   let timeoutId;
   try {
     const rawJob = JSON.parse(jobData.Body);
-    timeoutId = failAfterMaxJobTime(priority, receiptHandle, rawJob);
+    timeoutId = failAfterMaxJobTime(priority, receiptHandle, rawJob, logger);
     const job = await sanitizeJob(rawJob);
 
     setCurrentJobId(job.id);
-    const pingerHandle = sqs.changeMessageVisibilityRecurring(priority, jobData.ReceiptHandle, job.id);
+    const pingerHandle = sqs.changeMessageVisibilityRecurring(priority, jobData.ReceiptHandle, job.id, logger);
 
     logger.info(`Doing job MessageId=${jobData.MessageId} BuildId=${job.id} ${Date.now()}`);
 
-    const cancelled = await redis.checkIfCancelled(job.id);
+    const cancelled = await redis.checkIfCancelled(job.id, logger);
     if (cancelled) {
       logger.info('The job has been cancelled');
     } else {
-      redis.registerListener(job.id, () => deleteMessage(priority, receiptHandle));
+      redis.registerListener(job.id, () => deleteMessage(priority, receiptHandle, logger), logger);
       const startTimestamp = Date.now();
       let buildFailed = false;
       const buildType = _.get(job, 'config.buildType', 'default');
       try {
-        const status = await build(job);
+        const status = await build(job, logger);
         buildStatusMetric.add(buildType, status);
         buildFailed = !status;
       } catch (err) {
@@ -93,13 +92,13 @@ async function processJob(jobData: any) {
           );
           buildDurationMetric.addTotalDuration(buildType, totalBuildDurationSecs, !buildFailed);
         }
-        redis.unregisterListeners();
+        redis.unregisterListeners(logger);
       }
       logger.info(`Job done MessageId=${jobData.MessageId} BuildId=${job.id} ${Date.now()}`);
     }
     clearInterval(pingerHandle);
   } finally {
-    await deleteMessage(priority, receiptHandle);
+    await deleteMessage(priority, receiptHandle, logger);
     if (timeoutId) {
       logger.info('Clearing job failer timeout...');
       clearTimeout(timeoutId);
@@ -107,11 +106,11 @@ async function processJob(jobData: any) {
   }
 }
 
-function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any) {
+function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any, logger: any) {
   return setTimeout(async () => {
     try {
-      sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, { turtleVersion });
-      await deleteMessage(priority, receiptHandle);
+      sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, { turtleVersion }, logger);
+      await deleteMessage(priority, receiptHandle, logger);
     } finally {
       logger.error('Going to terminate turtle agent, just in case...');
       process.exit(1);
@@ -119,26 +118,27 @@ function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any)
   }, config.builder.maxJobTimeMs);
 }
 
-async function build(job: any) {
+async function build(job: any, logger: any) {
   const startTimestamp = Date.now();
-  const s3Url = await logger.initForJob(job);
+  const s3Url = await logger.init(job);
+  const buildLogger = logger.child({ jobID: job.id, experienceName: job.experienceName });
   const calculateBuildDuration = () => Math.ceil((Date.now() - startTimestamp) / 1000);
 
   try {
     await sqs.sendMessage(job.id, BUILD.JOB_STATES.IN_PROGRESS, {
       logUrl: s3Url,
       logFormat: 'json',
-    });
-    const result = await (builders as any)[job.platform](job);
+    }, buildLogger);
+    const result = await builders[job.platform](job, buildLogger);
     const buildDuration = calculateBuildDuration();
     sqs.sendMessage(job.id, BUILD.JOB_STATES.FINISHED, {
       ...result,
       turtleVersion,
       buildDuration,
-    });
+    }, buildLogger);
     return true;
   } catch (err) {
-    logErrorOnce(err);
+    logErrorOnce(err, buildLogger);
     const buildDuration = calculateBuildDuration();
     let reason;
     if (err instanceof BuildError) {
@@ -148,14 +148,14 @@ async function build(job: any) {
       turtleVersion,
       buildDuration,
       ...reason && { reason },
-    });
+    }, buildLogger);
     return false;
   } finally {
-    await logger.cleanup();
+    await logger.cleanup(job);
   }
 }
 
-async function deleteMessage(priority: string, receiptHandle: string) {
+async function deleteMessage(priority: string, receiptHandle: string, logger: any) {
   try {
     setCurrentJobId(null);
     await sqs.deleteMessage(priority, receiptHandle);

src/logger/index.ts

@@ -1,75 +1,82 @@
 import bunyan from 'bunyan';
+import bunyanDebugStream from 'bunyan-debug-stream';
+import { SentryStream } from 'bunyan-sentry-stream';
+import { Client as RavenClient } from 'raven';
 
 import config from 'turtle/config';
 import { IJob } from 'turtle/job';
-import streams from 'turtle/logger/streams';
+import S3Stream from 'turtle/logger/s3Stream';
+import { isOffline } from 'turtle/turtleContext';
 
-interface ILoggerFields {
-  [key: string]: string | number;
+// type error when using import
+// https://github.com/googleapis/nodejs-logging-bunyan/issues/241
+// tslint:disable-next-line:no-var-requires
+const { LoggingBunyan } = require('@google-cloud/logging-bunyan');
+
+interface IStream {
+  stream: any;
+  type?: string;
+  level?: string;
+  reemitErrorEvents?: boolean;
 }
 
-class Logger {
-  private parentLogger: any;
-  private currentLogger: any;
+export const s3logger = new S3Stream();
 
-  constructor(parentLogger?: any) {
-    this.parentLogger = parentLogger || bunyan.createLogger({
-      name: 'turtle',
-      level: config.logger.level,
-      platform: config.platform,
-      ...config.deploymentEnv && { environment: config.deploymentEnv },
-      streams: Object.values(streams),
-    });
-    this.currentLogger = this.parentLogger;
-  }
+const streams: IStream[] = [];
 
-  public async initForJob(job: IJob) {
-    this.currentLogger = this.parentLogger.child({
-      jobID: job.id,
-      experienceName: job.experienceName,
-    });
-    const s3Url = await streams.s3.stream.init(job);
-    return s3Url;
-  }
-
-  public async cleanup() {
-    this.currentLogger = this.parentLogger;
-    await streams.s3.stream.cleanup();
-  }
-
-  public trace(...all: any[]) {
-    this.currentLogger.trace(...all);
-  }
-
-  public debug(...all: any[]) {
-    this.currentLogger.debug(...all);
-  }
-
-  public info(...all: any[]) {
-    this.currentLogger.info(...all);
-  }
-
-  public warn(...all: any[]) {
-    this.currentLogger.warn(...all);
-  }
-
-  public error(...all: any[]) {
-    this.currentLogger.error(...all);
-  }
-
-  public fatal(...all: any[]) {
-    this.currentLogger.fatal(...all);
-  }
-
-  public child(fields: ILoggerFields) {
-    const newLogger = this.currentLogger.child(fields);
-    return new Logger(newLogger);
-  }
-
-  // only for backward compatibility
-  public withFields(fields: ILoggerFields) {
-    return this.child(fields);
-  }
+if (isOffline()) {
+  const prettyStdOut = new bunyanDebugStream({ forceColor: true });
+  streams.push({ stream: prettyStdOut, type: 'raw', level: config.logger.client.level });
+} else {
+  streams.push(
+    { stream: process.stdout },
+    {
+      type: 'raw',
+      stream: s3logger,
+      reemitErrorEvents: true,
+      level: config.logger.client.level,
+    },
+  );
 }
 
-export default new Logger();
+if (config.sentry.dsn && config.deploymentEnv !== 'development') {
+  const raven = new RavenClient();
+  raven.config(config.sentry.dsn, { environment: config.deploymentEnv });
+  streams.push({
+    level: 'error',
+    type: 'raw', // Mandatory type for SentryStream
+    stream: new SentryStream(raven),
+  });
+}
+
+if (config.google.credentials) {
+  const resource = {
+    type: 'generic_node',
+    labels: {
+      node_id: config.hostname,
+      location: '', // default value
+      namespace: '', // default value
+    },
+  };
+  streams.push(new LoggingBunyan({ name: 'turtle', resource }).stream('info'));
+}
+
+const logger = bunyan.createLogger({
+  name: 'turtle',
+  level: config.logger.level,
+  platform: config.platform,
+  environment: config.deploymentEnv,
+  streams,
+});
+
+logger.init = async (job: IJob) => {
+  return await s3logger.init(job);
+};
+
+logger.cleanup = async () => {
+  // a little hacky, but works
+  logger.info({ lastBuildLog: true }, 'this is the last log from this build');
+  await s3logger.waitForLogger();
+};
+
+export default logger;

src/logger/s3Stream.ts

@@ -0,0 +1,111 @@
+
+import EventEmitter from 'events';
+import path from 'path';
+
+import fs from 'fs-extra';
+import _ from 'lodash';
+
+import { uploadFile } from 'turtle/aws/s3';
+import config from 'turtle/config';
+import * as constants from 'turtle/constants/logger';
+import { IJob } from 'turtle/job';
+
+export default class S3Stream extends EventEmitter {
+  private s3Url: string | null;
+  private lastFlush: number | null;
+  private filePath: string | null;
+  private fileHandle: number | null;
+  private finishedPromise: Promise<any> | null;
+  private finishedPromiseResolveFn: any;
+  private secrets: any;
+
+  constructor() {
+    super();
+    this.s3Url = null;
+    this.lastFlush = null;
+    this.filePath = null;
+    this.fileHandle = null;
+    this.finishedPromise = null;
+    this.finishedPromiseResolveFn = null;
+  }
+
+  public async init(job: IJob) {
+    this.s3Url = `logs/${job.experienceName}/${job.id}`;
+    const dir = config.directories.tempS3LogsDir;
+    const exists = await fs.pathExists(dir);
+    if (!exists) {
+      await fs.mkdir(dir);
+    }
+    this.filePath = path.join(dir, job.id);
+    this.fileHandle = await fs.open(this.filePath, 'w+', 0o660);
+    this.secrets = {};
+    this.finishedPromise = new Promise((resolveFn) => (this.finishedPromiseResolveFn = resolveFn));
+    const res = await this.flush() as any;
+    return res.Location;
+  }
+
+  public async cleanup() {
+    const res = this.finishedPromiseResolveFn;
+    try {
+      await this.flush(true);
+      await fs.close(this.fileHandle as number);
+      this.fileHandle = null;
+      this.lastFlush = null;
+      this.secrets = {};
+    } finally {
+      this.finishedPromise = null;
+      this.finishedPromiseResolveFn = null;
+      res();
+    }
+  }
+
+  public flush(force = false) {
+    if (force || !this.lastFlush) {
+      return this._flush();
+    }
+    const now = Date.now();
+    if (now - this.lastFlush > config.logger.interval) {
+      return this._flush();
+    }
+  }
+
+  public write(recRaw: any) {
+    if (this.fileHandle) {
+      const rec = JSON.stringify({
+        ...recRaw,
+        level: constants.LEVELS_MAPPING[recRaw.level].toLowerCase(),
+      });
+      if (recRaw.lastBuildLog) {
+        return this.cleanup();
+      }
+      const recSanitized = Object.keys(this.secrets).reduce(
+        (acc, secret) => acc.replace(new RegExp(secret, 'g'), this.secrets[secret]),
+        rec,
+      );
+      fs
+        .write(this.fileHandle as number, `${recSanitized}\n`)
+        .then(() => {
+          this.flush();
+        });
+    }
+  }
+
+  public addSecretsToFilter(secrets: any) {
+    const filteredSecrets = secrets.filter((i: any) => i !== undefined && i !== null);
+    const replacements = filteredSecrets.map((i: any) => 'X'.repeat(i.length));
+    const toAdd = _.zipObject(filteredSecrets, replacements);
+    this.secrets = Object.assign({}, this.secrets, toAdd);
+  }
+
+  public waitForLogger() {
+    return this.finishedPromise;
+  }
+
+  private _flush() {
+    this.lastFlush = Date.now();
+    return uploadFile({
+      srcPath: this.filePath as string,
+      key: this.s3Url as string,
+    });
+  }
+}

src/logger/streams/gcloud.ts

@@ -1,29 +0,0 @@
-import config from 'turtle/config';
-
-// type error when using import
-// https://github.com/googleapis/nodejs-logging-bunyan/issues/241
-// tslint:disable-next-line:no-var-requires
-const { LoggingBunyan } = require('@google-cloud/logging-bunyan');
-
-export default function create() {
-  if (!config.google.credentials) {
-    return null;
-  }
-
-  const gcloudStream = new LoggingBunyan({
-    name: 'turtle',
-    resource: {
-      type: 'generic_node',
-      labels: {
-        node_id: config.hostname,
-        location: '', // default value
-        namespace: '', // default value
-      },
-    },
-  });
-  return {
-    level: config.logger.level,
-    type: 'raw',
-    stream: gcloudStream,
-  };
-}

src/logger/streams/index.ts

@@ -1,38 +0,0 @@
-import pickBy from 'lodash/pickBy';
-
-import createGCloudStream from 'turtle/logger/streams/gcloud';
-import createOfflineStream from 'turtle/logger/streams/offline';
-import createS3Stream from 'turtle/logger/streams/s3';
-import createSentryStream from 'turtle/logger/streams/sentry';
-import { isOffline } from 'turtle/turtleContext';
-
-interface IStream {
-  stream: any;
-  type?: string;
-  level?: string;
-  reemitErrorEvents?: boolean;
-}
-
-interface IStreamMap {
-  [key: string]: IStream;
-}
-
-function prepareStreams(): IStreamMap {
-  if (isOffline()) {
-    return {
-      stdout: createOfflineStream(),
-    };
-  } else {
-    const allStreams = {
-      stdout: { stream: process.stdout },
-      gcloud: createGCloudStream(),
-      s3: createS3Stream(),
-      sentry: createSentryStream(),
-    };
-    return pickBy(allStreams) as IStreamMap;
-  }
-}
-
-const streams = prepareStreams();
-
-export default streams;

src/logger/streams/offline.ts

@@ -1,11 +0,0 @@
-import bunyanDebugStream from 'bunyan-debug-stream';
-import config from 'turtle/config';
-
-export default function create() {
-  const prettyStdOut = new bunyanDebugStream({ forceColor: true });
-  return {
-    stream: prettyStdOut,
-    type: 'raw',
-    level: config.logger.level,
-  };
-}

src/logger/streams/s3.ts

@@ -1,106 +0,0 @@
-
-import EventEmitter from 'events';
-import path from 'path';
-
-import fs from 'fs-extra';
-import _ from 'lodash';
-
-import { uploadFile } from 'turtle/aws/s3';
-import config from 'turtle/config';
-import { IJob } from 'turtle/job';
-
-export default function create() {
-  const s3 = new S3Stream();
-  return {
-    type: 'raw',
-    stream: s3,
-    reemitErrorEvents: true,
-    level: config.logger.level,
-  };
-}
-
-class S3Stream extends EventEmitter {
-  private s3Url: string | null = null;
-  private lastFlush: number | null = null;
-  private filePath: string | null = null;
-  private fileHandle: number | null = null;
-  private uploadingPromise: Promise<void> | null = null;
-  private uploadingPromiseResolveFn: any;
-  private waitingOnPromise: boolean = false;
-
-  public async init(job: IJob) {
-    this.s3Url = `logs/${job.experienceName}/${job.id}`;
-    const dir = config.directories.tempS3LogsDir;
-    const exists = await fs.pathExists(dir);
-    if (!exists) {
-      await fs.mkdir(dir);
-    }
-    this.filePath = path.join(dir, job.id);
-    this.fileHandle = await fs.open(this.filePath, 'w+', 0o660);
-    this.lastFlush = null;
-
-    const res = await this.flush() as any;
-    return res.Location;
-  }
-
-  public async cleanup() {
-    await this.flush(true);
-    await fs.close(this.fileHandle as number);
-    this.fileHandle = null;
-    this.waitingOnPromise = false;
-  }
-
-  public flush(force = false) {
-    if (force || !this.lastFlush) {
-      return this.flushInternal();
-    }
-    const now = Date.now();
-    if (now - this.lastFlush > config.logger.intervalMs) {
-      return this.flushInternal();
-    }
-  }
-
-  public write(rec: any) {
-    if (!this.fileHandle) {
-      return;
-    }
-    fs
-      .write(this.fileHandle as number, `${JSON.stringify(rec)}\n`)
-      .then(() => {
-        this.flush();
-      });
-  }
-
-  private flushInternal(): any {
-    this.lastFlush = Date.now();
-    if (!this.uploadingPromise) {
-      this.uploadingPromise = new Promise((res) => {
-        this.uploadingPromiseResolveFn = res;
-      });
-      return this
-        .upload()
-        .then((result) => {
-          setTimeout(() => this.uploadingPromiseResolveFn(), 0);
-          return result;
-        });
-    } else {
-      if (this.waitingOnPromise) {
-        return;
-      }
-      this.waitingOnPromise = true;
-      return this.uploadingPromise
-        .then(() => {
-          this.waitingOnPromise = false;
-          this.uploadingPromise = null;
-          return this.flushInternal();
-        });
-    }
-  }
-
-  private upload() {
-    return uploadFile({
-      srcPath: this.filePath as string,
-      key: this.s3Url as string,
-    });
-  }
-}

src/logger/streams/sentry.ts

@@ -1,18 +0,0 @@
-import { SentryStream } from 'bunyan-sentry-stream';
-import { Client as RavenClient } from 'raven';
-
-import config from 'turtle/config';
-
-export default function create() {
-  if (!config.sentry.dsn || config.deploymentEnv === 'development') {
-    return null;
-  }
-
-  const raven = new RavenClient();
-  raven.config(config.sentry.dsn, { environment: config.deploymentEnv });
-  return {
-    level: 'error',
-    type: 'raw', // Mandatory type for SentryStream
-    stream: new SentryStream(raven),
-  };
-}

src/metrics/buildDuration.ts

@@ -1,11 +1,10 @@
 import * as cloudwatch from 'turtle/aws/cloudwatch';
 import { TYPE_DIMENSIONS } from 'turtle/aws/cloudwatch/dimensions';
-import logger from 'turtle/logger';
 
 const TURTLE_BUILD_DURATION_METRIC_NAME = 'turtleBuildDuration';
 const TOTAL_BUILD_DURATION_METRIC_NAME = 'totalBuildDuration';
 
-export function register() {
+export function register(logger: any) {
   for (const metricName of [TURTLE_BUILD_DURATION_METRIC_NAME, TOTAL_BUILD_DURATION_METRIC_NAME]) {
     logger.info(`Registered AWS Cloudwatch metric: ${metricName}`);
     cloudwatch.registerMetric({

src/metrics/buildStatus.ts

@@ -2,11 +2,10 @@ import * as cloudwatch from 'turtle/aws/cloudwatch';
 import { TYPE_DIMENSIONS } from 'turtle/aws/cloudwatch/dimensions';
 import * as reducers from 'turtle/aws/cloudwatch/reducers';
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 
 const BUILD_METRIC_NAME = 'build';
 
-export function register() {
+export function register(logger: any) {
   logger.info(`Registered AWS Cloudwatch metric: ${BUILD_METRIC_NAME}`);
   cloudwatch.registerMetric({
     metricName: BUILD_METRIC_NAME,

src/metrics/index.ts

@@ -1,12 +1,11 @@
 import * as cloudwatch from 'turtle/aws/cloudwatch';
-import logger from 'turtle/logger';
 import * as buildDuration from 'turtle/metrics/buildDuration';
 import * as buildStatus from 'turtle/metrics/buildStatus';
 
 const metricsToRegister = [buildStatus, buildDuration];
 
-export async function init() {
+export async function init(logger: any) {
   logger.info('Initializing Cloudwatch metrics');
-  await cloudwatch.init();
-  metricsToRegister.forEach((metric) => metric.register());
+  await cloudwatch.init(logger);
+  metricsToRegister.forEach((metric) => metric.register(logger));
 }

src/server.ts

@@ -31,11 +31,11 @@ async function main() {
   );
   LoggerDetach.configure(logger);
   if (setup[config.platform]) {
-    await setup[config.platform]();
+    await setup[config.platform](logger);
   }
 
   try {
-    const redis = await getRedisClient(RedisClient.Configuration);
+    const redis = await getRedisClient(logger, RedisClient.Configuration);
     await redis.set(REDIS_TURTLE_VERSION_KEY, turtleVersion);
     logger.info(`Register Turtle version ${turtleVersion} in Redis`);
   } catch (err) {
@@ -44,7 +44,7 @@ async function main() {
 
   while (true) {
     try {
-      await doJob();
+      await doJob(logger);
     } catch (err) {
       logger.error({ err }, 'Failed to do a job');
     }

src/setup/android.ts

@@ -1,8 +1,7 @@
-import logger from 'turtle/logger';
 import commonSetup from 'turtle/setup/common';
 
-export default async function setup() {
+export default async function setup(logger: any) {
   logger.info('Setting up environment...');
-  await commonSetup();
+  await commonSetup(logger);
   logger.info('Finished setting up environment');
 }

src/setup/common.ts

@@ -3,8 +3,8 @@ import fs from 'fs-extra';
 import config from 'turtle/config';
 import * as metrics from 'turtle/metrics';
 
-export default async function commonSetup() {
-  await metrics.init();
+export default async function commonSetup(logger: any) {
+  await metrics.init(logger);
   await fs.remove(config.directories.temporaryFilesRoot);
   await fs.remove(config.directories.tempS3LogsDir);
 }

src/setup/ios.ts

@@ -3,7 +3,6 @@ import os from 'os';
 import path from 'path';
 
 import * as keychain from 'turtle/builders/utils/ios/keychain';
-import logger from 'turtle/logger';
 import commonSetup from 'turtle/setup/common';
 
 async function deleteProvisioningProfilesFromHomedir() {
@@ -17,10 +16,10 @@ async function deleteProvisioningProfilesFromHomedir() {
   }
 }
 
-export default async function setup() {
+export default async function setup(logger: any) {
   logger.info('Setting up environment...');
   await deleteProvisioningProfilesFromHomedir();
   await keychain.cleanUp();
-  await commonSetup();
+  await commonSetup(logger);
   logger.info('Finished setting up environment');
 }

src/utils/priorities.ts

@@ -2,7 +2,6 @@ import Joi from 'joi';
 import os from 'os';
 
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 import { getRedisClient, RedisClient } from 'turtle/utils/redis';
 
 export const NORMAL = 'normalPriority';
@@ -10,7 +9,6 @@ export const HIGH = 'highPriority';
 
 export const NORMAL_CONFIGURATION = [NORMAL, HIGH];
 export const HIGH_CONFIGURATION = [HIGH, NORMAL];
-export const HIGH_ONLY_CONFIGURATION = [HIGH];
 
 const REDIS_GET_CONFIG = `
   local i = 0
@@ -41,7 +39,7 @@ export function createDefaultConfigurationKey(platform = config.platform) {
   return `${configPrefix(platform)}:default`;
 }
 
-export async function getPriorities() {
+export async function getPriorities(logger: any) {
   if (config.env === 'test') {
     return NORMAL_CONFIGURATION;
   }

src/utils/redis.ts

@@ -1,7 +1,6 @@
 import Redis from 'ioredis';
 
 import config from 'turtle/config';
-import logger from 'turtle/logger';
 
 const MILLIS_TO_UPLOAD_LOGS = 3000;
 export const MILLIS_CONNECTION_TIMEOUT = 10000;
@@ -20,7 +19,7 @@ const clientToUrl: IClientConfig = {
   [RedisClient.Configuration]: config.redis.configUrl,
 };
 
-export function connect(timeoutMs: number, type: string): Promise<any> {
+export function connect(timeoutMs: number, type: string) {
   return new Promise((resolve, reject) => {
     const redisClient = new Redis(clientToUrl[type], {
       maxRetriesPerRequest: 2,
@@ -47,7 +46,7 @@ const redisClients: IRedisClients = {
   [RedisClient.Configuration]: null,
 };
 
-export async function getRedisClient(type = RedisClient.Default) {
+export async function getRedisClient(logger: any, type = RedisClient.Default) {
   if (!redisClients[type]) {
     try {
       redisClients[type] = await connect(MILLIS_CONNECTION_TIMEOUT, type);
@@ -58,9 +57,9 @@ export async function getRedisClient(type = RedisClient.Default) {
   return redisClients[type];
 }
 
-export async function checkIfCancelled(jobId: string) {
+export async function checkIfCancelled(jobId: string, logger: any) {
   try {
-    const redis = await getRedisClient();
+    const redis = await getRedisClient(logger);
     return await redis.get(`jobs:cancelled:${jobId}`);
   } catch (err) {
     if (config.deploymentEnv === 'development') {
@@ -71,7 +70,7 @@ export async function checkIfCancelled(jobId: string) {
   }
 }
 
-export async function registerListener(jobId: string, deleteMessage: any) {
+export async function registerListener(jobId: string, deleteMessage: any, logger: any) {
   try {
     const redis = await getRedisClient(RedisClient.Subscriber);
     redis.subscribe('jobs:cancelled');
@@ -88,9 +87,9 @@ export async function registerListener(jobId: string, deleteMessage: any) {
   }
 }
 
-export async function unregisterListeners() {
+export async function unregisterListeners(logger: any) {
   try {
-    const redis = await getRedisClient();
+    const redis = await getRedisClient(logger);
     redis.removeAllListeners('message');
   } catch (err) {
     logger.error(err);

tools/bin/node

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" nodejs-8_x --command node "$@"

tools/bin/npm

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" nodejs-8_x --command npm "$@"

tools/bin/npx

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" nodejs-8_x --command npx "$@"

tools/bin/yarn

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" yarn --command yarn "$@"

tools/bin/yarnpkg

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+# This file was generated by nix-binstubs
+exec nix run -f "$(dirname "$0")/../../nix" yarn --command yarnpkg "$@"

tools/ci/bin/node

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../../nix" nodejs-8_x --command node "$@"

tools/ci/bin/npm

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../../nix" nodejs-8_x --command npm "$@"

tools/ci/bin/npx

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../../nix" nodejs-8_x --command npx "$@"

tools/ci/bin/yarn

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../../nix" yarn --command yarn "$@"

tools/ci/bin/yarnpkg

@@ -1,3 +0,0 @@
-#! /usr/bin/env bash
-# This file was generated by nix-binstubs
-exec nix run -f "$(dirname "$0")/../../../nix" yarn --command yarnpkg "$@"

tsconfig.json

@@ -6,20 +6,17 @@
     "sourceMap": true,
     "inlineSources": true,
     "outDir": "./build",
+    "rootDir": "./src",
     "strict": true,
     "noUnusedLocals": true,
     "noUnusedParameters": true,
     "noFallthroughCasesInSwitch": true,
     "esModuleInterop": true,
-    "baseUrl": ".",
+    "baseUrl": "./",
     "paths": {
       "turtle/*": ["src/*"],
     },
-    "allowJs": true,
-    "plugins": [
-      { "transform": "ts-transformer-imports" }
-    ]
+    "allowJs": true
   },
-  "include": ["src"],
   "exclude": ["node_modules", "build", "workingdir"]
 }