Release 0.7.2

* remove client logger level env

* Update CHANGELOG.md

.circleci/config.yml

@@ -33,10 +33,12 @@ commands:
           command: |
             nix-env -iA \
               nixpkgs.binutils \
+              nixpkgs.curl \
               nixpkgs.direnv \
               nixpkgs.findutils \
               nixpkgs.gawk \
               nixpkgs.git \
+              nixpkgs.gnused \
               nixpkgs.gnutar \
               nixpkgs.gzip \
               nixpkgs.openssh \

CHANGELOG.md

@@ -6,6 +6,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.7.2] - 2019-05-09
+### Removed
+- Support for `CLIENT_LOGGER_LEVEL` env variable.
+### Added
+- Support for Android tarballs without `universe-package.json`.
+### Changed
+- Upgraded `@expo/traveling-fastlane-darwin` to 1.9.4 (fixes issues with generating provisioning profiles for ad-hoc builds).
+
+## [0.7.1] - 2019-05-06
+### Fixed
+- Resolving `fastlane` version when `LC_ALL` is not set.
+
+## [0.7.0] - 2019-05-06
+### Added
+- Support for custom Android builds.
+### Changed
+- Refactored logger.
+### Fixed
+- `platform` log field wrongly set to `ios` for Android builds (when using `turtle-cli`).
+- Brought back `jobID` and `experienceName` fields to Android logs.
+
 ## [0.6.1] - 2019-04-23
 ### Added
 - Checking fastlane version before running build (turtle-cli).

Dockerfile

@@ -103,11 +103,15 @@ ADD . /app
 
 RUN for SDK_VERSION in `ls /app/workingdir/android/`; do \
       cd /app/workingdir/android/$SDK_VERSION && \
+      if [ -f universe-package.json ]; then \
       mv package.json exponent-package.json && \
       mv universe-package.json package.json && \
       yarn install && \
       mv package.json universe-package.json && \
-      mv exponent-package.json package.json \
+      mv exponent-package.json package.json; \
+      else \
+      yarn install; \
+      fi \
     ; done
 
 WORKDIR /app

bin/turtle.js

@@ -1,6 +1,15 @@
 #!/usr/bin/env node
 'use strict';
 
+const subcommand = process.argv[2];
+if (subcommand) {
+  const [, platform] = subcommand.split(':');
+  if (platform === 'ios' || platform === 'android') {
+    // so that logs have correct platform set
+    process.env.PLATFORM = platform;
+  }
+}
+
 require('source-map-support').install();
 require('../build/bin/env').initOfflineEnv();
 require('../build/bin/index').run('turtle');

package.json

@@ -1,6 +1,6 @@
 {
   "name": "turtle-cli",
-  "version": "0.6.1",
+  "version": "0.7.2",
   "description": "The command-line tool for building standalone Expo apps",
   "preferGlobal": true,
   "scripts": {
@@ -130,9 +130,9 @@
     "tar": "^4.4.6",
     "uuid": "^3.2.1",
     "which": "^1.3.1",
-    "xdl": "53.5.3"
+    "xdl": "53.5.4"
   },
   "optionalDependencies": {
-    "@expo/traveling-fastlane-darwin": "^1.8.1"
+    "@expo/traveling-fastlane-darwin": "1.9.4"
   }
 }

scripts/setWorkingdirMode.sh

@@ -36,7 +36,7 @@ setup_local() {
   mkdir -p $WORKINGDIR
   echo "$WORKINGDIR directory created"
 
-  LINKS=("template-files" "package.json" "android" "expokit-npm-package" "packages" "client-builds" "cpp" "exponent-view-template" "ios" "shellAppBase-builds" "shellAppWorkspaces")
+  LINKS=("template-files" "package.json" "android" "expokit-npm-package" "packages" "client-builds" "cpp" "exponent-view-template" "ios" "shellAppBase-builds" "shellAppWorkspaces" "node_modules")
 
   for LINKNAME in ${LINKS[@]}; do
     ensure_link_exists "$EXPO_UNIVERSE_DIR/exponent/$LINKNAME" "$WORKINGDIR/$LINKNAME"
@@ -55,6 +55,14 @@ setup_remote() {
       $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
     fi
   fi
+  
+  for sdk in $ROOT_DIR/workingdir/android/* ; do
+    if [[ -f "$sdk/universe-package.json" ]]; then
+      mv $sdk/package.json $sdk/exponent-package.json
+      mv $sdk/universe-package.json $sdk/package.json
+      cd $sdk && yarn
+    fi
+  done
 }
 
 switch_local() {

src/bin/commands/setup.ts

@@ -6,7 +6,7 @@ import setup from 'turtle/bin/setup/setup';
 import { PLATFORMS } from 'turtle/constants';
 import logger from 'turtle/logger';
 
-const l = logger.withFields({ buildPhase: 'setting up environment' });
+const l = logger.child({ buildPhase: 'setting up environment' });
 
 function createSetupCommand(platform: string, os?: string) {
   return (program: any) => {

src/bin/setup/android/index.ts

@@ -41,7 +41,7 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
   },
 ];
 const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
-const l = logger.withFields(LOGGER_FIELDS);
+const l = logger.child(LOGGER_FIELDS);
 
 export default async function setup(sdkVersion?: string) {
   await checkSystem(REQUIRED_TOOLS);

src/bin/setup/android/ndk.ts

@@ -12,7 +12,7 @@ import logger from 'turtle/logger';
 const ANDROID_NDK_URL = `https://dl.google.com/android/repository/android-ndk-r10e-${os.platform()}-x86_64.zip`;
 
 const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
-const l = logger.withFields(LOGGER_FIELDS);
+const l = logger.child(LOGGER_FIELDS);
 
 export default async function ensureAndroidNDKIsPresent() {
   const androidNdkDir = path.join(config.directories.androidDependenciesDir, 'ndk');

src/bin/setup/android/sdk.ts

@@ -13,7 +13,7 @@ import logger from 'turtle/logger';
 const ANDROID_SDK_URL = 'https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip';
 
 const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
-const l = logger.withFields(LOGGER_FIELDS);
+const l = logger.child(LOGGER_FIELDS);
 
 export default async function ensureAndroidSDKIsPresent() {
   const androidSdkDir = path.join(config.directories.androidDependenciesDir, 'sdk');

src/bin/setup/ios.ts

@@ -25,11 +25,13 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
       const { stdout } = await ExponentTools.spawnAsyncThrowError(
         'fastlane',
         ['--version'],
-        { stdio: 'pipe', env: { ...process.env, FASTLANE_SKIP_UPDATE_CHECK: '1' } },
+        { stdio: 'pipe', env: { ...process.env, FASTLANE_SKIP_UPDATE_CHECK: '1', LC_ALL: 'en_US.UTF-8' } },
       );
       const fastlaneVersion = parseFastlaneVersion(stdout);
       if (fastlaneVersion && !semver.satisfies(fastlaneVersion, `>= ${MINIMAL_VERSION}`)) {
-        throw new Error(`Please install newer version of fastlane (>= ${MINIMAL_VERSION})`);
+        throw new Error(
+          `Your fastlane on version ${fastlaneVersion}. Please upgrade it to at least ${MINIMAL_VERSION}.`,
+        );
       }
     },
   },

src/bin/setup/utils/common.ts

@@ -10,7 +10,7 @@ import { IShellAppDirectoryConfig } from 'turtle/builders/utils/workingdir';
 import config from 'turtle/config';
 import logger from 'turtle/logger';
 
-const l = logger.withFields({ buildPhase: 'setting up environment' });
+const l = logger.child({ buildPhase: 'setting up environment' });
 
 interface IShellAppFormaters {
   formatShellAppDirectory: (config: IShellAppDirectoryConfig) => string;

src/bin/setup/utils/toolsDetector.ts

@@ -4,7 +4,7 @@ import _which from 'which';
 import logger from 'turtle/logger';
 
 const which = util.promisify(_which);
-const l = logger.withFields({ buildPhase: 'setting up environment' });
+const l = logger.child({ buildPhase: 'setting up environment' });
 
 export interface IToolDefinition {
   command: string;

src/builders/android.ts

@@ -2,6 +2,7 @@ import path from 'path';
 
 import fs from 'fs-extra';
 import _ from 'lodash';
+import semver from 'semver';
 import uuidv4 from 'uuid/v4';
 import { AndroidShellApp, ImageUtils } from 'xdl';
 
@@ -9,6 +10,7 @@ import getOrCreateCredentials from 'turtle/builders/utils/android/credentials';
 import { formatShellAppDirectory } from 'turtle/builders/utils/android/workingdir';
 import * as commonUtils from 'turtle/builders/utils/common';
 import * as imageHelpers from 'turtle/builders/utils/image';
+import { resolveNativeModules } from 'turtle/builders/utils/unimodules';
 import { uploadBuildToS3 } from 'turtle/builders/utils/uploader';
 import { ensureCanBuildSdkVersion } from 'turtle/builders/utils/version';
 import config from 'turtle/config';
@@ -66,6 +68,11 @@ async function runShellAppBuilder(
   const sdkVersion = _.get(manifest, 'sdkVersion', sdkVersionFromJob);
   const workingDir = formatShellAppDirectory({ sdkVersion });
 
+  logger.info({ buildPhase: 'resolve native modules' }, 'Resolving universal modules dependencies');
+  const enabledModules = semver.satisfies(sdkVersion, '>= 33.0.0')
+    ? await resolveNativeModules(workingDir, manifest.dependencies)
+    : null;
+
   try {
     await AndroidShellApp.createAndroidShellAppAsync({
       url: commonUtils.getExperienceUrl(jobData),
@@ -79,6 +86,7 @@ async function runShellAppBuilder(
       releaseChannel: jobConfig.releaseChannel,
       workingDir,
       outputFile: outputFilePath,
+      modules: enabledModules,
     });
   } catch (err) {
     commonUtils.logErrorOnce(err);

src/builders/ios/archive.ts

@@ -29,7 +29,7 @@ export default async function buildArchive(ctx: IContext, job: IJob) {
 }
 
 async function buildAndSignIPA(ctx: IContext, job: IJob, keychainPath: string, manifest: any) {
-  const l = logger.withFields({ buildPhase: 'building and signing IPA' });
+  const l = logger.child({ buildPhase: 'building and signing IPA' });
   l.info('building and signing IPA');
 
   const {

src/builders/ios/simulator.ts

@@ -4,7 +4,7 @@ import { IJob } from 'turtle/job';
 import logger from 'turtle/logger';
 
 export default async function buildSimulator(ctx: IContext, job: IJob) {
-  const l = logger.withFields({ buildPhase: 'running simulator builder' });
+  const l = logger.child({ buildPhase: 'running simulator builder' });
   l.info('running simulator build');
   await runShellAppBuilder(ctx, job);
   l.info('build complete');

src/builders/utils/android/credentials.ts

@@ -12,7 +12,7 @@ import { isOffline } from 'turtle/turtleContext';
 
 async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredentials> {
   if (!jobData.credentials) {
-    const l = logger.withFields({ buildPhase: 'generating keystore' });
+    const l = logger.child({ buildPhase: 'generating keystore' });
     const credentials: any = {};
     l.info('Creating keystore');
     const keystoreFilename = `/tmp/keystore-${jobData.id}.tmp.jks`;

src/builders/utils/ios/keychain.ts

@@ -9,7 +9,7 @@ interface IKeychain {
 }
 
 export async function create(ctx: IContext): Promise<IKeychain> {
-  const l = logger.withFields({ buildPhase: 'creating keychain' });
+  const l = logger.child({ buildPhase: 'creating keychain' });
   try {
     l.info('creating keychain...');
     const keychainInfo = await IosKeychain.createKeychain(ctx.appUUID, false);
@@ -22,7 +22,7 @@ export async function create(ctx: IContext): Promise<IKeychain> {
 }
 
 export async function remove(ctx: IContext, keychainPath: string) {
-  const l = logger.withFields({ buildPhase: 'deleting keychain' });
+  const l = logger.child({ buildPhase: 'deleting keychain' });
   try {
     l.info('delete keychain...');
     const keychainInfo = await IosKeychain.deleteKeychain({
@@ -49,7 +49,7 @@ export async function importCert(
     certPassword,
   }: { keychainPath: string; certP12: string; certPassword: string },
 ) {
-  const l = logger.withFields({ buildPhase: 'importing certificate into keychain' });
+  const l = logger.child({ buildPhase: 'importing certificate into keychain' });
   try {
     l.info('importing distribution certificate into keychain...');
     const { tempCertPath: certPath } = ctx;

src/builders/utils/unimodules.ts

@@ -0,0 +1,167 @@
+import fs from 'fs-extra';
+import filter from 'lodash/filter';
+import keys from 'lodash/keys';
+import merge from 'lodash/merge';
+import values from 'lodash/values';
+import path from 'path';
+import logger from 'turtle/logger';
+
+export interface IUnimoduleEntry {
+  name: string;
+  version: string;
+  dirname: string;
+}
+
+interface IPackageInfo {
+  name: string;
+  version: string;
+  dirname: string; // could be different than name
+  dependencies: string[];
+}
+
+export async function resolveNativeModules(workingdir: string, packages?: string[]): Promise<IUnimoduleEntry[]> {
+  const resolver = new Resolver(workingdir);
+  await resolver.init();
+
+  if (!packages) {
+    resolver.addAll();
+  } else {
+    const corePackages = await readCorePackages(workingdir);
+    for (const pkg of corePackages) {
+      if (resolver.isUnimodule(pkg)) {
+        resolver.addModule(pkg);
+      }
+    }
+    for (const pkg of packages) {
+      if (resolver.isUnimodule(pkg)) {
+        resolver.addModule(pkg);
+      }
+    }
+  }
+
+  return resolver.getModules();
+}
+
+async function readCorePackages(workingdir: string) {
+  const pkgJsonPath = path.join(workingdir, 'node_modules', 'react-native-unimodules', 'package.json');
+  if (await fs.pathExists(pkgJsonPath)) {
+    const packageJson = JSON.parse(await fs.readFile(pkgJsonPath, 'utf-8'));
+    return keys(packageJson.dependencies);
+  } else {
+    logger.warn('No core packages detected');
+    return [];
+  }
+}
+
+interface IDependencyInfo {
+  [key: string]: IPackageInfo;
+}
+
+class Resolver {
+  private workingdir: string;
+  private modulesMap: {[key: string]: IUnimoduleEntry};
+  private dependencyMap: IDependencyInfo;
+
+  constructor(workingdir: string) {
+    this.workingdir = workingdir;
+    this.modulesMap = {};
+    this.dependencyMap = {};
+  }
+
+  public async init() {
+    this.dependencyMap = await generateDependenciesInfo(this.workingdir);
+  }
+
+  public isUnimodule(moduleName: string): boolean {
+    return !!this.dependencyMap[moduleName];
+  }
+
+  public addAll() {
+    for (const moduleName of Object.keys(this.dependencyMap)) {
+      this.addModule(moduleName);
+    }
+  }
+
+  public addModule(moduleName: string) {
+    if (this.dependencyMap[moduleName]) {
+      this.resolveModule(moduleName);
+    } else {
+      throw new Error('Attempting to add unknown module');
+    }
+  }
+
+  public getModules(): IUnimoduleEntry[] {
+     return values(this.modulesMap);
+  }
+
+  private async resolveModule(moduleName: string) {
+    if (this.modulesMap[moduleName]) {
+      return;
+    }
+    const pkg = this.dependencyMap[moduleName];
+    logger.info(`Adding ${moduleName}:${pkg.version}`);
+    this.modulesMap[moduleName] = {
+      name: pkg.name,
+      dirname: pkg.dirname,
+      version: pkg.version,
+    };
+    for (const dep of pkg.dependencies) {
+      this.resolveModule(dep);
+    }
+  }
+}
+
+async function generateDependenciesInfo(workingdir: string): Promise<IDependencyInfo> {
+  const dir = await fs.readdir(path.join(workingdir, 'packages'));
+  const map: IDependencyInfo = {};
+
+  // scrap directory for package metadata
+  for (const pkgName of dir) {
+    const pkg = await getPackage(workingdir, pkgName);
+    if (pkg) {
+      map[pkg.name] = pkg;
+    }
+  }
+
+  // handle @unimodules directory
+  const unimodulesDir = await fs.readdir(path.join(workingdir, 'packages', '@unimodules'));
+  for (const pkgName of unimodulesDir) {
+    const pkg = await getPackage(workingdir, path.join('@unimodules', pkgName));
+    if (pkg) {
+      map[pkg.name] = pkg;
+    }
+  }
+
+  // remove non unimodule dependencies
+  for (const pkg of Object.keys(map)) {
+    map[pkg].dependencies = filter(map[pkg].dependencies, (dep) => !!map[dep]);
+  }
+  return map;
+}
+
+async function getPackage(workingdir: string, pkgName: string): Promise<IPackageInfo | null> {
+  const pkgPath = path.join(workingdir, 'packages', pkgName);
+  const isAndroid = await fs.pathExists(path.join(pkgPath, 'android'));
+  const isIos = await fs.pathExists(path.join(pkgPath, 'ios'));
+  const isUnimodule = await fs.pathExists(path.join(pkgPath, 'unimodule.json'));
+  const hasPackageJson = await fs.pathExists(path.join(pkgPath, 'package.json'));
+  if (!isUnimodule || !hasPackageJson || (!isAndroid && !isIos)) {
+    return null;
+  }
+
+  const packageJson = JSON.parse(await fs.readFile(path.join(pkgPath, 'package.json'), 'utf-8'));
+
+  const pkgDeps = merge({}, // reading everything to make sure
+    packageJson.dependencies,
+    packageJson.peerDependencies,
+    packageJson.unimodulePeerDependencies,
+  );
+  const info: IPackageInfo = {
+    name: packageJson.name,
+    version: packageJson.version,
+    dirname: pkgName,
+    dependencies: Object.keys(pkgDeps),
+  };
+
+  return info;
+}

src/builders/utils/uploader.ts

@@ -12,14 +12,14 @@ interface IUploadCtx {
 
 export async function uploadBuildToS3(ctx: IUploadCtx) {
   if (config.builder.fakeUpload) {
-    const l = logger.withFields({ buildPhase: 'copying build artifact' });
+    const l = logger.child({ buildPhase: 'copying build artifact' });
     const { fakeUploadBuildPath, uploadPath } = ctx;
     l.info('copying build to fake upload directory');
     await fs.copy(uploadPath, fakeUploadBuildPath as string);
     l.info(`copied build to ${fakeUploadBuildPath}`);
     return fakeUploadBuildPath as string;
   } else {
-    const l = logger.withFields({ buildPhase: 'uploading to S3' });
+    const l = logger.child({ buildPhase: 'uploading to S3' });
     l.info('uploading build artifact to S3');
     const { Location: fileLocation } = await uploadFile({
       key: ctx.s3FileKey as string,

src/config.ts

@@ -43,10 +43,7 @@ export default {
   },
   logger: {
     level: env('LOGGER_LEVEL', 'info'),
-    interval: envNum('LOGGER_INTERVAL', 5),
-    client: {
-      level: env('CLIENT_LOGGER_LEVEL', 'info'),
-    },
+    intervalMs: envNum('LOGGER_INTERVAL_MS', 5000),
   },
   google: {
     credentials: env('GOOGLE_APPLICATION_CREDENTIALS', ''),

src/jobManager.ts

@@ -121,7 +121,7 @@ function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any)
 
 async function build(job: any) {
   const startTimestamp = Date.now();
-  const s3Url = await logger.init(job);
+  const s3Url = await logger.initForJob(job);
   const calculateBuildDuration = () => Math.ceil((Date.now() - startTimestamp) / 1000);
 
   try {
@@ -151,7 +151,7 @@ async function build(job: any) {
     });
     return false;
   } finally {
-    await logger.cleanup(job);
+    await logger.cleanup();
   }
 }
 

src/logger/gcloudStream.ts

@@ -1,58 +0,0 @@
-import EventEmitter from 'events';
-
-import { IJob } from 'turtle/job';
-
-// type error when using import
-// https://github.com/googleapis/nodejs-logging-bunyan/issues/241
-// tslint:disable-next-line:no-var-requires
-const { LoggingBunyan } = require('@google-cloud/logging-bunyan');
-
-export interface IGCloudConfig {
-  name: string;
-  resource: {
-    type: string;
-    labels: {
-      node_id: string;
-      location: string;
-      namespace: string;
-    };
-  };
-}
-
-interface IGCloudExtraFields {
-  jobID: string;
-  experienceName: string;
-}
-
-export default class GCloudStream extends EventEmitter {
-  private logsStream: any;
-  private extraFields: IGCloudExtraFields | null;
-
-   constructor(gcloudConfig: IGCloudConfig) {
-    super();
-
-    const { name, resource } = gcloudConfig;
-    this.logsStream = new LoggingBunyan({ name, resource });
-    this.extraFields = null;
-  }
-
-  public init(job: IJob) {
-    this.extraFields = {
-      jobID: job.id,
-      experienceName: job.experienceName,
-    };
-  }
-
-  public cleanup() {
-    this.extraFields = null;
-  }
-
-  public write(rec: any) {
-    const { msg: message, ...other } = rec;
-    this.logsStream.write({
-      message,
-      ...other,
-      data: this.extraFields || {},
-    });
-  }
-}

src/logger/index.ts

@@ -1,103 +1,75 @@
 import bunyan from 'bunyan';
-import bunyanDebugStream from 'bunyan-debug-stream';
-import { SentryStream } from 'bunyan-sentry-stream';
-import { Client as RavenClient } from 'raven';
 
 import config from 'turtle/config';
-import * as constants from 'turtle/constants/logger';
 import { IJob } from 'turtle/job';
-import GCloudStream from 'turtle/logger/gcloudStream';
-import S3Stream from 'turtle/logger/s3Stream';
-import { isOffline } from 'turtle/turtleContext';
+import streams from 'turtle/logger/streams';
 
-interface IStream {
-  stream: any;
-  type?: string;
-  level?: string;
-  reemitErrorEvents?: boolean;
+interface ILoggerFields {
+  [key: string]: string | number;
 }
 
-export const s3logger = new S3Stream();
-let gcloudStream: GCloudStream;
+class Logger {
+  private parentLogger: any;
+  private currentLogger: any;
 
-const streams: IStream[] = [];
-
-if (isOffline()) {
-  const prettyStdOut = new bunyanDebugStream({ forceColor: true });
-  streams.push({ stream: prettyStdOut, type: 'raw', level: config.logger.client.level });
-} else {
-  streams.push(
-    { stream: process.stdout },
-    {
-      type: 'raw',
-      stream: s3logger,
-      reemitErrorEvents: true,
-      level: config.logger.client.level,
-    },
-  );
-}
-
-if (config.sentry.dsn && config.deploymentEnv !== 'development') {
-  const raven = new RavenClient();
-  raven.config(config.sentry.dsn, { environment: config.deploymentEnv });
-  streams.push({
-    level: 'error',
-    type: 'raw', // Mandatory type for SentryStream
-    stream: new SentryStream(raven),
-  });
-}
-
-if (config.google.credentials) {
-  const gcloudConfig = {
-    name: 'turtle',
-    resource: {
-      type: 'generic_node',
-      labels: {
-        node_id: config.hostname,
-        location: '', // default value
-        namespace: '', // default value
-      },
-    },
-  };
-  gcloudStream = new GCloudStream(gcloudConfig);
-  streams.push({
-    level: 'info',
-    type: 'raw',
-    stream: gcloudStream,
-  });
-}
-
-const logger = bunyan.createLogger({
-  name: 'turtle',
-  level: config.logger.level,
-  platform: config.platform,
-  environment: config.deploymentEnv,
-  streams,
-});
-
-logger.withFields = (extraFields: any) => withFields(logger, extraFields);
-
-logger.init = async (job: IJob) => {
-  if (gcloudStream) {
-    gcloudStream.init(job);
+  constructor(parentLogger?: any) {
+    this.parentLogger = parentLogger || bunyan.createLogger({
+      name: 'turtle',
+      level: config.logger.level,
+      platform: config.platform,
+      ...config.deploymentEnv && { environment: config.deploymentEnv },
+      streams: Object.values(streams),
+    });
+    this.currentLogger = this.parentLogger;
   }
-  return await s3logger.init(job);
-};
 
-logger.cleanup = async () => {
-  // a little hacky, but works
-  logger.info({ lastBuildLog: true }, 'this is the last log from this build');
-  if (gcloudStream) {
-    gcloudStream.cleanup();
+  public async initForJob(job: IJob) {
+    this.currentLogger = this.parentLogger.child({
+      jobID: job.id,
+      experienceName: job.experienceName,
+    });
+    const s3Url = await streams.s3.stream.init(job);
+    return s3Url;
   }
-  await s3logger.waitForLogger();
-};
 
-export default logger;
+  public async cleanup() {
+    this.currentLogger = this.parentLogger;
+    await streams.s3.stream.cleanup();
+  }
 
-export function withFields(loggerObj: any, extraFields: any) {
-  return constants.LEVELS.reduce((obj, level) => {
-    obj[level] = (...args: any[]) => loggerObj[level](extraFields, ...args);
-    return obj;
-  }, {} as any);
+  public trace(...all: any[]) {
+    this.currentLogger.trace(...all);
+  }
+
+  public debug(...all: any[]) {
+    this.currentLogger.debug(...all);
+  }
+
+  public info(...all: any[]) {
+    this.currentLogger.info(...all);
+  }
+
+  public warn(...all: any[]) {
+    this.currentLogger.warn(...all);
+  }
+
+  public error(...all: any[]) {
+    this.currentLogger.error(...all);
+  }
+
+  public fatal(...all: any[]) {
+    this.currentLogger.fatal(...all);
+  }
+
+  public child(fields: ILoggerFields) {
+    const newLogger = this.currentLogger.child(fields);
+    return new Logger(newLogger);
+  }
+
+  // only for backward compatibility
+  public withFields(fields: ILoggerFields) {
+    return this.child(fields);
+  }
 }
+
+export default new Logger();

src/logger/s3Stream.ts

@@ -1,111 +0,0 @@
-
-import EventEmitter from 'events';
-import path from 'path';
-
-import fs from 'fs-extra';
-import _ from 'lodash';
-
-import { uploadFile } from 'turtle/aws/s3';
-import config from 'turtle/config';
-import * as constants from 'turtle/constants/logger';
-import { IJob } from 'turtle/job';
-
-export default class S3Stream extends EventEmitter {
-  private s3Url: string | null;
-  private lastFlush: number | null;
-  private filePath: string | null;
-  private fileHandle: number | null;
-  private finishedPromise: Promise<any> | null;
-  private finishedPromiseResolveFn: any;
-  private secrets: any;
-
-  constructor() {
-    super();
-    this.s3Url = null;
-    this.lastFlush = null;
-    this.filePath = null;
-    this.fileHandle = null;
-    this.finishedPromise = null;
-    this.finishedPromiseResolveFn = null;
-  }
-
-  public async init(job: IJob) {
-    this.s3Url = `logs/${job.experienceName}/${job.id}`;
-    const dir = config.directories.tempS3LogsDir;
-    const exists = await fs.pathExists(dir);
-    if (!exists) {
-      await fs.mkdir(dir);
-    }
-    this.filePath = path.join(dir, job.id);
-    this.fileHandle = await fs.open(this.filePath, 'w+', 0o660);
-    this.secrets = {};
-    this.finishedPromise = new Promise((resolveFn) => (this.finishedPromiseResolveFn = resolveFn));
-    const res = await this.flush() as any;
-    return res.Location;
-  }
-
-  public async cleanup() {
-    const res = this.finishedPromiseResolveFn;
-    try {
-      await this.flush(true);
-      await fs.close(this.fileHandle as number);
-      this.fileHandle = null;
-      this.lastFlush = null;
-      this.secrets = {};
-    } finally {
-      this.finishedPromise = null;
-      this.finishedPromiseResolveFn = null;
-      res();
-    }
-  }
-
-  public flush(force = false) {
-    if (force || !this.lastFlush) {
-      return this._flush();
-    }
-    const now = Date.now();
-    if (now - this.lastFlush > config.logger.interval) {
-      return this._flush();
-    }
-  }
-
-  public write(recRaw: any) {
-    if (this.fileHandle) {
-      const rec = JSON.stringify({
-        ...recRaw,
-        level: constants.LEVELS_MAPPING[recRaw.level].toLowerCase(),
-      });
-      if (recRaw.lastBuildLog) {
-        return this.cleanup();
-      }
-      const recSanitized = Object.keys(this.secrets).reduce(
-        (acc, secret) => acc.replace(new RegExp(secret, 'g'), this.secrets[secret]),
-        rec,
-      );
-      fs
-        .write(this.fileHandle as number, `${recSanitized}\n`)
-        .then(() => {
-          this.flush();
-        });
-    }
-  }
-
-  public addSecretsToFilter(secrets: any) {
-    const filteredSecrets = secrets.filter((i: any) => i !== undefined && i !== null);
-    const replacements = filteredSecrets.map((i: any) => 'X'.repeat(i.length));
-    const toAdd = _.zipObject(filteredSecrets, replacements);
-    this.secrets = Object.assign({}, this.secrets, toAdd);
-  }
-
-  public waitForLogger() {
-    return this.finishedPromise;
-  }
-
-  private _flush() {
-    this.lastFlush = Date.now();
-    return uploadFile({
-      srcPath: this.filePath as string,
-      key: this.s3Url as string,
-    });
-  }
-}

src/logger/streams/gcloud.ts

@@ -0,0 +1,29 @@
+import config from 'turtle/config';
+
+// type error when using import
+// https://github.com/googleapis/nodejs-logging-bunyan/issues/241
+// tslint:disable-next-line:no-var-requires
+const { LoggingBunyan } = require('@google-cloud/logging-bunyan');
+
+export default function create() {
+  if (!config.google.credentials) {
+    return null;
+  }
+
+  const gcloudStream = new LoggingBunyan({
+    name: 'turtle',
+    resource: {
+      type: 'generic_node',
+      labels: {
+        node_id: config.hostname,
+        location: '', // default value
+        namespace: '', // default value
+      },
+    },
+  });
+  return {
+    level: config.logger.level,
+    type: 'raw',
+    stream: gcloudStream,
+  };
+}

src/logger/streams/index.ts

@@ -0,0 +1,38 @@
+import pickBy from 'lodash/pickBy';
+
+import createGCloudStream from 'turtle/logger/streams/gcloud';
+import createOfflineStream from 'turtle/logger/streams/offline';
+import createS3Stream from 'turtle/logger/streams/s3';
+import createSentryStream from 'turtle/logger/streams/sentry';
+import { isOffline } from 'turtle/turtleContext';
+
+interface IStream {
+  stream: any;
+  type?: string;
+  level?: string;
+  reemitErrorEvents?: boolean;
+}
+
+interface IStreamMap {
+  [key: string]: IStream;
+}
+
+function prepareStreams(): IStreamMap {
+  if (isOffline()) {
+    return {
+      stdout: createOfflineStream(),
+    };
+  } else {
+    const allStreams = {
+      stdout: { stream: process.stdout },
+      gcloud: createGCloudStream(),
+      s3: createS3Stream(),
+      sentry: createSentryStream(),
+    };
+    return pickBy(allStreams) as IStreamMap;
+  }
+}
+
+const streams = prepareStreams();
+
+export default streams;

src/logger/streams/offline.ts

@@ -0,0 +1,11 @@
+import bunyanDebugStream from 'bunyan-debug-stream';
+import config from 'turtle/config';
+
+export default function create() {
+  const prettyStdOut = new bunyanDebugStream({ forceColor: true });
+  return {
+    stream: prettyStdOut,
+    type: 'raw',
+    level: config.logger.level,
+  };
+}

src/logger/streams/s3.ts

@@ -0,0 +1,106 @@
+
+import EventEmitter from 'events';
+import path from 'path';
+
+import fs from 'fs-extra';
+import _ from 'lodash';
+
+import { uploadFile } from 'turtle/aws/s3';
+import config from 'turtle/config';
+import { IJob } from 'turtle/job';
+
+export default function create() {
+  const s3 = new S3Stream();
+  return {
+    type: 'raw',
+    stream: s3,
+    reemitErrorEvents: true,
+    level: config.logger.level,
+  };
+}
+
+class S3Stream extends EventEmitter {
+  private s3Url: string | null = null;
+  private lastFlush: number | null = null;
+  private filePath: string | null = null;
+  private fileHandle: number | null = null;
+  private uploadingPromise: Promise<void> | null = null;
+  private uploadingPromiseResolveFn: any;
+  private waitingOnPromise: boolean = false;
+
+  public async init(job: IJob) {
+    this.s3Url = `logs/${job.experienceName}/${job.id}`;
+    const dir = config.directories.tempS3LogsDir;
+    const exists = await fs.pathExists(dir);
+    if (!exists) {
+      await fs.mkdir(dir);
+    }
+    this.filePath = path.join(dir, job.id);
+    this.fileHandle = await fs.open(this.filePath, 'w+', 0o660);
+    this.lastFlush = null;
+
+    const res = await this.flush() as any;
+    return res.Location;
+  }
+
+  public async cleanup() {
+    await this.flush(true);
+    await fs.close(this.fileHandle as number);
+    this.fileHandle = null;
+    this.waitingOnPromise = false;
+  }
+
+  public flush(force = false) {
+    if (force || !this.lastFlush) {
+      return this.flushInternal();
+    }
+    const now = Date.now();
+    if (now - this.lastFlush > config.logger.intervalMs) {
+      return this.flushInternal();
+    }
+  }
+
+  public write(rec: any) {
+    if (!this.fileHandle) {
+      return;
+    }
+    fs
+      .write(this.fileHandle as number, `${JSON.stringify(rec)}\n`)
+      .then(() => {
+        this.flush();
+      });
+  }
+
+  private flushInternal(): any {
+    this.lastFlush = Date.now();
+    if (!this.uploadingPromise) {
+      this.uploadingPromise = new Promise((res) => {
+        this.uploadingPromiseResolveFn = res;
+      });
+      return this
+        .upload()
+        .then((result) => {
+          setTimeout(() => this.uploadingPromiseResolveFn(), 0);
+          return result;
+        });
+    } else {
+      if (this.waitingOnPromise) {
+        return;
+      }
+      this.waitingOnPromise = true;
+      return this.uploadingPromise
+        .then(() => {
+          this.waitingOnPromise = false;
+          this.uploadingPromise = null;
+          return this.flushInternal();
+        });
+    }
+  }
+
+  private upload() {
+    return uploadFile({
+      srcPath: this.filePath as string,
+      key: this.s3Url as string,
+    });
+  }
+}

src/logger/streams/sentry.ts

@@ -0,0 +1,18 @@
+import { SentryStream } from 'bunyan-sentry-stream';
+import { Client as RavenClient } from 'raven';
+
+import config from 'turtle/config';
+
+export default function create() {
+  if (!config.sentry.dsn || config.deploymentEnv === 'development') {
+    return null;
+  }
+
+  const raven = new RavenClient();
+  raven.config(config.sentry.dsn, { environment: config.deploymentEnv });
+  return {
+    level: 'error',
+    type: 'raw', // Mandatory type for SentryStream
+    stream: new SentryStream(raven),
+  };
+}

src/scripts/setConfigurations.ts

@@ -5,7 +5,7 @@ import {
   createConfigurationsKey,
   createDefaultConfigurationKey,
   HIGH_CONFIGURATION,
-  NORMAL_CONFIGURATION,
+  HIGH_ONLY_CONFIGURATION,
 } from 'turtle/utils/priorities';
 import { connect, RedisClient } from 'turtle/utils/redis';
 
@@ -27,31 +27,31 @@ const TIMEOUT = 3000;
 // If there are no more available configurations from the table, the default configuration is used.
 
 const AVAILABLE_CONFIGURATIONS_IOS = [
+  HIGH_CONFIGURATION,
+  HIGH_ONLY_CONFIGURATION,
+  HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
-  NORMAL_CONFIGURATION,
-  NORMAL_CONFIGURATION,
   HIGH_CONFIGURATION,
-  NORMAL_CONFIGURATION,
 ];
 
 const AVAILABLE_CONFIGURATIONS_ANDROID = [
+  HIGH_CONFIGURATION,
+  HIGH_ONLY_CONFIGURATION,
+  HIGH_CONFIGURATION,
+  HIGH_ONLY_CONFIGURATION,
+  HIGH_CONFIGURATION,
+  HIGH_CONFIGURATION,
+  HIGH_CONFIGURATION,
+  HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
-  NORMAL_CONFIGURATION,
-  NORMAL_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
-  NORMAL_CONFIGURATION,
-  NORMAL_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
-  NORMAL_CONFIGURATION,
-  NORMAL_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
-  NORMAL_CONFIGURATION,
-  NORMAL_CONFIGURATION,
 ];
 
 const DEFAULT_CONFIGURATION = HIGH_CONFIGURATION;

src/utils/priorities.ts

@@ -10,6 +10,7 @@ export const HIGH = 'highPriority';
 
 export const NORMAL_CONFIGURATION = [NORMAL, HIGH];
 export const HIGH_CONFIGURATION = [HIGH, NORMAL];
+export const HIGH_ONLY_CONFIGURATION = [HIGH];
 
 const REDIS_GET_CONFIG = `
   local i = 0

yarn.lock

@@ -161,10 +161,10 @@
     mv "~2"
     safe-json-stringify "~1"
 
-"@expo/config@^1.0.2":
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/@expo/config/-/config-1.0.2.tgz#1c2c4683895ecc5407f1a3a8868debeee22904bb"
-  integrity sha512-5xp1RLbIxxNw2MwT+zRKdGNMrsPj4OcmBM5T602yQNRc+kmzXAVAHcpVlSk1QfZ5GylTUmGlwcM5X/yhiTBsGw==
+"@expo/config@^1.0.3":
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/@expo/config/-/config-1.0.3.tgz#7e99eae02f103aa9e2c0597ebef9e509a84ceb7d"
+  integrity sha512-eTX0PedX+x9jjYUlc/dO9lwsLt01V7zDiDsS4n02GVY1X+dLGuq7NiZ3nfazJ3EBE45fJn3lGNydOdRGZFC6lQ==
   dependencies:
     "@expo/json-file" "^8.1.2"
     fs-extra "^7.0.1"
@@ -304,15 +304,15 @@
     "@types/cross-spawn" "^6.0.0"
     cross-spawn "^6.0.5"
 
-"@expo/traveling-fastlane-darwin@^1.8.1":
-  version "1.8.1"
-  resolved "https://registry.yarnpkg.com/@expo/traveling-fastlane-darwin/-/traveling-fastlane-darwin-1.8.1.tgz#37c16a253e92d3ef6f543e7271b7239e6edbbb5f"
-  integrity sha512-Nt+579oBMrMEzVn0Esv7c8nIOwoDX/xS6CThoWPRFMVIeQ52aK1mf7wu91/+FCU/fvfUFZ/evERIDWS1lObRxA==
+"@expo/traveling-fastlane-darwin@1.9.4":
+  version "1.9.4"
+  resolved "https://registry.yarnpkg.com/@expo/traveling-fastlane-darwin/-/traveling-fastlane-darwin-1.9.4.tgz#29b370a6d6e115e69c4c4ea53003d679779b4592"
+  integrity sha512-bKw/sRPd0gXyzXhrvU9ewhav6FXstQC/g9P9ulf3x0Y2bxNYu7g1zv3hrNJxVbKx0DsFMzXqesV8HD8mU6jwrw==
 
-"@expo/webpack-config@^0.5.2":
-  version "0.5.2"
-  resolved "https://registry.yarnpkg.com/@expo/webpack-config/-/webpack-config-0.5.2.tgz#eafc7ac6ec6581629e40962eafe8d71f7c2eef20"
-  integrity sha512-YcySFO5O++UbBJbP1FNHCQJ+RoIvBkUZ8fVcQ/46mdMSULsZoiNouUG4bJUhFiAdzwbzwIv8psOdp7hhLcCKMQ==
+"@expo/webpack-config@^0.5.3":
+  version "0.5.3"
+  resolved "https://registry.yarnpkg.com/@expo/webpack-config/-/webpack-config-0.5.3.tgz#51b0d376b6404f1eeb61fbe6753868d68dcd6973"
+  integrity sha512-mltsA3jLK4gld7uzjWdzeEVvOQG9fuj1fTFC3zVH/AZrxi8NDWspZOGdtucew20RkqzZTNgLrMiZ+n56n6mGnw==
   dependencies:
     "@babel/core" "^7.0.0"
     "@babel/polyfill" "^7.2.5"
@@ -13069,19 +13069,19 @@ xdg-basedir@^3.0.0:
   resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-3.0.0.tgz#496b2cc109eca8dbacfe2dc72b603c17c5870ad4"
   integrity sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=
 
-[email protected].3:
-  version "53.5.3"
-  resolved "https://registry.yarnpkg.com/xdl/-/xdl-53.5.3.tgz#2aa96cd67772372d9d4abfc54a026a390b0d631b"
-  integrity sha512-zN4/Urah/VyLBA8bdAlDVgsp2EaxMklVx0TMAU86ftuzmgeUBvvLMd5/wVuiVhlNg8sKsav4T/cWTPgk8ukE/Q==
+[email protected].4:
+  version "53.5.4"
+  resolved "https://registry.yarnpkg.com/xdl/-/xdl-53.5.4.tgz#50e300bb89cd2703f7a458227eba5e4aa3a16863"
+  integrity sha512-1H4xYzm51zQD9KaifOf2ike3DByHGpdIjYM2TRjCBwPbi82mVbeXUiGL+0azYcHCWqE07LnRzMvlcCD9+WhBUg==
   dependencies:
     "@expo/bunyan" "3.0.2"
-    "@expo/config" "^1.0.2"
+    "@expo/config" "^1.0.3"
     "@expo/json-file" "^8.1.2"
     "@expo/ngrok" "2.4.3"
     "@expo/osascript" "^1.9.2"
     "@expo/schemer" "^1.2.4"
     "@expo/spawn-async" "1.4.2"
-    "@expo/webpack-config" "^0.5.2"
+    "@expo/webpack-config" "^0.5.3"
     analytics-node "^2.1.0"
     axios v0.19.0-beta.1
     chalk "2.4.1"