Compare commits

..
Author SHA1 Message Date
Dominik Sokal 73fa81a1f4 oops 2019-03-27 12:07:52 +01:00
Dominik Sokal 57f2d985c5 debug 2019-03-27 12:06:55 +01:00
62 changed files with 4902 additions and 1078 deletions

No files matched your search

+16
View File
@@ -0,0 +1,16 @@
{
"presets": [
["env", {
"targets": {
"node": "8.9"
}
}]
],
"plugins": [
["module-resolver", {
"alias": {
"turtle": "./build"
}
}]
]
}
+18 -198
View File
@@ -1,212 +1,32 @@
version: 2.1
executors:
nix: &nix
docker:
- image: lnl7/nix:2.2
# Symlink to the readline-enabled bash which is the default command of the container
shell: /run/current-system/sw/bin/bash -leo pipefail
working_directory: ~/turtle
mac:
macos: # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions
xcode: '10.1.0'
working_directory: /Users/distiller/project
shell: /bin/bash -leo pipefail
android:
<<: *nix
resource_class: xlarge
commands:
install_nix:
steps:
- run:
name: Install nix
command: curl https://nixos.org/releases/nix/nix-2.2.1/install | bash -s -- --no-daemon
- run:
name: Add nix-initializing line to .bashrc
command: echo ". ~/.nix-profile/etc/profile.d/nix.sh" >> ~/.bashrc
setup:
steps:
- run:
name: Install common packages
command: |
nix-env -iA \
nixpkgs.binutils \
nixpkgs.direnv \
nixpkgs.findutils \
nixpkgs.gawk \
nixpkgs.git \
nixpkgs.gnutar \
nixpkgs.gzip \
nixpkgs.openssh \
nixpkgs.python \
nixpkgs.xz
- run:
name: Configure direnv
command: |
mkdir -p ~/.config/direnv
echo -e "[whitelist]\nprefix = [ \"$HOME\" ]" > ~/.config/direnv/config.toml
echo 'eval "$(direnv export bash)"' >> ~/.bash_profile
- run:
name: Configure yarn
command: echo '--frozen-lockfile true' >> ~/.yarnrc
- checkout
- run:
name: Configure aws-cli
command: |
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
setup_nix:
steps:
- run:
name: Add cp and rm stubs
command: |
cat \<< 'EOF' >> /bin/cp
#! /usr/bin/env bash
/usr/bin/env cp "$@"
EOF
chmod +x /bin/cp
cat \<< 'EOF' >> /bin/rm
#! /usr/bin/env bash
/usr/bin/env rm "$@"
EOF
chmod +x /bin/rm
decrypt_secrets_if_possible:
steps:
- run:
name: Decrypt secrets if possible
command: |
if [[ -z "$TURTLE_GIT_CRYPT_KEY_BASE64" ]]; then
echo 'git-crypt key not present in environment'
else
git crypt unlock <(echo $TURTLE_GIT_CRYPT_KEY_BASE64 | base64 --decode)
fi
restore_yarn_cache:
parameters:
workingdir:
type: string
default: .
steps:
- restore_cache:
key: yarn-v1-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
save_yarn_cache:
parameters:
workingdir:
type: string
default: .
steps:
- save_cache:
key: yarn-v1-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
paths:
- ~/.cache/yarn # `yarn cache dir`
restore_workingdir_cache:
parameters:
platform:
type: string
default: ios
sdk_version:
type: string
default: sdk32
steps:
- restore_cache:
key: "workingdir-v1-<< parameters.platform >>-<< parameters.sdk_version >>"
save_workingdir_cache:
parameters:
workingdir:
type: string
default: /Users/distiller/project/workingdir
platform:
type: string
default: ios
sdk_version:
type: string
default: sdk32
steps:
- save_cache:
key: "workingdir-v1-<< parameters.platform >>-<< parameters.sdk_version >>"
paths:
- "<< parameters.workingdir >>/<< parameters.platform >>/<< parameters.sdk_version >>"
version: 2
jobs:
build:
executor: nix
docker:
- image: circleci/node:8.12
steps:
- setup
- restore_yarn_cache
- checkout
- run: yarn
- save_yarn_cache
- run: yarn lint
- run: yarn build
unit_tests:
executor: nix
test:
docker:
- image: circleci/node:8.12
steps:
- setup
- restore_yarn_cache
- checkout
- run: yarn
- save_yarn_cache
- run:
name: Run unit tests
command: ./scripts/tests/runUnitTestsCI.sh
ios_smoke_tests:
executor: mac
- run: ./scripts/testUnitCI.sh
abc:
docker:
- image: circleci/node:8.12
steps:
- install_nix
- setup
- decrypt_secrets_if_possible
- restore_yarn_cache
- run: yarn
- save_yarn_cache
- restore_workingdir_cache:
sdk_version: sdk32
platform: ios
- run:
name: Run iOS smoke tests
command: ./scripts/tests/runIOSSmokeTestsCI.sh
- save_workingdir_cache:
workingdir: /Users/distiller/project/workingdir
sdk_version: sdk32
platform: ios
android_smoke_tests:
executor: android
steps:
- setup
- setup_nix
- restore_yarn_cache
- run: yarn
- save_yarn_cache
- restore_workingdir_cache:
sdk_version: sdk32
platform: android
- run:
name: Fetch Android shell app
command: SDK_VERSION=sdk32 ./scripts/android/fetchRemoteAndroidTarball.sh
- run:
name: Run Android smoke tests
command: |
nix-shell nix/android-shell.nix --pure \
--run "./scripts/tests/runAndroidSmokeTestsCI.sh"
- save_workingdir_cache:
workingdir: ~/turtle/workingdir
sdk_version: sdk32
platform: android
- checkout
- run: echo "123"
workflows:
checks:
version: 2
build_and_test:
jobs:
- build
tests:
jobs:
- unit_tests
- ios_smoke_tests
- android_smoke_tests
- test
- abc
+2 -8
View File
@@ -1,4 +1,4 @@
#!/usr/bin/env bash
#!/bin/bash
source_local() {
file=./.envrc.local
@@ -15,10 +15,4 @@ export NODE_ENV=development
export EXPO_TURTLE_TOOLS_DIR="$(pwd)/tools"
if [ -z "$EXPO_UNIVERSE_DIR" ]; then
PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
fi
if [[ $CI = "true" ]]; then
PATH_add $EXPO_TURTLE_TOOLS_DIR/ci/bin
fi
PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
+1 -3
View File
@@ -1,12 +1,10 @@
!/src/node_modules
node_modules
yarn-error.log
/build/
/workingdir/*
/artifacts/*
shell.apk
node_modules
.secrets*
*.swp
-1
View File
@@ -1 +0,0 @@
v8.15.1
+1
View File
@@ -2,6 +2,7 @@ LOGGER_LEVEL="debug"
API_PROTOCOL="http"
API_HOSTNAME="localhost"
API_PORT="3000"
LOGGLY_TOKEN=""
SENTRY_DSN=""
ENVIRONMENT="development"
REDIS_URL="redis://localhost:6379/0"
-16
View File
@@ -5,22 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Changed
- Updated iOS shell app for SDK 32 to fix AppAuth issue
## [0.6.0] - 2019-04-09
### Changed
- Removed Branch module from Android shell apps for all SDK versions
## [0.5.15] - 2019-04-03
### Added
- Turtle smoke tests are now being executed on CircleCI from the context of this repository.
### Changed
- Updated iOS shell app for SDK 32 (fixes [this loading issue for slow connections](https://github.com/expo/expo/issues/3574)).
- Minor infra changes (doesn't affect users).
- Error logs format - error objects are now passed in an object being passed as the first argument to logger function.
### Removed
- Sending logs to loggly.
## [0.5.14] - 2019-03-22
### Added
+10
View File
@@ -99,6 +99,16 @@ RUN rm gradle-4.4-all.zip
ENV GRADLE_HOME /usr/local/gradle-4.4
ENV PATH ${GRADLE_HOME}/bin:$PATH
# Install Gulp
RUN npm install -g gulp-cli
# Install Git
RUN echo 'deb http://http.debian.net/debian wheezy-backports main' >> /etc/apt/sources.list && \
apt-get update && \
apt-get -t wheezy-backports install git-core && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
ADD . /app
RUN for SDK_VERSION in `ls /app/workingdir/android/`; do \
+91
View File
@@ -0,0 +1,91 @@
'use strict';
const childProcess = require('child_process');
const util = require('util');
const path = require('path');
const gulp = require('gulp');
const babel = require('gulp-babel');
const changed = require('gulp-changed');
const nodemon = require('gulp-nodemon');
const plumber = require('gulp-plumber');
const sourcemaps = require('gulp-sourcemaps');
const ts = require('gulp-typescript');
const rimraf = require('rimraf');
const rimrafAsync = util.promisify(rimraf);
const tsProject = ts.createProject('tsconfig.json');
gulp.task('clean', async () => {
console.log(`Removing build directories...`);
await rimrafAsync('build');
});
gulp.task('build:server', () => {
console.log(`Compiling the Node.js server JavaScript...`);
return tsProject
.src()
.pipe(plumber())
.pipe(changed('build'))
.pipe(sourcemaps.init())
.pipe(tsProject())
.pipe(babel())
.pipe(
sourcemaps.write('.', {
includeContent: false,
sourceRoot: '../src',
})
)
.pipe(gulp.dest('build'));
});
gulp.task('watch:source', async () => {
console.log(`Watching for changes to the Node.js server JavaScript...`);
gulp.watch('src/**/*.(t|j)s', gulp.series('build:server'));
});
gulp.task('watch:local', createWatchTask('development'));
gulp.task('watch:server', createWatchTask('production'));
function createWatchTask(nodeEnv) {
return async () => {
console.log(`Starting the Node.js server and watching for changes (NODE_ENV=${nodeEnv})...`);
// Run nodemon in a separate process to isolate its side effects on stdio streams
let nodeCmd = 'node';
if (nodeEnv === 'development') {
nodeCmd += ' --inspect=:9315';
}
nodemon({
script: './build/server.js',
execMap: {
js: nodeCmd,
},
env: {
NODE_ENV: nodeEnv,
},
watch: ['./build'],
stdout: false,
readable: false,
}).on('readable', function() {
const bunyan = childProcess.fork(
path.join('.', 'node_modules', '.bin', 'bunyan'),
['-o', 'short', '-l', 'debug', '--color'],
{ silent: true }
);
bunyan.stdout.pipe(process.stdout);
bunyan.stderr.pipe(process.stderr);
this.stdout.pipe(bunyan.stdin);
this.stderr.pipe(bunyan.stdin);
});
};
}
gulp.task(
'watch',
gulp.series('clean', 'build:server', gulp.parallel('watch:source', 'watch:local'))
);
gulp.task('default', gulp.series('watch'));
@@ -50,6 +50,8 @@ spec:
value: exp-shell-app-assets
- name: AWS_S3_REGION
value: us-west-1
- name: LOGGLY_SUBDOMAIN
value: exponent
- name: TURTLE_TEMPORARY_FILES_DIR
value: /tmp/turtle
# vars from k8s secrets
@@ -98,6 +100,11 @@ spec:
secretKeyRef:
name: turtle-js-config
key: aws_cloudwatch_region
- name: LOGGLY_TOKEN
valueFrom:
secretKeyRef:
name: turtle-js-config
key: loggly_token
- name: REDIS_URL
valueFrom:
secretKeyRef:
-5
View File
@@ -1,5 +0,0 @@
self: super:
{
nodejs = super.nodejs-8_x;
}
-50
View File
@@ -1,50 +0,0 @@
with (import ./default.nix {});
let
ndk = androidenv.androidndk_17c.override { fullNDK = true; };
ndk_root = "${ndk}/libexec/${ndk.name}";
sdk_path = if stdenv.isDarwin
then "/Library/Android/sdk"
else "/Android/Sdk"; # intentional Capital S
in
mkShell rec {
LANG="en_US.UTF-8";
JAVA_HOME=openjdk8;
ANDROID_SDK_ROOT=builtins.getEnv("HOME") + sdk_path;
ANDROID_NDK_ROOT="${ANDROID_SDK_ROOT}/ndk-bundle";
ANDROID_HOME=ANDROID_SDK_ROOT;
ANDROID_NDK=ANDROID_NDK_ROOT;
nativeBuildInputs = [
awscli
curl
git
nodejs-8_x
openjdk8
yarn
];
passthru = { inherit ndk; };
shellHook = ''
${./install-ndk-17c.sh} ${ndk} ${ndk_root}
yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --licenses --sdk_root="$ANDROID_SDK_ROOT"
yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --sdk_root="$ANDROID_SDK_ROOT" "platforms;android-28" | grep -v =
yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --sdk_root="$ANDROID_SDK_ROOT" "build-tools;28.0.3" | grep -v =
${lib.optionalString stdenv.isLinux ''
for dep in lib lib64; do
if [ -L /$dep ] || [ ! -e /$dep ]; then
echo "Creating /$dep"
ln -s ${stdenv.cc.libc}/$dep /$dep
else
echo "Using existing /$dep"
fi
done
''}
'';
}
-19
View File
@@ -1,19 +0,0 @@
let
pin = builtins.fromJSON (builtins.readFile ./nixpkgs.json);
nixpkgs = builtins.fetchTarball {
url = "https://github.com/NixOS/nixpkgs-channels/archive/${pin.rev}.tar.gz";
inherit (pin) sha256;
};
in
{ ... } @ args:
import nixpkgs (args // {
config = {
allowUnfree = true;
android_sdk.accept_license = true;
};
overlays = [ (import ./all-packages.nix) ];
})
-14
View File
@@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
scriptdir="$(dirname ${BASH_SOURCE[0]})"
shellnix="$scriptdir/android-shell.nix"
ndk=${1:-$(nix-build "$shellnix" -A ndk --no-out-link)}
ndkRoot=${2:-$(nix eval --file "$shellnix" --raw ANDROID_NDK_ROOT)}
mkdir -p "$ANDROID_NDK_ROOT" # Create full path if necessary
# Replace any existing link or directory with link
rm -r "$ANDROID_NDK_ROOT"
ln -sn "$ndkRoot" "$ANDROID_NDK_ROOT"
-7
View File
@@ -1,7 +0,0 @@
{
"url": "https://github.com/NixOS/nixpkgs-channels.git",
"rev": "80738ed9dc0ce48d7796baed5364eef8072c794d",
"date": "2018-11-17T11:18:10+01:00",
"sha256": "0anmvr6b47gbbyl9v2fn86mfkcwgpbd5lf0yf3drgm8pbv57c1dc",
"fetchSubmodules": false
}
-6
View File
@@ -1,6 +0,0 @@
{
"watch": ["src"],
"ext": "ts",
"ignore": ["src/**/*.test.ts"],
"exec": "ts-node -r tsconfig-paths/register --files src/server.ts"
}
+24 -29
View File
@@ -1,17 +1,17 @@
{
"name": "turtle-cli",
"version": "0.6.0",
"version": "0.5.14",
"description": "The command-line tool for building standalone Expo apps",
"preferGlobal": true,
"scripts": {
"start": "EXPENV_OVERRIDES=local-common,local,local-queue expenv nodemon | bunyan -o short -l debug --color",
"start": "EXPENV_OVERRIDES=local-common,local,local-queue expenv gulp",
"start:ios": "PLATFORM=ios yarn start",
"start:android": "PLATFORM=android yarn start",
"watch": "EXPENV_OVERRIDES=local-common,local,local-queue expenv gulp watch:source",
"bin:turtle": "./bin/turtle.js",
"lint": "tslint -c tslint.json 'src/**/*.ts'",
"lint:fix": "yarn lint --fix",
"prepare": "yarn build",
"build": "rimraf ./build && ttsc",
"build": "gulp build:server",
"build:production": "./scripts/build.sh",
"build:android-tarball:local": "./scripts/android/buildAndroidTarballLocally.sh",
"secrets:init-private": "./scripts/initPrivateSecrets.sh",
@@ -26,7 +26,7 @@
"test:integration": "yarn test --config jest/integration-config.json",
"test:unit": "yarn test --config jest/unit-config.json",
"test:smoke": "yarn test --config jest/smoke-config.json --forceExit",
"test:smoke:android": "PLATFORM=android yarn test:smoke --testNamePattern=android",
"test:smoke:android": "yarn test:smoke --testNamePattern=android",
"test:smoke:ios": "PLATFORM=ios yarn test:smoke --testNamePattern=iOS",
"release": "./scripts/release.sh",
"release:beta": "./scripts/releaseBeta.sh",
@@ -34,21 +34,9 @@
"ssh:android:connect": "./scripts/ssh/connectToAndroidPod.sh",
"ssh:ios:ips": "./scripts/ssh/findIosIps.sh",
"set-priorities": "yarn set-priorities:local",
"set-priorities:default": "expenv ts-node -r tsconfig-paths/register --files src/scripts/setConfigurations.ts",
"set-priorities:local": "EXPENV_OVERRIDES=local-common,local,local-queue yarn set-priorities:default",
"set-priorities:staging": "yarn set-priorities:default",
"set-priorities:production": "EXPENV_OVERRIDES=prod yarn set-priorities:default"
},
"husky": {
"hooks": {
"pre-commit": "lint-staged"
}
},
"lint-staged": {
"src/**/*.ts": [
"yarn lint:fix",
"git add"
]
"set-priorities:local": "EXPENV_OVERRIDES=local-common,local,local-queue expenv babel-node scripts/setConfigurations.js",
"set-priorities:staging": "expenv babel-node scripts/setConfigurations.js",
"set-priorities:production": "EXPENV_OVERRIDES=prod expenv babel-node scripts/setConfigurations.js"
},
"bin": {
"turtle": "./bin/turtle.js"
@@ -88,21 +76,27 @@
"@types/tar": "^4.0.0",
"@types/uuid": "^3.4.3",
"@types/which": "^1.3.1",
"husky": "^1.3.1",
"babel-cli": "^6.26.0",
"babel-core": "^6.26.3",
"babel-jest": "^23.6.0",
"babel-plugin-module-resolver": "^3.1.1",
"babel-preset-env": "^1.6.1",
"gulp": "github:gulpjs/gulp#4.0",
"gulp-babel": "^7.0.1",
"gulp-changed": "^3.2.0",
"gulp-nodemon": "^2.4.1",
"gulp-plumber": "^1.2.0",
"gulp-sourcemaps": "^2.6.4",
"gulp-typescript": "^4.0.2",
"jest": "^23.6.0",
"lint-staged": "^8.1.5",
"nock": "^10.0.1",
"nodemon": "^1.18.11",
"nodemon": "^1.18.7",
"release-it": "^10.0.0",
"rimraf": "^2.6.2",
"source-map-loader": "^0.2.3",
"ts-jest": "^23.10.4",
"ts-node": "^8.0.3",
"ts-transformer-imports": "^0.4.1",
"tsconfig-paths": "^3.8.0",
"tslint": "^5.10.0",
"tslint-eslint-rules": "^5.4.0",
"ttypescript": "^1.5.6",
"typescript": "^3.4.2"
"typescript": "^3.0.1"
},
"dependencies": {
"@expo/spawn-async": "^1.4.0",
@@ -111,6 +105,7 @@
"aws-sdk": "^2.226.1",
"bunyan": "^1.8.12",
"bunyan-debug-stream": "^1.0.8",
"bunyan-loggly": "^1.3.4",
"bunyan-sentry-stream": "^1.2.1",
"commander": "^2.15.1",
"copy": "^0.3.1",
+21 -1
View File
@@ -1,4 +1,24 @@
#!/usr/bin/env bash
set -xeo pipefail
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
$ROOT_DIR/scripts/fetchRemoteTarballs.sh android
ARTIFACTS_DIR="$ROOT_DIR/artifacts"
WORKING_DIR="$ROOT_DIR/workingdir/android"
ANDROID_SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/android"
mkdir -p $ARTIFACTS_DIR
for SDK_VERSION in `ls $ANDROID_SHELL_TARBALLS_DIRECTORY`; do
mkdir -p $WORKING_DIR/$SDK_VERSION
TARBALL_URI=$(head -n 1 $ANDROID_SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
if command -v run >/dev/null 2>&1 ; then
run expo.s3env --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --command retry5 aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
else
# outside of universe aws credentials might not be set
if ! aws iam get-user >/dev/null 2>&1 ; then aws configure ; fi
aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
fi
tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
rm -rf $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
done
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env bash
set -xeo pipefail
PLATFORM=$1
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
ARTIFACTS_DIR="$ROOT_DIR/artifacts"
WORKING_DIR="$ROOT_DIR/workingdir/$PLATFORM"
SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/$PLATFORM"
mkdir -p $ARTIFACTS_DIR
convert_s3_to_https() {
BUCKET=`echo $1 | sed 's/s3:\/\/\([^/]*\).*/\1/'`
KEY=`echo $1 | sed 's/s3:\/\/[^/]*\/\(.*\)/\1/'`
echo "https://s3.amazonaws.com/$BUCKET/$KEY"
}
download_shell_app() {
SDK_VERSION=$1
mkdir -p $WORKING_DIR/$SDK_VERSION
TARBALL_S3_URL=$(head -n 1 $SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
TARBALL_HTTPS_URL=$(convert_s3_to_https $TARBALL_S3_URL)
curl -s $TARBALL_HTTPS_URL -o $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
rm -rf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
}
if [[ -z "$SDK_VERSION" ]]; then
for SDK_VERSION in `ls $SHELL_TARBALLS_DIRECTORY`; do
download_shell_app $SDK_VERSION
done
else
if [ ! -d "$WORKING_DIR/$SDK_VERSION" ]; then
download_shell_app $SDK_VERSION
fi
fi
+21 -1
View File
@@ -1,4 +1,24 @@
#!/usr/bin/env bash
set -xeo pipefail
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
$ROOT_DIR/scripts/fetchRemoteTarballs.sh ios
ARTIFACTS_DIR="$ROOT_DIR/artifacts"
WORKING_DIR="$ROOT_DIR/workingdir/ios"
IOS_SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/ios"
mkdir -p $ARTIFACTS_DIR
for SDK_VERSION in `ls $IOS_SHELL_TARBALLS_DIRECTORY`; do
mkdir -p $WORKING_DIR/$SDK_VERSION
TARBALL_URI=$(head -n 1 $IOS_SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
if command -v run >/dev/null 2>&1 ; then
run expo.s3env --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --command retry5 aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
else
# outside of universe aws credentials might not be set
if ! aws iam get-user >/dev/null 2>&1 ; then aws configure ; fi
aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
fi
tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
rm -rf $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
done
@@ -1,13 +1,12 @@
import { env } from 'turtle/utils/env';
import { connect, RedisClient } from 'turtle/utils/redis';
import { PLATFORMS } from 'turtle/constants';
import logger from 'turtle/logger';
import {
createConfigurationsKey,
createDefaultConfigurationKey,
HIGH_CONFIGURATION,
NORMAL_CONFIGURATION,
} from 'turtle/utils/priorities';
import { connect, RedisClient } from 'turtle/utils/redis';
const TIMEOUT = 3000;
@@ -26,6 +25,7 @@ const TIMEOUT = 3000;
// they should come back to using configurations no. 1 and 2.
// If there are no more available configurations from the table, the default configuration is used.
const AVAILABLE_CONFIGURATIONS_IOS = [
HIGH_CONFIGURATION,
HIGH_CONFIGURATION,
@@ -54,6 +54,7 @@ const AVAILABLE_CONFIGURATIONS_ANDROID = [
NORMAL_CONFIGURATION,
];
const DEFAULT_CONFIGURATION = HIGH_CONFIGURATION;
async function run() {
@@ -67,8 +68,6 @@ async function run() {
redis.disconnect();
}
if (require.main === module) {
run()
.then(() => logger.info('All done'))
.catch((err) => logger.error('Error:', err));
}
run()
.then(() => console.log('All done'))
.catch(err => console.error('Error:', err));
@@ -1,25 +1,34 @@
#!/usr/bin/env bash
set -ueo pipefail
export NODE_ENV=test
export ENVIRONMENT="development"
export AWS_ACCESS_KEY_ID="aws_id"
export AWS_SECRET_ACCESS_KEY="secret"
export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in.fifo"
export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in_priority.fifo"
export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in.fifo"
export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in_priority.fifo"
export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_out.fifo"
export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_ios_in.fifo"
export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_ios_in.fifo"
export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_android_in.fifo"
export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_android_in.fifo"
export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_out.fifo"
export AWS_SQS_REGION="aws.west2"
export AWS_S3_BUCKET="exp-shell-app-assets"
export AWS_S3_REGION="us-west-1"
export AWS_CLOUDWATCH_REGION="us-west-1"
export API_PROTOCOL="http"
export API_HOSTNAME="staging.expo.io"
export API_PORT="443"
export API_HOSTNAME="localhost"
export API_PORT="3000"
export REDIS_URL="noop"
export REDIS_CONFIG_URL="noop"
yarn test:smoke:android
smoke_result=$?
yarn test:unit
unit_result=$?
[[ $smoke_result != 0 || $unit_result != 0 ]] && \
echo && \
echo "Tests failed." && \
exit -1
exit 0
@@ -1,7 +1,5 @@
#!/usr/bin/env bash
set -ueo pipefail
export NODE_ENV=test
export ENVIRONMENT="development"
@@ -20,7 +18,18 @@ export API_PORT="443"
export REDIS_URL="noop"
export REDIS_CONFIG_URL="noop"
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
SDK_VERSION=sdk32 $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
$ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
yarn test:smoke:ios
smoke_result=$?
yarn test:unit
unit_result=$?
[[ $smoke_result != 0 || $unit_result != 0 ]] && \
echo && \
echo "Tests failed." && \
exit -1
exit 0
@@ -1,7 +1,5 @@
#!/usr/bin/env bash
set -ueo pipefail
export NODE_ENV=test
export ENVIRONMENT="development"
@@ -23,3 +21,6 @@ export REDIS_URL="noop"
export REDIS_CONFIG_URL="noop"
yarn test:unit
unit_result=$?
exit $unit_result
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-89b2d24c86737d5a489c6a18467e31b2f9ce9005.tar.gz
s3://exp-artifacts/android-shell-builder-5c339ff7bab6f2161d378cc70e1bf04d32ad98dd.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-0cab68670fc50ce1dc16b7d86ca7976870062c39.tar.gz
s3://exp-artifacts/android-shell-builder-a4bd3bd26b1b31bdfa66c39b6b4d0d1d138c32fc.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/ios-shell-builder-sdk-latest-dc981c7110b7d508de8e9c352064dc3d745c9f72.tar.gz
s3://exp-artifacts/ios-shell-builder-sdk-latest-d9cabffbcba20e074e91e4d7f8d862c98acfc680.tar.gz
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -179,7 +179,7 @@ function _pushMetrics(metrics: IMetricsChunk[]) {
metrics.forEach((metricsChunk) => {
cloudWatch.putMetricData(metricsChunk, (err) => {
if (err) {
logger.warn({ err }, '[cloudwatch]');
logger.warn('[cloudwatch]', err);
}
});
});
+1 -1
View File
@@ -7,7 +7,7 @@ export function sum(metrics: IMetric[]) {
const reduced = metrics.reduce((acc: IMetric | null, i: IMetric) => {
let result = acc;
if (!result) {
result = { ...i };
result = {...i};
} else {
result.Value += i.Value;
}
+3 -3
View File
@@ -36,7 +36,7 @@ export async function receiveMessage(priority: string) {
}
})
.catch((err) => {
logger.error({ err }, 'Receive error');
logger.error('Receive error', err);
throw err;
});
});
@@ -68,7 +68,7 @@ export function changeMessageVisibilityRecurring(priority: string, receiptHandle
return setInterval(() => {
if (getCurrentJobId() === jobId) {
changeMessageVisibility(priority, receiptHandle).catch((err) => {
logger.warn({ err }, 'Error at change msg visibility');
logger.warn('Error at change msg visibility', err);
});
}
}, (VISIBILITY_TIMEOUT_SEC * 1000) / 3);
@@ -86,6 +86,6 @@ export async function sendMessage(jobId: string, status: JOB_UPDATE_TYPE, data =
return await sqs.sendMessage(params).promise();
});
} catch (err) {
logger.error({ err }, `Error sending SQS message: ${JSON.stringify(params)}}`);
logger.error(`Error sending SQS message: ${JSON.stringify(params)}}`, err);
}
}
+2 -1
View File
@@ -33,7 +33,8 @@ async function setupAction(program: any, cmd: any, platform: string, os?: string
await setup(platform, sdkVersion);
l.info('it\'s all set!');
} catch (err) {
logger.error({ err }, `Failed to setup environment for ${platform} builds`);
logger.error(`Failed to setup environment for ${platform} builds`);
logger.error(err.stack);
if (err instanceof ErrorWithCommandHelp) {
cmd.help();
} else if (err instanceof ErrorWithProgramHelp) {
+4 -4
View File
@@ -17,16 +17,16 @@ Command.prototype.asyncAction = function asyncAction(asyncFn: (...asyncFnArgs: a
return this.action(async (...args) => {
try {
await checkForUpdateAsync();
} catch (err) {
logger.warn({ err }, 'Failed to check for turtle-cli update.');
} catch (e) {
logger.warn('Failed to check for turtle-cli update.');
}
return await asyncFn(...args);
});
};
export function run(programName: string) {
runAsync(programName).catch((err) => {
logger.error({ err }, 'Uncaught Error');
runAsync(programName).catch((e) => {
logger.error('Uncaught Error', e);
process.exit(1);
});
}
+2 -2
View File
@@ -26,9 +26,9 @@ export async function ensureToolsAreInstalled(tools: IToolDefinition[]) {
} catch (err) {
isAnyToolMissing = true;
if (!testFn) {
l.error({ err }, `${command} is missing in your $PATH`);
l.error(`${command} is missing in your $PATH`);
}
l.error({ err }, missingDescription);
l.error(missingDescription);
}
}
if (isAnyToolMissing) {
+2 -1
View File
@@ -72,7 +72,8 @@ export function createBuilderAction({
const job = await sanitizeJob(rawJob);
await builder(job);
} catch (err) {
logger.error({ err }, 'Failed to build standalone app');
logger.error(`Failed to build standalone app: ${err.message}`);
logger.error(err.stack);
if (err instanceof ErrorWithCommandHelp) {
command.help();
} else if (err instanceof ErrorWithProgramHelp) {
+3 -3
View File
@@ -16,7 +16,7 @@ export async function create(ctx: IContext): Promise<IKeychain> {
l.info('done creating keychain');
return keychainInfo;
} catch (err) {
l.error({ err }, 'unable to create keychain');
l.error('unable to create keychain');
throw err;
}
}
@@ -32,7 +32,7 @@ export async function remove(ctx: IContext, keychainPath: string) {
l.info('done deleting keychain');
return keychainInfo;
} catch (err) {
l.error({ err }, 'unable to delete keychain');
l.error('unable to delete keychain');
throw err;
}
}
@@ -57,7 +57,7 @@ export async function importCert(
await IosKeychain.importIntoKeychain({ keychainPath, certPath, certPassword });
l.info('done importing distribution certificate into keychain');
} catch (err) {
l.error({ err }, 'unable to import distribution certificate into keychain');
l.error('unable to import distribution certificate into keychain');
throw err;
}
}
+5
View File
@@ -47,6 +47,11 @@ export default {
client: {
level: env('CLIENT_LOGGER_LEVEL', 'info'),
},
loggly: {
token: env('LOGGLY_TOKEN', ''),
subdomain: env('LOGGLY_SUBDOMAIN', ''),
buffer: envNum('LOGGLY_BUFFER', 100),
},
},
google: {
credentials: env('GOOGLE_APPLICATION_CREDENTIALS', ''),
+2 -2
View File
@@ -47,7 +47,7 @@ export async function getJob() {
return job;
}
} catch (err) {
logger.error({ err }, 'Error at receiving messages');
logger.error('Error at receiving messages', err);
}
}
}
@@ -160,6 +160,6 @@ async function deleteMessage(priority: string, receiptHandle: string) {
setCurrentJobId(null);
await sqs.deleteMessage(priority, receiptHandle);
} catch (err) {
logger.error({ err }, 'Error at deleting msg');
logger.error('Error at deleting msg', err);
}
}
+48
View File
@@ -0,0 +1,48 @@
import EventEmitter from 'events';
import BunyanLoggly from 'bunyan-loggly';
import config from 'turtle/config';
import * as constants from 'turtle/constants/logger';
import { IJob } from 'turtle/job';
export interface ILogglyConfig {
token: string;
subdomain: string;
tags: string[];
}
interface ILogglyExtraFields {
jobID: string;
experienceName: string;
}
export default class HackyLogglyStream extends EventEmitter {
private logglyStream: BunyanLoggly;
private extraFields: ILogglyExtraFields | null;
constructor(logglyConfig: ILogglyConfig) {
super();
this.logglyStream = new BunyanLoggly(logglyConfig, config.logger.loggly.buffer);
this.extraFields = null;
}
public init(job: IJob) {
this.extraFields = {
jobID: job.id,
experienceName: job.experienceName,
};
}
public cleanup() {
this.extraFields = null;
}
public write(rec: any) {
const { msg: message, hostname: host, level, ...other } = rec;
this.logglyStream.write({
message,
level: constants.LEVELS_MAPPING[level],
host,
...other,
data: this.extraFields || {},
});
}
}
+24
View File
@@ -6,6 +6,7 @@ import { Client as RavenClient } from 'raven';
import config from 'turtle/config';
import * as constants from 'turtle/constants/logger';
import { IJob } from 'turtle/job';
import HackyLogglyStream from 'turtle/logger/hackyLogglyStream';
import S3Stream from 'turtle/logger/s3Stream';
import { isOffline } from 'turtle/turtleContext';
@@ -22,6 +23,7 @@ interface IStream {
}
export const s3logger = new S3Stream();
let logglyStream: HackyLogglyStream;
const streams: IStream[] = [];
@@ -50,6 +52,22 @@ if (config.sentry.dsn && config.deploymentEnv !== 'development') {
});
}
if (config.logger.loggly.token) {
const logglyConfig = {
token: config.logger.loggly.token,
subdomain: config.logger.loggly.subdomain,
// prettier-ignore
tags: [
'app-shell-apps',
],
};
logglyStream = new HackyLogglyStream(logglyConfig);
streams.push({
type: 'raw',
stream: logglyStream,
});
}
if (config.google.credentials) {
const resource = {
type: 'generic_node',
@@ -73,12 +91,18 @@ const logger = bunyan.createLogger({
logger.withFields = (extraFields: any) => withFields(logger, extraFields);
logger.init = async (job: IJob) => {
if (logglyStream) {
logglyStream.init(job);
}
return await s3logger.init(job);
};
logger.cleanup = async () => {
// a little hacky, but works
logger.info({ lastBuildLog: true }, 'this is the last log from this build');
if (logglyStream) {
logglyStream.cleanup();
}
await s3logger.waitForLogger();
};
+13
View File
@@ -0,0 +1,13 @@
export = BunyanLoggly;
declare class BunyanLoggly {
constructor(options: BunyanLoggly.IOptions, bufferLength?: number, bufferTimeout?: number, logglyCallback?: any);
public write(data: any): void;
}
declare namespace BunyanLoggly {
interface IOptions {
token: string;
subdomain: string;
}
}
+4 -4
View File
@@ -9,7 +9,7 @@ import { getRedisClient, RedisClient } from 'turtle/utils/redis';
const REDIS_TURTLE_VERSION_KEY = 'turtle:version';
process.on('unhandledRejection', (err) => logger.error({ err }, 'Unhandled promise rejection'));
process.on('unhandledRejection', (err) => logger.error('Unhandled promise rejection:', err));
function handleExit() {
if (checkShouldExit()) {
@@ -39,19 +39,19 @@ async function main() {
await redis.set(REDIS_TURTLE_VERSION_KEY, turtleVersion);
logger.info(`Register Turtle version ${turtleVersion} in Redis`);
} catch (err) {
logger.error({ err }, 'Failed to register Turtle version in Redis');
logger.error('Failed to register Turtle version in Redis', err);
}
while (true) {
try {
await doJob();
} catch (err) {
logger.error({ err }, 'Failed to do a job');
logger.error('Failed to do a job', err);
}
}
}
main()
.then(() => logger.error('This should never happen...'))
.catch((err) => logger.error({ err }, 'Something went terribly wrong'))
.catch((err) => logger.error('Something went terribly wrong :(', err))
.then(() => process.exit(1));
+5 -4
View File
@@ -60,23 +60,24 @@ export async function getPriorities() {
logger.debug(`Using configuration at index ${configurationsIndex} pulled from redis`);
const { error, value: configValue } = configSchema.validate(configurations[configurationsIndex]);
if (error) {
logger.warn({ err: error });
logger.warn(error);
throw new Error('Received configuration is not valid');
}
return configValue;
} catch (err) {
logger.warn({ err });
logger.warn(err);
const defaultConfig = JSON.parse(await redis.get(createDefaultConfigurationKey()));
const { error, value: defaultConfigValue } = configSchema.validate(defaultConfig);
if (error) {
logger.warn({ err: error });
logger.warn(error);
throw new Error('Received default configuration is not valid');
}
logger.warn('Using default configuration pulled from redis');
return defaultConfigValue;
}
} catch (err) {
logger.warn({ err }, 'Using configuration chosen locally');
logger.warn(err);
logger.warn('Using configuration chosen locally');
if (Math.random() < 0.7) {
return HIGH_CONFIGURATION;
} else {
+3 -3
View File
@@ -20,7 +20,7 @@ const clientToUrl: IClientConfig = {
[RedisClient.Configuration]: config.redis.configUrl,
};
export function connect(timeoutMs: number, type: string): Promise<any> {
export function connect(timeoutMs: number, type: string) {
return new Promise((resolve, reject) => {
const redisClient = new Redis(clientToUrl[type], {
maxRetriesPerRequest: 2,
@@ -52,7 +52,7 @@ export async function getRedisClient(type = RedisClient.Default) {
try {
redisClients[type] = await connect(MILLIS_CONNECTION_TIMEOUT, type);
} catch (err) {
logger.error({ err });
logger.error(err);
}
}
return redisClients[type];
@@ -66,7 +66,7 @@ export async function checkIfCancelled(jobId: string) {
if (config.deploymentEnv === 'development') {
logger.warn('Did you turn on redis server? Run `yarn start-docker` in server/www');
}
logger.error({ err });
logger.error(err);
return false;
}
}
-3
View File
@@ -1,3 +0,0 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws "$@"
-3
View File
@@ -1,3 +0,0 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws_bash_completer "$@"
-3
View File
@@ -1,3 +0,0 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws_completer "$@"
-3
View File
@@ -1,3 +0,0 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../nix" git-crypt --command git-crypt "$@"
-19
View File
@@ -1,19 +0,0 @@
#! /usr/bin/env bash
set -euo pipefail
scriptDir=$(dirname "$0")
nixDir="$scriptDir/../../nix"
nix build -f $nixDir --no-link "$@"
for package in "$@"; do
for b in $(ls $(nix path-info -f $nixDir $package)/bin); do
cat << EOF > "$scriptDir/$b"
#! /usr/bin/env bash
# This file was generated by $(basename "$0")
exec nix run -f "\$(dirname "\$0")/../../nix" $package --command $b "\$@"
EOF
chmod +x "$scriptDir/$b"
done
done
-3
View File
@@ -1,3 +0,0 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../../nix" nodejs-8_x --command node "$@"
-3
View File
@@ -1,3 +0,0 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../../nix" nodejs-8_x --command npm "$@"
-3
View File
@@ -1,3 +0,0 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../../nix" nodejs-8_x --command npx "$@"
-3
View File
@@ -1,3 +0,0 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../../nix" yarn --command yarn "$@"
-3
View File
@@ -1,3 +0,0 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../../nix" yarn --command yarnpkg "$@"
+3 -6
View File
@@ -6,20 +6,17 @@
"sourceMap": true,
"inlineSources": true,
"outDir": "./build",
"rootDir": "./src",
"strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noFallthroughCasesInSwitch": true,
"esModuleInterop": true,
"baseUrl": ".",
"baseUrl": "./",
"paths": {
"turtle/*": ["src/*"],
},
"allowJs": true,
"plugins": [
{ "transform": "ts-transformer-imports" }
]
"allowJs": true
},
"include": ["src"],
"exclude": ["node_modules", "build", "workingdir"]
}
+1 -4
View File
@@ -5,10 +5,7 @@
"rules": {
"quotemark": [true, "single", "jsx-double"],
"object-literal-sort-keys": false,
"object-curly-spacing": true,
"no-namespace": false
},
"rulesDirectory": [
"node_modules/tslint-eslint-rules/dist/rules"
]
"rulesDirectory": []
}
+1641 -565
View File
File diff suppressed because it is too large. Load diff