Compare commits

..
Author SHA1 Message Date
Nick Novitski 0da20f4ab5 Instantiate logger only at entrypoints
This ensures buildPhase, jobID, and experienceName labels are propagated when appropriate.
2019-04-11 12:47:06 -07:00
Nick Novitski 05d55bf731 Remove logger.withFields 2019-04-10 16:57:33 -07:00
Nick Novitski 074ba521f9 yarn add '@google-cloud/logging-bunyan@latest' 2019-04-10 16:53:31 -07:00
Nick Novitski 451999f0b9 Search tools ignore fixtures 2019-04-10 16:53:01 -07:00
Eric Samelson c0cf273b7c update iOS shell app for SDK 32 to fix AppAuth issue 2019-04-10 12:53:22 -07:00
Eric Samelson cbb8faef12 update CHANGELOG for 0.6.0 2019-04-09 18:38:49 -07:00
Eric Samelson d24b82b5c2 Release 0.6.0 2019-04-09 18:37:19 -07:00
Eric Samelson 3f5ddf6a22 update all old android shell tarballs to remove Branch module 2019-04-08 17:34:40 -07:00
Eric Samelson 80f30bb29c update android SDK 32 tarball to 2.10.8 (removes Branch module) 2019-04-08 12:34:05 -07:00
Eric Samelson c6e2b07fff update CHANGELOG for 0.5.15 2019-04-03 15:59:56 -07:00
Eric Samelson 80b0029d57 Release 0.5.15 2019-04-03 15:58:05 -07:00
Eric Samelson 17bda59676 update ios SDK 32 shell tarball to 2.10.6 2019-04-03 14:28:42 -07:00
Dominik Sokal 2fbcbaa525 Fix lint errors before commit (#34)
Copy-pasted from expo-cli.
2019-04-03 11:37:44 +02:00
Dominik Sokal 22e47e4e43 update CHANGELOG 2019-04-03 10:15:11 +02:00
Dominik Sokal 1d4728e402 goodbye loggly (#32) 2019-04-03 09:53:01 +02:00
Nick Novitski de4837f361 Log error objects where possible (#33)
* Log error objects where possible

* fix lint errors
2019-04-03 09:50:54 +02:00
Dominik Sokal 20d393e48e add object-curly-spacing rule 2019-04-03 09:37:32 +02:00
Dominik Sokal 95714bdf5e update CHANGELOG 2019-04-02 15:36:15 +02:00
Dominik Sokal 8e2b593e3e only alter path if not in universe 2019-04-02 13:24:11 +02:00
Dominik Sokal 885cf08a9d fix fetchRemoteTarballs.sh 2019-04-01 15:46:46 +02:00
Dominik Sokal bd816e1124 make scripts/fetchRemoteTarballs.sh executable 2019-04-01 15:10:15 +02:00
Dominik Sokal 98549f147c don't use awscli for downloading shell app tarballs 2019-04-01 14:48:33 +02:00
Dominik Sokal 9de691c60e adds running smoke tests on CircleCI (#28)
Closes #24.

Turtle smoke tests are being executed from the context of `expo/universe` repository so far. That doesn't make too much sense. I moved running those tests to this repository.
2019-04-01 13:25:30 +02:00
Dominik Sokal 71e270dc50 don't install git in docker 2019-03-28 14:59:56 +01:00
Eric Samelson 26e684069d update CHANGELOG 2019-03-22 15:58:50 -07:00
Eric Samelson 1c67df516e Release 0.5.14 2019-03-22 15:56:25 -07:00
Eric Samelson d7ed489876 update ios sdk32 tarball to 2.10.5 2019-03-22 11:49:35 -07:00
Dominik Sokal 267e487494 change queues urls 2019-03-21 10:45:03 +01:00
Dominik Sokal b07d830eb9 replace ios recording file (#29) 2019-03-20 17:14:03 +01:00
Dominik Sokal bfaaf09e57 add config param (#27)
* add config param

* update CHANGELOG
2019-03-14 10:15:24 +01:00
Dominik Sokal 33b2a569aa Release 0.5.13 2019-03-12 15:10:40 +01:00
Dominik Sokal 2febf6bcdd upgrade xdl to 53.1.6-alpha.2 2019-03-12 15:08:41 +01:00
Wojciech Kozyra e2c9ee0e39 [android-builder] check if sdk is supported (#25) 2019-03-06 15:56:31 +01:00
Dominik Sokal f017540571 don't fetch android shell apps when running tests on ci (they are already there) 2019-02-28 11:55:26 +01:00
Dominik Sokal 5b65eb3d6e add json to moduleFileExtensions (jest) 2019-02-28 11:37:53 +01:00
Dominik Sokal 60874f31c9 Release 0.5.12 2019-02-26 09:51:28 +01:00
Dominik Sokal 78beb3c0b3 fix yarn.lock 2019-02-26 09:51:01 +01:00
Dominik Sokal 03633d94e5 update CHANGELOG 2019-02-26 09:46:17 +01:00
Nick Novitski 392e4487b7 Add google cloud logger (#21) 2019-02-25 13:10:11 -08:00
sierrakn f316cb0e0c Sending manifest when building Android shell app (#22)
# Why

Please describe the motivation for this PR, and link to relevant GitHub issues, forums posts, or feature requests.

# How

How did you build this feature or fix this bug and why?

# Test Plan

Please describe how you tested this change and how a reviewer could reproduce your test, especially if this PR does not include automated tests! If possible, please also provide terminal output and/or screenshots demonstrating your test/reproduction.
2019-02-25 11:11:21 +01:00
Wojciech Kozyra 968ce7d8bb update traveling-fastlane 2019-02-22 14:56:10 +01:00
Wojciech Kozyra 5876a11568 Update CHANGELOG.md 2019-02-22 13:48:02 +01:00
Wojciech Kozyra 7448e1f4e4 Release 0.5.11 2019-02-22 13:45:28 +01:00
Dominik Sokal 4338e71005 [ios] add support for expo client builds (#19)
* [ios] add support for expo client builds

* change client shell app tarball url

* add creating an ad hoc provisioning profile

* cleanup logs

* update xdl and traveling-fastlane

* fix for breaking changes in xdl
2019-02-22 13:18:08 +01:00
Eric Samelson 7323412095 update CHANGELOG for 0.5.10 2019-02-19 14:07:32 -08:00
Eric Samelson c1d5a17e45 Release 0.5.10 2019-02-19 14:04:44 -08:00
Eric Samelson 2771ad0829 Revert "Android splash screen standalone app adjustments (#3)"
This reverts commit 31c17a4d72.
2019-02-19 11:29:00 -08:00
Eric Samelson 32cc988072 update android sdk 32 shell app tarball to 2.10.7 2019-02-19 11:28:01 -08:00
Wojciech Kozyra 0b3ce2c8e0 fix setup script 2019-02-14 10:26:11 +01:00
Wojciech Kozyra 9c46ee143e register turtle version in redis (#20)
register turtle version in redis
2019-02-13 15:53:03 +01:00
Bartłomiej Bukowski 31c17a4d72 Android splash screen standalone app adjustments (#3)
* Android splash screen standalone adjustments

* Update xdl
2019-02-06 14:49:43 +01:00
Stanisław Chmiela ee3f265294 Release 0.5.9 2019-02-06 14:02:31 +01:00
Stanisław Chmiela c1cca91e68 [changelog] Add changelog entry for 0.5.8 2019-02-06 14:01:45 +01:00
Stanisław Chmiela 038bafe20b [ios] Fix bare notifications in standalone mode on SDK32 (#18)
See https://github.com/expo/expo/pull/3431.
2019-02-06 13:58:01 +01:00
Eric Samelson f2fef264d7 update CHANGELOG for 0.5.7 2019-01-31 17:40:08 -08:00
Eric Samelson fae77b4455 Release 0.5.7 2019-01-31 17:36:57 -08:00
Eric Samelson a3ffa0411e update android sdk 32 shell app with bundled assets fix 2019-01-31 15:52:31 -08:00
Dominik Sokal 27dd680cbd update CHANGELOG 2019-01-25 13:35:12 +01:00
Dominik Sokal 2b5a370f48 Release 0.5.6 2019-01-25 13:30:00 +01:00
Dominik Sokal 21933f5a6a update shell apps for sdk 30, 31 and 32 (built with xcode 10.1) 2019-01-25 13:27:45 +01:00
Dominik Sokal a5bd4f0c97 Update CHANGELOG 2019-01-24 09:35:16 +01:00
Dominik Sokal 8e10aacf59 Release 0.5.5 2019-01-24 09:33:54 +01:00
Dominik Sokal 3dd74ef406 fix releaseBeta.sh 2019-01-23 17:00:49 +01:00
Dominik Sokal 87f7085fc4 print a reminder about updating CHANGELOG when releasing turtle-cli 2019-01-23 16:58:51 +01:00
Dominik Sokal 0b0b5a7c72 update CONTRIBUTING.md 2019-01-23 16:45:16 +01:00
Dominik Sokal d6dee79bcf update CHANGELOG 2019-01-23 16:45:09 +01:00
Dominik Sokal db05eeeed3 send build duration to www (#15) 2019-01-23 16:29:17 +01:00
Stanisław Chmiela cc317a6572 Release 0.5.4 2019-01-22 12:07:20 +01:00
Stanisław Chmiela 149073466d [ios] Update iOS shell tarball for SDK32 (#13)
Updated iOS shell tarball to https://github.com/expo/expo/commit/0847c1073d38c366866f3d9279ddd662da47dc9b
2019-01-22 11:50:53 +01:00
Dominik Sokal 776aec445b Update CHANGELOG.md 2019-01-22 09:56:41 +01:00
106 changed files with 5269 additions and 4027 deletions

No files matched your search

+195 -9
View File
@@ -1,4 +1,139 @@
version: 2
version: 2.1
executors:
nix: &nix
docker:
- image: lnl7/nix:2.1.2
# Symlink to the readline-enabled bash which is the default command of the container
shell: /run/current-system/sw/bin/bash -leo pipefail
working_directory: ~/turtle
mac:
macos: # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions
xcode: '10.1.0'
working_directory: /Users/distiller/project
shell: /bin/bash -leo pipefail
android:
<<: *nix
resource_class: xlarge
commands:
install_nix:
steps:
- run:
name: Install nix
command: curl https://nixos.org/releases/nix/nix-2.2.1/install | bash -s -- --no-daemon
- run:
name: Add nix-initializing line to .bashrc
command: echo ". ~/.nix-profile/etc/profile.d/nix.sh" >> ~/.bashrc
setup:
steps:
- run:
name: Install common packages
command: |
nix-env -iA \
nixpkgs.binutils \
nixpkgs.direnv \
nixpkgs.findutils \
nixpkgs.gawk \
nixpkgs.git \
nixpkgs.gnutar \
nixpkgs.gzip \
nixpkgs.openssh \
nixpkgs.python \
nixpkgs.xz
- run:
name: Configure direnv
command: |
mkdir -p ~/.config/direnv
echo -e "[whitelist]\nprefix = [ \"$HOME\" ]" > ~/.config/direnv/config.toml
echo 'eval "$(direnv export bash)"' >> ~/.bash_profile
- run:
name: Configure yarn
command: echo '--frozen-lockfile true' >> ~/.yarnrc
- checkout
- run:
name: Configure aws-cli
command: |
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
setup_nix:
steps:
- run:
name: Add cp and rm stubs
command: |
cat \<< 'EOF' >> /bin/cp
#! /usr/bin/env bash
/usr/bin/env cp "$@"
EOF
chmod +x /bin/cp
cat \<< 'EOF' >> /bin/rm
#! /usr/bin/env bash
/usr/bin/env rm "$@"
EOF
chmod +x /bin/rm
decrypt_secrets_if_possible:
steps:
- run:
name: Decrypt secrets if possible
command: |
if [[ -z "$TURTLE_GIT_CRYPT_KEY_BASE64" ]]; then
echo 'git-crypt key not present in environment'
else
git crypt unlock <(echo $TURTLE_GIT_CRYPT_KEY_BASE64 | base64 --decode)
fi
restore_yarn_cache:
parameters:
workingdir:
type: string
default: .
steps:
- restore_cache:
key: yarn-v1-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
save_yarn_cache:
parameters:
workingdir:
type: string
default: .
steps:
- save_cache:
key: yarn-v1-{{ checksum "<< parameters.workingdir >>/yarn.lock" }}
paths:
- ~/.cache/yarn # `yarn cache dir`
restore_workingdir_cache:
parameters:
platform:
type: string
default: ios
sdk_version:
type: string
default: sdk32
steps:
- restore_cache:
key: "workingdir-v1-<< parameters.platform >>-<< parameters.sdk_version >>"
save_workingdir_cache:
parameters:
workingdir:
type: string
default: /Users/distiller/project/workingdir
platform:
type: string
default: ios
sdk_version:
type: string
default: sdk32
steps:
- save_cache:
key: "workingdir-v1-<< parameters.platform >>-<< parameters.sdk_version >>"
paths:
- "<< parameters.workingdir >>/<< parameters.platform >>/<< parameters.sdk_version >>"
jobs:
build:
@@ -9,17 +144,68 @@ jobs:
- run: yarn
- run: yarn lint
- run: yarn build
test:
docker:
- image: circleci/node:8.12
unit_tests:
executor: nix
steps:
- checkout
- setup
- restore_yarn_cache
- run: yarn
- run: ./scripts/testUnitCI.sh
- save_yarn_cache
- run:
name: Run iOS smoke tests
command: ./scripts/tests/runUnitTestsCI.sh
ios_smoke_tests:
executor: mac
steps:
- install_nix
- setup
- decrypt_secrets_if_possible
- restore_yarn_cache
- run: yarn
- save_yarn_cache
- restore_workingdir_cache:
sdk_version: sdk32
platform: ios
- run:
name: Run iOS smoke tests
command: ./scripts/tests/runIOSSmokeTestsCI.sh
- save_workingdir_cache:
workingdir: /Users/distiller/project/workingdir
sdk_version: sdk32
platform: ios
android_smoke_tests:
executor: android
steps:
- setup
- setup_nix
- restore_yarn_cache
- run: yarn
- save_yarn_cache
- restore_workingdir_cache:
sdk_version: sdk32
platform: android
- run:
name: Fetch Android shell app
command: SDK_VERSION=sdk32 ./scripts/android/fetchRemoteAndroidTarball.sh
- run:
name: Run Android smoke tests
command: |
nix-shell nix/android-shell.nix --pure \
--run "./scripts/tests/runAndroidSmokeTestsCI.sh"
- save_workingdir_cache:
workingdir: ~/turtle/workingdir
sdk_version: sdk32
platform: android
workflows:
version: 2
build_and_test:
checks:
jobs:
- build
- test
tests:
jobs:
- unit_tests
- ios_smoke_tests
- android_smoke_tests
+3 -1
View File
@@ -15,4 +15,6 @@ export NODE_ENV=development
export EXPO_TURTLE_TOOLS_DIR="$(pwd)/tools"
PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
if [ -z "$EXPO_UNIVERSE_DIR" ]; then
PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
fi
+3 -1
View File
@@ -1,10 +1,12 @@
!/src/node_modules
node_modules
yarn-error.log
/build/
/workingdir/*
/artifacts/*
shell.apk
node_modules
.secrets*
*.swp
+1
View File
@@ -0,0 +1 @@
fixtures/
-1
View File
@@ -2,7 +2,6 @@ LOGGER_LEVEL="debug"
API_PROTOCOL="http"
API_HOSTNAME="localhost"
API_PORT="3000"
LOGGLY_TOKEN=""
SENTRY_DSN=""
ENVIRONMENT="development"
REDIS_URL="redis://localhost:6379/0"
+63 -2
View File
@@ -5,6 +5,67 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Changed
- Updated iOS shell app for SDK 32 to fix AppAuth issue
## [0.6.0] - 2019-04-09
### Changed
- Removed Branch module from Android shell apps for all SDK versions
## [0.5.15] - 2019-04-03
### Added
- Turtle smoke tests are now being executed on CircleCI from the context of this repository.
### Changed
- Updated iOS shell app for SDK 32 (fixes [this loading issue for slow connections](https://github.com/expo/expo/issues/3574)).
- Minor infra changes (doesn't affect users).
- Error logs format - error objects are now passed in an object being passed as the first argument to logger function.
### Removed
- Sending logs to loggly.
## [0.5.14] - 2019-03-22
### Added
- `--config` parameter for turtle-cli commands, which allows to specify a custom path to app.json file
### Changed
- Updated iOS shell app for SDK 32 (fixes [this notifications issue](https://github.com/expo/expo/issues/3663)).
## [0.5.13] - 2019-03-12
### Added
- Android SDK version support validation.
### Changed
- iOS Enterprise builds don't compile bitcode now.
## [0.5.12] - 2019-02-26
### Added
- Google Cloud logger (as a bunyan stream).
### Changed
- Android builder passes the manifest from a build job message if it's available.
## [0.5.11] - 2019-02-22
### Added
- Builder for iOS client apps.
## [0.5.10] - 2019-02-19
### Added
- Register Turtle version in Redis.
### Changed
- Update Android shell app for SDK 32 to fix [update](https://github.com/expo/expo/issues/3504) and [back button](https://github.com/expo/expo/issues/1786) issues.
## [0.5.9] - 2019-02-06
- Update iOS shell app for SDK 32 (fixes bare notifications [issue](https://github.com/expo/expo/issues/3223)).
## [0.5.7] - 2019-01-31
- Update Android shell app for SDK 32 to fix bundled assets issue.
## [0.5.6] - 2019-01-25
- Update iOS shell app for SDK 30, 31 and 32 (built with xcode 10.1).
## [0.5.5] - 2019-01-24
### Added
- Sending build duration in AWS SQS messages (of `FINISHED` and `ERRORED` types).
## [0.5.4] - 2019-01-22
### Changed
- Update iOS shell app for SDK32 to [`expo/expo#0847c10`](https://github.com/expo/expo/commit/0847c1073d38c366866f3d9279ddd662da47dc9b), i.e. v2.10.3 with fixed WebView navigation scheme and upgraded Facebook SDK.
## [0.5.3] - 2019-01-21
### Fixed
@@ -22,11 +83,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [0.5.1] - 2019-01-16
### Changed
- Update Android shell app for SDK32 that includes react-native-screens@1.0.0-alpha.22
- Update Android shell app for SDK32 that includes `[email protected]`.
## [0.5.0] - 2019-01-16
### Added
- Turtle version is added to onFinish and onError build messages.
- Turtle version is added to the `onFinish` and `onError` build messages.
### Changed
- A version property value being sent to the queue. It's now the version number
from `package.json` instead of `new`/`old` indicator.
+2
View File
@@ -29,3 +29,5 @@ To submit a pull request:
## Publishing a release
- To release a new version of `turtle-cli`, run `yarn release` command.
- If you wish to release a new *beta* version of `turtle-cli`, run `yarn release:beta` instead.
- Update [Changelog](CHANGELOG.md).
- In a terminal, `cd` into [expo/turtle-cli-example](https://github.com/expo/turtle-cli-example) repository, run `yarn update-turtle-cli-version` (this script updates `turtle-cli` version number in CI configuration files and commits the changes) and then run `git push`. Thanks!
-7
View File
@@ -102,13 +102,6 @@ ENV PATH ${GRADLE_HOME}/bin:$PATH
# Install Gulp
RUN npm install -g gulp-cli
# Install Git
RUN echo 'deb http://http.debian.net/debian wheezy-backports main' >> /etc/apt/sources.list && \
apt-get update && \
apt-get -t wheezy-backports install git-core && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
ADD . /app
RUN for SDK_VERSION in `ls /app/workingdir/android/`; do \
+1 -1
View File
@@ -9,6 +9,6 @@
"^.+\\.ts$": "ts-jest"
},
"testRegex": "(test|spec)\\.ts$",
"moduleFileExtensions": ["ts", "js"],
"moduleFileExtensions": ["ts", "js", "json"],
"verbose": true
}
+1 -1
View File
@@ -9,6 +9,6 @@
"^.+\\.ts$": "ts-jest"
},
"testMatch": [ "**/__smoke__/*.test.ts" ],
"moduleFileExtensions": ["ts", "js"],
"moduleFileExtensions": ["ts", "js", "json"],
"verbose": true
}
+1 -1
View File
@@ -9,6 +9,6 @@
"^.+\\.ts$": "ts-jest"
},
"testMatch": [ "**/__tests__/*.ts" ],
"moduleFileExtensions": ["ts", "js"],
"moduleFileExtensions": ["ts", "js", "json"],
"verbose": true
}
@@ -50,8 +50,6 @@ spec:
value: exp-shell-app-assets
- name: AWS_S3_REGION
value: us-west-1
- name: LOGGLY_SUBDOMAIN
value: exponent
- name: TURTLE_TEMPORARY_FILES_DIR
value: /tmp/turtle
# vars from k8s secrets
@@ -100,11 +98,6 @@ spec:
secretKeyRef:
name: turtle-js-config
key: aws_cloudwatch_region
- name: LOGGLY_TOKEN
valueFrom:
secretKeyRef:
name: turtle-js-config
key: loggly_token
- name: REDIS_URL
valueFrom:
secretKeyRef:
+5
View File
@@ -0,0 +1,5 @@
self: super:
{
nodejs = super.nodejs-8_x;
}
+50
View File
@@ -0,0 +1,50 @@
with (import ./default.nix {});
let
ndk = androidenv.androidndk_17c.override { fullNDK = true; };
ndk_root = "${ndk}/libexec/${ndk.name}";
sdk_path = if stdenv.isDarwin
then "/Library/Android/sdk"
else "/Android/Sdk"; # intentional Capital S
in
mkShell rec {
LANG="en_US.UTF-8";
JAVA_HOME=openjdk8;
ANDROID_SDK_ROOT=builtins.getEnv("HOME") + sdk_path;
ANDROID_NDK_ROOT="${ANDROID_SDK_ROOT}/ndk-bundle";
ANDROID_HOME=ANDROID_SDK_ROOT;
ANDROID_NDK=ANDROID_NDK_ROOT;
nativeBuildInputs = [
awscli
curl
git
nodejs-8_x
openjdk8
yarn
];
passthru = { inherit ndk; };
shellHook = ''
${./install-ndk-17c.sh} ${ndk} ${ndk_root}
yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --licenses --sdk_root="$ANDROID_SDK_ROOT"
yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --sdk_root="$ANDROID_SDK_ROOT" "platforms;android-28" | grep -v =
yes | ${androidenv.androidsdk_9_0}/bin/sdkmanager --sdk_root="$ANDROID_SDK_ROOT" "build-tools;28.0.3" | grep -v =
${lib.optionalString stdenv.isLinux ''
for dep in lib lib64; do
if [ -L /$dep ] || [ ! -e /$dep ]; then
echo "Creating /$dep"
ln -s ${stdenv.cc.libc}/$dep /$dep
else
echo "Using existing /$dep"
fi
done
''}
'';
}
+19
View File
@@ -0,0 +1,19 @@
let
pin = builtins.fromJSON (builtins.readFile ./nixpkgs.json);
nixpkgs = builtins.fetchTarball {
url = "https://github.com/NixOS/nixpkgs-channels/archive/${pin.rev}.tar.gz";
inherit (pin) sha256;
};
in
{ ... } @ args:
import nixpkgs (args // {
config = {
allowUnfree = true;
android_sdk.accept_license = true;
};
overlays = [ (import ./all-packages.nix) ];
})
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
scriptdir="$(dirname ${BASH_SOURCE[0]})"
shellnix="$scriptdir/android-shell.nix"
ndk=${1:-$(nix-build "$shellnix" -A ndk --no-out-link)}
ndkRoot=${2:-$(nix eval --file "$shellnix" --raw ANDROID_NDK_ROOT)}
mkdir -p "$ANDROID_NDK_ROOT" # Create full path if necessary
# Replace any existing link or directory with link
rm -r "$ANDROID_NDK_ROOT"
ln -sn "$ndkRoot" "$ANDROID_NDK_ROOT"
+7
View File
@@ -0,0 +1,7 @@
{
"url": "https://github.com/NixOS/nixpkgs-channels.git",
"rev": "80738ed9dc0ce48d7796baed5364eef8072c794d",
"date": "2018-11-17T11:18:10+01:00",
"sha256": "0anmvr6b47gbbyl9v2fn86mfkcwgpbd5lf0yf3drgm8pbv57c1dc",
"fetchSubmodules": false
}
+25 -6
View File
@@ -1,6 +1,6 @@
{
"name": "turtle-cli",
"version": "0.5.3",
"version": "0.6.0",
"description": "The command-line tool for building standalone Expo apps",
"preferGlobal": true,
"scripts": {
@@ -10,6 +10,7 @@
"watch": "EXPENV_OVERRIDES=local-common,local,local-queue expenv gulp watch:source",
"bin:turtle": "./bin/turtle.js",
"lint": "tslint -c tslint.json 'src/**/*.ts'",
"lint:fix": "yarn lint --fix",
"prepare": "yarn build",
"build": "gulp build:server",
"build:production": "./scripts/build.sh",
@@ -26,10 +27,10 @@
"test:integration": "yarn test --config jest/integration-config.json",
"test:unit": "yarn test --config jest/unit-config.json",
"test:smoke": "yarn test --config jest/smoke-config.json --forceExit",
"test:smoke:android": "yarn test:smoke --testNamePattern=android",
"test:smoke:android": "PLATFORM=android yarn test:smoke --testNamePattern=android",
"test:smoke:ios": "PLATFORM=ios yarn test:smoke --testNamePattern=iOS",
"release": "release-it",
"release:beta": "yarn release -c .release-it.beta.json --preRelease=beta",
"release": "./scripts/release.sh",
"release:beta": "./scripts/releaseBeta.sh",
"ssh:android:pods-names": "./scripts/ssh/findAndroidPods.sh",
"ssh:android:connect": "./scripts/ssh/connectToAndroidPod.sh",
"ssh:ios:ips": "./scripts/ssh/findIosIps.sh",
@@ -38,6 +39,17 @@
"set-priorities:staging": "expenv babel-node scripts/setConfigurations.js",
"set-priorities:production": "EXPENV_OVERRIDES=prod expenv babel-node scripts/setConfigurations.js"
},
"husky": {
"hooks": {
"pre-commit": "lint-staged"
}
},
"lint-staged": {
"src/**/*.ts": [
"yarn lint:fix",
"git add"
]
},
"bin": {
"turtle": "./bin/turtle.js"
},
@@ -88,7 +100,9 @@
"gulp-plumber": "^1.2.0",
"gulp-sourcemaps": "^2.6.4",
"gulp-typescript": "^4.0.2",
"husky": "^1.3.1",
"jest": "^23.6.0",
"lint-staged": "^8.1.5",
"nock": "^10.0.1",
"nodemon": "^1.18.7",
"release-it": "^10.0.0",
@@ -96,14 +110,16 @@
"source-map-loader": "^0.2.3",
"ts-jest": "^23.10.4",
"tslint": "^5.10.0",
"tslint-eslint-rules": "^5.4.0",
"typescript": "^3.0.1"
},
"dependencies": {
"@expo/spawn-async": "^1.4.0",
"@google-cloud/logging-bunyan": "^0.10.1",
"async-retry": "^1.2.1",
"aws-sdk": "^2.226.1",
"bunyan": "^1.8.12",
"bunyan-debug-stream": "^1.0.8",
"bunyan-loggly": "^1.3.4",
"bunyan-sentry-stream": "^1.2.1",
"commander": "^2.15.1",
"copy": "^0.3.1",
@@ -122,6 +138,9 @@
"tar": "^4.4.6",
"uuid": "^3.2.1",
"which": "^1.3.1",
"xdl": "52.0.14-alpha.6+371be041"
"xdl": "53.1.6-alpha.2"
},
"optionalDependencies": {
"@expo/traveling-fastlane-darwin": "^1.8.1"
}
}
+1 -21
View File
@@ -1,24 +1,4 @@
#!/usr/bin/env bash
set -xeo pipefail
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
ARTIFACTS_DIR="$ROOT_DIR/artifacts"
WORKING_DIR="$ROOT_DIR/workingdir/android"
ANDROID_SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/android"
mkdir -p $ARTIFACTS_DIR
for SDK_VERSION in `ls $ANDROID_SHELL_TARBALLS_DIRECTORY`; do
mkdir -p $WORKING_DIR/$SDK_VERSION
TARBALL_URI=$(head -n 1 $ANDROID_SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
if command -v run >/dev/null 2>&1 ; then
run expo.s3env --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --command retry5 aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
else
# outside of universe aws credentials might not be set
if ! aws iam get-user >/dev/null 2>&1 ; then aws configure ; fi
aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
fi
tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
rm -rf $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
done
$ROOT_DIR/scripts/fetchRemoteTarballs.sh android
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
set -xeo pipefail
PLATFORM=$1
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
ARTIFACTS_DIR="$ROOT_DIR/artifacts"
WORKING_DIR="$ROOT_DIR/workingdir/$PLATFORM"
SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/$PLATFORM"
mkdir -p $ARTIFACTS_DIR
convert_s3_to_https() {
BUCKET=`echo $1 | sed 's/s3:\/\/\([^/]*\).*/\1/'`
KEY=`echo $1 | sed 's/s3:\/\/[^/]*\/\(.*\)/\1/'`
echo "https://s3.amazonaws.com/$BUCKET/$KEY"
}
download_shell_app() {
SDK_VERSION=$1
mkdir -p $WORKING_DIR/$SDK_VERSION
TARBALL_S3_URL=$(head -n 1 $SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
TARBALL_HTTPS_URL=$(convert_s3_to_https $TARBALL_S3_URL)
curl -s $TARBALL_HTTPS_URL -o $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
rm -rf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
}
if [[ -z "$SDK_VERSION" ]]; then
for SDK_VERSION in `ls $SHELL_TARBALLS_DIRECTORY`; do
download_shell_app $SDK_VERSION
done
else
if [ ! -d "$WORKING_DIR/$SDK_VERSION" ]; then
download_shell_app $SDK_VERSION
fi
fi
+1 -21
View File
@@ -1,24 +1,4 @@
#!/usr/bin/env bash
set -xeo pipefail
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
ARTIFACTS_DIR="$ROOT_DIR/artifacts"
WORKING_DIR="$ROOT_DIR/workingdir/ios"
IOS_SHELL_TARBALLS_DIRECTORY="$ROOT_DIR/shellTarballs/ios"
mkdir -p $ARTIFACTS_DIR
for SDK_VERSION in `ls $IOS_SHELL_TARBALLS_DIRECTORY`; do
mkdir -p $WORKING_DIR/$SDK_VERSION
TARBALL_URI=$(head -n 1 $IOS_SHELL_TARBALLS_DIRECTORY/$SDK_VERSION)
if command -v run >/dev/null 2>&1 ; then
run expo.s3env --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --command retry5 aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
else
# outside of universe aws credentials might not be set
if ! aws iam get-user >/dev/null 2>&1 ; then aws configure ; fi
aws s3 cp $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz --no-progress
fi
tar zxf $ARTIFACTS_DIR/$SDK_VERSION.tar.gz -C $WORKING_DIR/$SDK_VERSION
rm -rf $TARBALL_URI $ARTIFACTS_DIR/$SDK_VERSION.tar.gz
done
$ROOT_DIR/scripts/fetchRemoteTarballs.sh ios
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -eo pipefail
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
$ROOT_DIR/node_modules/.bin/release-it
echo
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo "!!!! PLEASE REMEMBER TO UPDATE CHANGELOG !!!!"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -eo pipefail
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
$ROOT_DIR/node_modules/.bin/release-it -c $ROOT_DIR/.release-it.beta.json --preRelease=beta
+2 -2
View File
@@ -48,10 +48,10 @@ setup_remote() {
echo "Shell apps already fetched"
else
echo "Fetching shell apps"
if [ "$PLATFORM" != 'ios']; then
if [ "$PLATFORM" != 'ios' ]; then
$ROOT_DIR/scripts/android/fetchRemoteAndroidTarball.sh
fi
if [ "$PLATFORM" != 'android']; then
if [ "$PLATFORM" != 'android' ]; then
$ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
fi
fi
@@ -1,37 +1,25 @@
#!/usr/bin/env bash
set -ueo pipefail
export NODE_ENV=test
export ENVIRONMENT="development"
export AWS_ACCESS_KEY_ID="aws_id"
export AWS_SECRET_ACCESS_KEY="secret"
export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_ios_in.fifo"
export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_ios_in.fifo"
export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_android_in.fifo"
export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_android_in.fifo"
export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_staging_out.fifo"
export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in.fifo"
export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in_priority.fifo"
export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in.fifo"
export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in_priority.fifo"
export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_out.fifo"
export AWS_SQS_REGION="aws.west2"
export AWS_S3_BUCKET="exp-shell-app-assets"
export AWS_S3_REGION="us-west-1"
export AWS_CLOUDWATCH_REGION="us-west-1"
export API_PROTOCOL="http"
export API_HOSTNAME="localhost"
export API_PORT="3000"
export API_HOSTNAME="staging.expo.io"
export API_PORT="443"
export REDIS_URL="noop"
export REDIS_CONFIG_URL="noop"
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
$ROOT_DIR/scripts/android/fetchRemoteAndroidTarball.sh
yarn test:smoke:android
smoke_result=$?
yarn test:unit
unit_result=$?
[[ $smoke_result != 0 || $unit_result != 0 ]] && \
echo && \
echo "Tests failed." && \
exit -1
exit 0
@@ -1,13 +1,15 @@
#!/usr/bin/env bash
set -ueo pipefail
export NODE_ENV=test
export ENVIRONMENT="development"
export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_ios_in.fifo"
export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_android_in.fifo"
export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_ios_in_priority.fifo"
export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_android_in_priority.fifo"
export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_jakub_out.fifo"
export AWS_SQS_ANDROID_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in.fifo"
export AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_android_in_priority.fifo"
export AWS_SQS_IOS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in.fifo"
export AWS_SQS_IOS_PRIORITY_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_ios_in_priority.fifo"
export AWS_SQS_OUT_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/274251141632/turtle_jobs_local_dominik_out.fifo"
export AWS_SQS_REGION="aws.west2"
export AWS_S3_BUCKET="exp-shell-app-assets"
export AWS_S3_REGION="us-west-1"
@@ -18,18 +20,7 @@ export API_PORT="443"
export REDIS_URL="noop"
export REDIS_CONFIG_URL="noop"
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
$ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )"
SDK_VERSION=sdk32 $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
yarn test:smoke:ios
smoke_result=$?
yarn test:unit
unit_result=$?
[[ $smoke_result != 0 || $unit_result != 0 ]] && \
echo && \
echo "Tests failed." && \
exit -1
exit 0
@@ -1,5 +1,7 @@
#!/usr/bin/env bash
set -ueo pipefail
export NODE_ENV=test
export ENVIRONMENT="development"
@@ -21,6 +23,3 @@ export REDIS_URL="noop"
export REDIS_CONFIG_URL="noop"
yarn test:unit
unit_result=$?
exit $unit_result
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-67bd02667e29221707f8c1dd11e739eada25ef55.tar.gz
s3://exp-artifacts/android-shell-builder-c19ce0d88a153f481fd790e23bdcd6f2d5ea7fd6.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-5c339ff7bab6f2161d378cc70e1bf04d32ad98dd.tar.gz
s3://exp-artifacts/android-shell-builder-89b2d24c86737d5a489c6a18467e31b2f9ce9005.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/android-shell-builder-cb03fdc12dc74f699a6d3f22219cbcd30e01ac46.tar.gz
s3://exp-artifacts/android-shell-builder-0cab68670fc50ce1dc16b7d86ca7976870062c39.tar.gz
+1
View File
@@ -0,0 +1 @@
s3://exp-artifacts/ios-expo-client-15bb883c22173d5b01df19421580170e776041d0.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/ios-shell-builder-sdk-30-7d017a96d00fbf18ee7e5581ea47528b9cc78afd.tar.gz
s3://exp-artifacts/ios-shell-builder-sdk-30-6863276e6b9605e31228c845a047387571959e9b.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/ios-shell-builder-sdk-latest-56824ecea6ba10f5a9d8991545a1c47ee3284f5f.tar.gz
s3://exp-artifacts/ios-shell-builder-sdk-latest-e1a8bd0ab2e1eba546792c0ffc282b5b202ccc39.tar.gz
+1 -1
View File
@@ -1 +1 @@
s3://exp-artifacts/ios-shell-builder-sdk-latest-e3855783ed0618f4dcf326dc353c6e2951e05c49.tar.gz
s3://exp-artifacts/ios-shell-builder-sdk-latest-dc981c7110b7d508de8e9c352064dc3d745c9f72.tar.gz
File diff suppressed because it is too large. Load diff
Binary file not shown.
+1 -1
View File
@@ -34,7 +34,7 @@ async function performSmokeTest(recording: string) {
});
}
await jobManager.doJob();
await jobManager.doJob(logger);
if (!fixtureExists) {
await fs.writeFile(recording, JSON.stringify(nock.recorder.play()));
}
+20 -19
View File
@@ -11,7 +11,6 @@ import {
MetricRegistrationObject,
} from 'turtle/aws/cloudwatch/types';
import config from 'turtle/config';
import logger from 'turtle/logger';
interface IState {
metrics: IMetric[];
@@ -38,12 +37,12 @@ const cloudWatch = new aws.CloudWatch({
secretAccessKey: config.aws.secretAccessKey,
});
export function init() {
export function init(logger: any) {
if (config.cloudwatch.disabled) {
return;
}
state.intervalId = setInterval(_dumpMetrics, config.cloudwatch.intervalMs);
state.intervalId = setInterval(_dumpMetrics(logger), config.cloudwatch.intervalMs);
}
export function addMetricData({
@@ -111,23 +110,25 @@ function _calculateGroupHash(metricName: string, extraDimensions: any[]) {
return `${metricName}${dimString}`;
}
function _dumpMetrics() {
if (!state.metrics.length && Object.keys(state.registeredMetrics).length < 0) {
// empty metrics list, skipping
return;
}
function _dumpMetrics(logger: any) {
return () => {
if (!state.metrics.length && Object.keys(state.registeredMetrics).length < 0) {
// empty metrics list, skipping
return;
}
logger.trace(
'[cloudwatch] lenghts for state.metrics=%d, state.registeredMetrics=%d',
state.metrics.length,
Object.keys(state.registeredMetrics).length,
);
logger.trace(
'[cloudwatch] lengths for state.metrics=%d, state.registeredMetrics=%d',
state.metrics.length,
Object.keys(state.registeredMetrics).length,
);
_addEmptyMetrics(state.metrics);
const formattedMetrics = _formatMetrics(state.metrics);
_pushMetrics(formattedMetrics);
_addEmptyMetrics(state.metrics);
const formattedMetrics = _formatMetrics(state.metrics);
_pushMetrics(formattedMetrics, logger);
state.metrics = [];
state.metrics = [];
};
}
function _addEmptyMetrics(metrics: IMetric[]) {
@@ -174,12 +175,12 @@ function _reduceMetrics(metrics: IMetric[]) {
);
}
function _pushMetrics(metrics: IMetricsChunk[]) {
function _pushMetrics(metrics: IMetricsChunk[], logger: any) {
logger.trace('[cloudwatch] pushing metrics', JSON.stringify(metrics, null, '\t'));
metrics.forEach((metricsChunk) => {
cloudWatch.putMetricData(metricsChunk, (err) => {
if (err) {
logger.warn('[cloudwatch]', err);
logger.warn({ err }, '[cloudwatch]');
}
});
});
+1 -1
View File
@@ -7,7 +7,7 @@ export function sum(metrics: IMetric[]) {
const reduced = metrics.reduce((acc: IMetric | null, i: IMetric) => {
let result = acc;
if (!result) {
result = {...i};
result = { ...i };
} else {
result.Value += i.Value;
}
+7 -8
View File
@@ -5,7 +5,6 @@ import _ from 'lodash';
import { OUTPUT_QUEUE_URL, QUEUE_URL } from 'turtle/aws/utils';
import config from 'turtle/config';
import { JOB_UPDATE_TYPE } from 'turtle/constants/build';
import logger from 'turtle/logger';
import { getCurrentJobId } from 'turtle/turtleContext';
const sqs = new AWS.SQS({
@@ -16,7 +15,7 @@ const sqs = new AWS.SQS({
const VISIBILITY_TIMEOUT_SEC = 30;
export async function receiveMessage(priority: string) {
export async function receiveMessage(priority: string, logger: any) {
const params = {
MaxNumberOfMessages: 1,
QueueUrl: QUEUE_URL(priority),
@@ -36,7 +35,7 @@ export async function receiveMessage(priority: string) {
}
})
.catch((err) => {
logger.error('Receive error', err);
logger.error({ err }, 'Receive error');
throw err;
});
});
@@ -62,19 +61,19 @@ export async function changeMessageVisibility(priority: string, receiptHandle: s
}
// Every VISIBILITY_TIMEOUT_SEC / 3 seconds we are telling AWS SQS
// that we're still proccessing the message, so it does not
// that we're still processing the message, so it does not
// send the build job to another turtle agent
export function changeMessageVisibilityRecurring(priority: string, receiptHandle: string, jobId: string) {
export function changeMessageVisibilityRecurring(priority: string, receiptHandle: string, jobId: string, logger: any) {
return setInterval(() => {
if (getCurrentJobId() === jobId) {
changeMessageVisibility(priority, receiptHandle).catch((err) => {
logger.warn('Error at change msg visibility', err);
logger.warn({ err }, 'Error at change msg visibility');
});
}
}, (VISIBILITY_TIMEOUT_SEC * 1000) / 3);
}
export async function sendMessage(jobId: string, status: JOB_UPDATE_TYPE, data = {}) {
export async function sendMessage(jobId: string, status: JOB_UPDATE_TYPE, data = {}, logger: any) {
const params = {
MessageBody: JSON.stringify({ jobId, status, ...data }),
QueueUrl: OUTPUT_QUEUE_URL(),
@@ -86,6 +85,6 @@ export async function sendMessage(jobId: string, status: JOB_UPDATE_TYPE, data =
return await sqs.sendMessage(params).promise();
});
} catch (err) {
logger.error(`Error sending SQS message: ${JSON.stringify(params)}}`, err);
logger.error({ err }, `Error sending SQS message: ${JSON.stringify(params)}}`);
}
}
+4 -3
View File
@@ -2,8 +2,8 @@ import { default as buildAndroid } from 'turtle/bin/commands/build/android';
import { default as buildIOS } from 'turtle/bin/commands/build/ios';
function createCommand(builder: any) {
return (program: any) => {
return builder(program, setCommonCommandOptions);
return (logger: any) => (program: any) => {
return builder(program, setCommonCommandOptions, logger);
};
}
@@ -20,7 +20,8 @@ function setCommonCommandOptions(cmd: any) {
.option(
'--release-channel <channel-name>',
'pull from specified release channel (default: default)',
);
)
.option('-c --config <config-file>', 'specify a path to app.json');
}
export default {
+6 -5
View File
@@ -1,8 +1,9 @@
import builders from 'turtle/bin/commands/build';
import setup from 'turtle/bin/commands/setup';
export const setupIOS = setup.ios;
export const setupAndroid = setup.android;
export const buildIOS = builders.ios;
export const buildAndroid = builders.android;
export default {
setupIOS: setup.ios,
setupAndroid: setup.android,
buildIOS: builders.ios,
buildAndroid: builders.android,
};
+5 -7
View File
@@ -4,9 +4,6 @@ import { ErrorWithCommandHelp } from 'turtle/bin/commands/ErrorWithCommandHelp';
import { ErrorWithProgramHelp } from 'turtle/bin/commands/ErrorWithProgramHelp';
import setup from 'turtle/bin/setup/setup';
import { PLATFORMS } from 'turtle/constants';
import logger from 'turtle/logger';
const l = logger.withFields({ buildPhase: 'setting up environment' });
function createSetupCommand(platform: string, os?: string) {
return (program: any) => {
@@ -23,18 +20,19 @@ function createSetupCommand(platform: string, os?: string) {
};
}
async function setupAction(program: any, cmd: any, platform: string, os?: string) {
async function setupAction(program: any, cmd: any, platform: string, logger: any, os?: string) {
try {
if (os && process.platform !== os) {
throw new Error('We don\'t support running standalone app builds for this platform on your operating system');
}
const { sdkVersion } = cmd;
await setup(platform, sdkVersion);
const l = logger.child({ buildPhase: 'setting up environment' });
await setup(platform, l, sdkVersion);
l.info('it\'s all set!');
} catch (err) {
logger.error(`Failed to setup environment for ${platform} builds`);
logger.error(err.stack);
logger.error({ err }, `Failed to setup environment for ${platform} builds`);
if (err instanceof ErrorWithCommandHelp) {
cmd.help();
} else if (err instanceof ErrorWithProgramHelp) {
+6 -6
View File
@@ -4,7 +4,7 @@ import program, { Command } from 'commander';
import fs from 'fs-extra';
import { LoggerDetach, ModuleVersion } from 'xdl';
import * as commands from 'turtle/bin/commands';
import commands from 'turtle/bin/commands';
import logger from 'turtle/logger';
const { name, version } = JSON.parse(fs.readFileSync(path.join(__dirname, '../../package.json'), 'utf-8'));
@@ -17,23 +17,23 @@ Command.prototype.asyncAction = function asyncAction(asyncFn: (...asyncFnArgs: a
return this.action(async (...args) => {
try {
await checkForUpdateAsync();
} catch (e) {
logger.warn('Failed to check for turtle-cli update.');
} catch (err) {
logger.warn({ err }, 'Failed to check for turtle-cli update.');
}
return await asyncFn(...args);
});
};
export function run(programName: string) {
runAsync(programName).catch((e) => {
logger.error('Uncaught Error', e);
runAsync(programName).catch((err) => {
logger.error({ err }, 'Uncaught Error');
process.exit(1);
});
}
async function runAsync(programName: string) {
program.version(version);
Object.values(commands).forEach((command) => registerCommand(program, command));
Object.values(commands).map((command) => command(logger)).forEach((command) => registerCommand(program, command));
program.parse(process.argv);
const subCommand = process.argv[2];
+13 -13
View File
@@ -12,7 +12,6 @@ import { IToolDefinition } from 'turtle/bin/setup/utils/toolsDetector';
import { formatShellAppDirectory } from 'turtle/builders/utils/android/workingdir';
import config from 'turtle/config';
import { PLATFORMS } from 'turtle/constants';
import logger from 'turtle/logger';
const which = util.promisify(_which);
const REQUIRED_TOOLS: IToolDefinition[] = [
@@ -40,25 +39,24 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
},
},
];
const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
const l = logger.withFields(LOGGER_FIELDS);
export default async function setup(sdkVersion?: string) {
await checkSystem(REQUIRED_TOOLS);
await prepareAndroidEnv();
export default async function setup(logger: any, sdkVersion?: string) {
await checkSystem(REQUIRED_TOOLS, logger);
await prepareAndroidEnv(logger);
if (sdkVersion) {
await ensureShellAppIsPresent(
sdkVersion,
{ formatShellAppDirectory, formatShellAppTarballUriPath },
logger,
_shellAppPostDownloadAction,
);
}
}
async function prepareAndroidEnv() {
async function prepareAndroidEnv(logger: any) {
await fs.ensureDir(config.directories.androidDependenciesDir);
const sdkConfig = await ensureAndroidSDKIsPresent();
const ndkConfig = await ensureAndroidNDKIsPresent();
const sdkConfig = await ensureAndroidSDKIsPresent(logger);
const ndkConfig = await ensureAndroidNDKIsPresent(logger);
_setEnvVars({ ...sdkConfig.envVars, ...ndkConfig.envVars });
_alterPath([...sdkConfig.path, ...sdkConfig.path]);
}
@@ -67,21 +65,23 @@ function formatShellAppTarballUriPath(sdkMajorVersion: string) {
return path.join(config.directories.shellTarballsDir, PLATFORMS.ANDROID, `sdk${sdkMajorVersion}`);
}
async function _shellAppPostDownloadAction(workingdir: string) {
async function _shellAppPostDownloadAction(workingdir: string, logger: any) {
const inWorkingdir = (filename: string) => path.join(workingdir, filename);
await fs.move(inWorkingdir('package.json'), inWorkingdir('exponent-package.json'));
await fs.move(inWorkingdir('universe-package.json'), inWorkingdir('package.json'));
await _installNodeModules(workingdir);
await _installNodeModules(workingdir, logger);
await fs.move(inWorkingdir('package.json'), inWorkingdir('universe-package.json'));
await fs.move(inWorkingdir('exponent-package.json'), inWorkingdir('package.json'));
// TODO: remove following lines after upgrading android shell app
const toolsPublicDir = path.join(workingdir, 'tools-public');
await _installNodeModules(toolsPublicDir);
await _installNodeModules(toolsPublicDir, logger);
}
async function _installNodeModules(cwd: string) {
async function _installNodeModules(cwd: string, logger: any) {
const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
const l = logger.child(LOGGER_FIELDS);
l.info(`installing dependencies in ${cwd} directory...`);
const command = await _shouldUseYarnOrNpm();
await ExponentTools.spawnAsyncThrowError(command, ['install'], {
+2 -5
View File
@@ -7,20 +7,17 @@ import fs from 'fs-extra';
import { formatArtifactDownloadPath } from 'turtle/bin/setup/utils/common';
import download from 'turtle/bin/setup/utils/downloader';
import config from 'turtle/config';
import logger from 'turtle/logger';
const ANDROID_NDK_URL = `https://dl.google.com/android/repository/android-ndk-r10e-${os.platform()}-x86_64.zip`;
const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
const l = logger.withFields(LOGGER_FIELDS);
export default async function ensureAndroidNDKIsPresent() {
export default async function ensureAndroidNDKIsPresent(logger: any) {
const androidNdkDir = path.join(config.directories.androidDependenciesDir, 'ndk');
if (!(await fs.pathExists(androidNdkDir))) {
await fs.ensureDir(androidNdkDir);
const androidNdkDownloadPath = formatArtifactDownloadPath(ANDROID_NDK_URL);
try {
const l = logger.child({ buildPhase: 'setting up environment' });
l.info('Downloading Android NDK');
await fs.ensureDir(config.directories.artifactsDir);
await download(ANDROID_NDK_URL, androidNdkDownloadPath);
+2 -5
View File
@@ -8,20 +8,17 @@ import { ExponentTools } from 'xdl';
import { formatArtifactDownloadPath } from 'turtle/bin/setup/utils/common';
import download from 'turtle/bin/setup/utils/downloader';
import config from 'turtle/config';
import logger from 'turtle/logger';
const ANDROID_SDK_URL = 'https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip';
const LOGGER_FIELDS = { buildPhase: 'setting up environment' };
const l = logger.withFields(LOGGER_FIELDS);
export default async function ensureAndroidSDKIsPresent() {
export default async function ensureAndroidSDKIsPresent(logger: any) {
const androidSdkDir = path.join(config.directories.androidDependenciesDir, 'sdk');
if (!(await fs.pathExists(androidSdkDir))) {
await fs.ensureDir(androidSdkDir);
const androidSdkDownloadPath = formatArtifactDownloadPath(ANDROID_SDK_URL);
try {
const l = logger.child(LOGGER_FIELDS);
l.info('Downloading Android SDK');
await fs.ensureDir(config.directories.artifactsDir);
await download(ANDROID_SDK_URL, androidSdkDownloadPath);
+3 -3
View File
@@ -34,13 +34,13 @@ const REQUIRED_TOOLS: IToolDefinition[] = [
},
];
export default async function setup(sdkVersion?: string) {
await checkSystem(REQUIRED_TOOLS);
export default async function setup(logger: any, sdkVersion?: string) {
await checkSystem(REQUIRED_TOOLS, logger);
if (sdkVersion) {
await ensureShellAppIsPresent(sdkVersion, {
formatShellAppDirectory,
formatShellAppTarballUriPath,
});
}, logger);
}
}
+3 -3
View File
@@ -4,15 +4,15 @@ import setupAndroid from 'turtle/bin/setup/android';
import setupIos from 'turtle/bin/setup/ios';
import { PLATFORMS } from 'turtle/constants';
export default async function setup(platform: string, sdkVersion?: string) {
export default async function setup(platform: string, logger: any, sdkVersion?: string) {
if (sdkVersion && !isSemver(sdkVersion)) {
throw new Error('SDK version is not valid.');
}
if (platform === PLATFORMS.IOS) {
return await setupIos(sdkVersion);
return await setupIos(logger, sdkVersion);
} else if (platform === PLATFORMS.ANDROID) {
return await setupAndroid(sdkVersion);
return await setupAndroid(logger, sdkVersion);
} else {
throw new Error('This should never happen :(');
}
+19 -14
View File
@@ -6,51 +6,56 @@ import { ExponentTools } from 'xdl';
import download from 'turtle/bin/setup/utils/downloader';
import { ensureToolsAreInstalled, IToolDefinition } from 'turtle/bin/setup/utils/toolsDetector';
import { IShellAppDirectoryConfig } from 'turtle/builders/utils/workingdir';
import config from 'turtle/config';
import logger from 'turtle/logger';
const l = logger.withFields({ buildPhase: 'setting up environment' });
interface IShellAppFormaters {
formatShellAppDirectory: (sdkVersion: string) => string;
formatShellAppDirectory: (config: IShellAppDirectoryConfig) => string;
formatShellAppTarballUriPath: (sdkMajorVersion: string) => string;
}
type PostDownloadAction = (workingdir: string) => void;
type PostDownloadAction = (workingdir: string, logger: any) => void;
export async function checkSystem(requiredTools: IToolDefinition[]) {
await ensureToolsAreInstalled(requiredTools);
export async function checkSystem(requiredTools: IToolDefinition[], logger: any) {
await ensureToolsAreInstalled(requiredTools, logger);
}
export async function ensureShellAppIsPresent(
sdkVersion: string,
formatters: IShellAppFormaters,
logger: any,
postDownloadAction?: PostDownloadAction,
) {
const workingdir = formatters.formatShellAppDirectory(sdkVersion);
const workingdir = formatters.formatShellAppDirectory({ sdkVersion });
const l = logger.child({ buildPhase: 'setting up environment' });
if (await fs.pathExists(workingdir)) {
return;
}
l.info(`shell app for SDK ${sdkVersion} doesn't exist, downloading...`);
await _downloadShellApp(sdkVersion, workingdir, formatters);
await _downloadShellApp(sdkVersion, workingdir, formatters, l);
if (postDownloadAction) {
await postDownloadAction(workingdir);
await postDownloadAction(workingdir, logger);
}
}
async function _downloadShellApp(sdkVersion: string, targetDirectory: string, formatters: IShellAppFormaters) {
async function _downloadShellApp(
sdkVersion: string,
targetDirectory: string,
formatters: IShellAppFormaters,
logger: any,
) {
const shellAppTarballS3Uri = await _readShellAppTarballS3Uri(sdkVersion, formatters);
const tarballDownloadTargetPath = formatArtifactDownloadPath(shellAppTarballS3Uri);
await fs.ensureDir(config.directories.artifactsDir);
await download(shellAppTarballS3Uri, tarballDownloadTargetPath);
l.info('shell app has been downloaded');
logger.info('shell app has been downloaded');
await fs.ensureDir(targetDirectory);
l.info('extracting shell app (this may take a while)...');
logger.info('extracting shell app (this may take a while)...');
await tar.x({
file: tarballDownloadTargetPath,
C: targetDirectory,
});
l.info('shell app extracted');
logger.info('shell app extracted');
}
export function formatArtifactDownloadPath(uri: string) {
+4 -6
View File
@@ -1,10 +1,7 @@
import util from 'util';
import _which from 'which';
import logger from 'turtle/logger';
const which = util.promisify(_which);
const l = logger.withFields({ buildPhase: 'setting up environment' });
export interface IToolDefinition {
command: string;
@@ -12,7 +9,7 @@ export interface IToolDefinition {
testFn?: () => Promise<boolean>;
}
export async function ensureToolsAreInstalled(tools: IToolDefinition[]) {
export async function ensureToolsAreInstalled(tools: IToolDefinition[], logger: any) {
let isAnyToolMissing = false;
for (const { command, missingDescription, testFn } of tools) {
try {
@@ -24,11 +21,12 @@ export async function ensureToolsAreInstalled(tools: IToolDefinition[]) {
await which(command);
}
} catch (err) {
const l = logger.child({ buildPhase: 'setting up environment' });
isAnyToolMissing = true;
if (!testFn) {
l.error(`${command} is missing in your $PATH`);
l.error({ err }, `${command} is missing in your $PATH`);
}
l.error(missingDescription);
l.error({ err }, missingDescription);
}
}
if (isAnyToolMissing) {
+4 -5
View File
@@ -5,10 +5,10 @@ import { ErrorWithProgramHelp } from 'turtle/bin/commands/ErrorWithProgramHelp';
import setup from 'turtle/bin/setup/setup';
import * as ProjectUtils from 'turtle/bin/utils/project';
import * as UserUtils from 'turtle/bin/utils/user';
import logger from 'turtle/logger';
import { sanitizeJob } from 'turtle/validator';
export function createBuilderAction({
logger,
program,
command,
prepareCredentials,
@@ -60,9 +60,9 @@ export function createBuilderAction({
publicUrl: cmd.publicUrl,
};
const appJSON = await ProjectUtils.loadAppJSON(projectDirArg);
const appJSON = await ProjectUtils.loadAppJSON(projectDirArg, cmd.config);
const sdkVersion = _.get(appJSON, 'expo.sdkVersion');
await setup(platform, sdkVersion);
await setup(platform, logger, sdkVersion);
const credentials = await prepareCredentials(cmd);
const rawJob = {
...buildJobObject(appJSON, args, credentials),
@@ -72,8 +72,7 @@ export function createBuilderAction({
const job = await sanitizeJob(rawJob);
await builder(job);
} catch (err) {
logger.error(`Failed to build standalone app: ${err.message}`);
logger.error(err.stack);
logger.error({ err }, 'Failed to build standalone app');
if (err instanceof ErrorWithCommandHelp) {
command.help();
} else if (err instanceof ErrorWithProgramHelp) {
+9 -4
View File
@@ -1,12 +1,17 @@
import fs from 'fs-extra';
import path from 'path';
export async function loadAppJSON(projectDirArg: string) {
const projectDir = resolveAbsoluteDir(projectDirArg);
const appJSONPath = path.join(projectDir, 'app.json');
export async function loadAppJSON(projectDirArg: string, config: string | null) {
let appJSONPath;
if (config) {
appJSONPath = resolveAbsoluteDir(config);
} else {
const projectDir = resolveAbsoluteDir(projectDirArg);
appJSONPath = path.join(projectDir, 'app.json');
}
const appJSONExists = await fs.pathExists(appJSONPath);
if (!appJSONExists) {
throw new Error(`Couldn't find app.json in ${projectDir} directory.`);
throw new Error(`Couldn't find app.json.`);
} else {
return require(appJSONPath);
}
+17
View File
@@ -0,0 +1,17 @@
class BuildError extends Error {
public reason: BuildErrorReason;
constructor(reason: BuildErrorReason, message: string) {
super(message);
this.name = 'BuildError';
this.reason = reason;
}
}
enum BuildErrorReason {
SESSION_EXPIRED = 'session-expired',
PROVISIONING_PROFILE_MISSING = 'provisioning-profile-missing',
}
export default BuildError;
export { BuildErrorReason };
+14 -10
View File
@@ -10,18 +10,20 @@ import { formatShellAppDirectory } from 'turtle/builders/utils/android/workingdi
import * as commonUtils from 'turtle/builders/utils/common';
import * as imageHelpers from 'turtle/builders/utils/image';
import { uploadBuildToS3 } from 'turtle/builders/utils/uploader';
import { ensureCanBuildSdkVersion } from 'turtle/builders/utils/version';
import config from 'turtle/config';
import { IAndroidCredentials, IJob, IJobResult } from 'turtle/job';
import logger from 'turtle/logger';
export default async function buildAndroid(jobData: IJob): Promise<IJobResult> {
const credentials = await getOrCreateCredentials(jobData);
const apkFilePath = await runShellAppBuilder(jobData, credentials);
export default async function buildAndroid(jobData: IJob, logger: any): Promise<IJobResult> {
await ensureCanBuildSdkVersion(jobData);
const credentials = await getOrCreateCredentials(jobData, logger);
const apkFilePath = await runShellAppBuilder(jobData, credentials, logger);
const randomHex = uuidv4().replace(/-/g, '');
const s3Filename = `${jobData.experienceName}-${randomHex}-signed.apk`;
const s3FileKey = `android/${s3Filename}`;
const fakeUploadFilename = s3Filename.replace('/', '\\');
const artifactUrl = await uploadBuildToS3({
logger,
uploadPath: apkFilePath,
s3FileKey,
...config.builder.fakeUpload && {
@@ -38,6 +40,7 @@ export default async function buildAndroid(jobData: IJob): Promise<IJobResult> {
async function runShellAppBuilder(
jobData: IJob,
credentials: IAndroidCredentials,
logger: any,
): Promise<string> {
const { temporaryFilesRoot } = config.directories;
await fs.ensureDir(temporaryFilesRoot);
@@ -60,25 +63,26 @@ async function runShellAppBuilder(
const outputFilePath = path.join(temporaryFilesRoot, `shell-signed-${jobData.id}.apk`);
const { manifest: { sdkVersion: sdkVersionFromManifest = null } = {}, sdkVersion: sdkVersionFromJob } = jobData;
const sdkVersion = sdkVersionFromJob || sdkVersionFromManifest;
const workingDir = formatShellAppDirectory(sdkVersion);
const { config: jobConfig, manifest, sdkVersion: sdkVersionFromJob } = jobData;
const sdkVersion = _.get(manifest, 'sdkVersion', sdkVersionFromJob);
const workingDir = formatShellAppDirectory({ sdkVersion });
try {
await AndroidShellApp.createAndroidShellAppAsync({
url: commonUtils.getExperienceUrl(jobData),
sdkVersion: _.get(jobData, 'manifest.sdkVersion') || jobData.sdkVersion,
sdkVersion,
keystore: tempKeystorePath,
manifest,
alias: credentials.keystoreAlias,
keystorePassword: credentials.keystorePassword,
keyPassword: credentials.keyPassword,
privateConfigFile: tempShellAppConfigPath,
releaseChannel: jobData.config.releaseChannel,
releaseChannel: jobConfig.releaseChannel,
workingDir,
outputFile: outputFilePath,
});
} catch (err) {
commonUtils.logErrorOnce(err);
commonUtils.logErrorOnce(err, logger);
throw err;
} finally {
if (!config.builder.skipCleanup) {
+4 -1
View File
@@ -1,7 +1,10 @@
import { IJob, IJobResult } from 'turtle/job';
import buildAndroid from './android';
import buildIOS from './ios';
export default {
const builders: { [index: string]: (job: IJob, logger: any) => Promise<IJobResult>} = {
ios: buildIOS,
android: buildAndroid,
};
export default builders;
+7 -9
View File
@@ -7,7 +7,6 @@ import * as fileUtils from 'turtle/builders/utils/file';
import * as keychain from 'turtle/builders/utils/ios/keychain';
import runShellAppBuilder from 'turtle/builders/utils/ios/shellAppBuilder';
import { IJob } from 'turtle/job';
import logger from 'turtle/logger/index';
export default async function buildArchive(ctx: IContext, job: IJob) {
let keychainInfo;
@@ -19,7 +18,7 @@ export default async function buildArchive(ctx: IContext, job: IJob) {
const manifest = await runShellAppBuilder(ctx, job);
await buildAndSignIPA(ctx, job, keychainInfo.path, manifest);
} catch (err) {
logErrorOnce(err);
logErrorOnce(err, ctx.logger);
throw err;
} finally {
if (keychainInfo) {
@@ -29,11 +28,11 @@ export default async function buildArchive(ctx: IContext, job: IJob) {
}
async function buildAndSignIPA(ctx: IContext, job: IJob, keychainPath: string, manifest: any) {
const l = logger.withFields({ buildPhase: 'building and signing IPA' });
l.info('building and signing IPA');
ctx.logger = ctx.logger.child({ buildPhase: 'building and signing IPA' });
ctx.logger.info('building and signing IPA');
const {
credentials: { provisioningProfile, certPassword, teamId, password },
credentials: { provisioningProfile, certPassword, teamId },
config: { bundleIdentifier: bundleIdentifierFromConfig },
} = job;
@@ -42,7 +41,7 @@ async function buildAndSignIPA(ctx: IContext, job: IJob, keychainPath: string, m
const { provisioningProfilePath } = ctx;
await fileUtils.writeBase64ToBinaryFile(provisioningProfilePath, provisioningProfile as string);
l.info('saved provisioning profile to temporary path');
ctx.logger.info('saved provisioning profile to temporary path');
const ipaBuilder = createIPABuilder({
keychainPath,
@@ -51,12 +50,11 @@ async function buildAndSignIPA(ctx: IContext, job: IJob, keychainPath: string, m
certPath: ctx.tempCertPath,
certPassword,
teamID: teamId,
password,
bundleIdentifier,
workingDir: ctx.workingDir,
workspacePath: ctx.workspacePath,
manifest,
});
await ipaBuilder.build();
l.info(`done building and signing IPA`);
ctx.logger.info(`done building and signing IPA`);
}
+56 -13
View File
@@ -15,6 +15,7 @@ export interface IContext {
baseArchiveDir: string;
buildDir: string;
fakeUploadBuildPath?: string;
logger: any;
outputPath: string;
provisioningProfilePath: string;
provisioningProfileDir: string;
@@ -22,12 +23,14 @@ export interface IContext {
tempCertPath: string;
uploadPath: string;
workingDir: string;
workspacePath: string;
applicationFilesSrc: string;
}
const { DEFAULT_EXPOKIT_WORKSPACE_NAME } = IosShellApp;
const { EXPOKIT_APP, EXPONENT_APP } = IosShellApp;
const { BUILD_TYPES } = IOS;
export function createBuilderContext(job: IJob): IContext {
export function createBuilderContext(job: IJob, logger: any): IContext {
const { join } = path;
const appUUID = uuidv4();
const {
@@ -39,12 +42,13 @@ export function createBuilderContext(job: IJob): IContext {
} = job;
const sdkVersion = sdkVersionFromJob || sdkVersionFromManifest;
const workingDir = formatShellAppDirectory(sdkVersion);
const workingDir = formatShellAppDirectory({ sdkVersion, buildType: buildType! });
const context: any = {
appDir: join(config.directories.temporaryFilesRoot, appUUID),
appUUID,
workingDir,
logger,
};
context.buildDir = join(context.appDir, 'build');
context.provisioningProfileDir = join(context.appDir, 'provisioning');
@@ -55,24 +59,63 @@ export function createBuilderContext(job: IJob): IContext {
context.tempCertPath = join(context.appDir, 'cert.p12');
context.baseArchiveDir = join(context.appDir, 'archive');
if (buildType === BUILD_TYPES.SIMULATOR) {
context.outputPath = join(context.appDir, 'archive.tar.gz');
context.uploadPath = join(context.appDir, 'archive.tar.gz');
context.archiveDir = join(
context.baseArchiveDir,
'Release',
`${DEFAULT_EXPOKIT_WORKSPACE_NAME}.app`,
if (buildType === BUILD_TYPES.CLIENT) {
context.applicationFilesSrc = join(
workingDir,
'expo-client-build',
'**',
'*',
);
} else if (buildType === BUILD_TYPES.ARCHIVE) {
} else {
context.applicationFilesSrc = join(
workingDir,
'shellAppBase-builds',
buildType as string,
'**',
'*',
);
}
if (buildType === BUILD_TYPES.ARCHIVE) {
context.outputPath = join(context.appDir, 'archive.xcarchive');
context.uploadPath = join(context.buildDir, 'archive.ipa');
context.archiveDir = join(
context.baseArchiveDir,
'Release',
`${DEFAULT_EXPOKIT_WORKSPACE_NAME}.xcarchive`,
`${EXPOKIT_APP}.xcarchive`,
'Products',
'Applications',
`${DEFAULT_EXPOKIT_WORKSPACE_NAME}.app`,
`${EXPOKIT_APP}.app`,
);
context.workspacePath = path.join(
workingDir,
'shellAppWorkspaces',
'ios',
'default',
`${EXPOKIT_APP}.xcworkspace`,
);
} else if (buildType === BUILD_TYPES.CLIENT) {
context.outputPath = join(context.appDir, 'archive.xcarchive');
context.uploadPath = join(context.buildDir, 'archive.ipa');
context.archiveDir = join(
context.baseArchiveDir,
`${EXPONENT_APP}.xcarchive`,
'Products',
'Applications',
`${EXPONENT_APP}.app`,
);
context.workspacePath = path.join(
workingDir,
'ios',
`${EXPONENT_APP}.xcworkspace`,
);
} else if (buildType === BUILD_TYPES.SIMULATOR) {
context.outputPath = join(context.appDir, 'archive.tar.gz');
context.uploadPath = join(context.appDir, 'archive.tar.gz');
context.archiveDir = join(
context.baseArchiveDir,
'Release',
`${EXPOKIT_APP}.app`,
);
}
+17 -35
View File
@@ -1,35 +1,39 @@
import path from 'path';
import fs from 'fs-extra';
import pick from 'lodash/pick';
import { ExponentTools } from 'xdl';
import buildArchive from 'turtle/builders/ios/archive';
import { createBuilderContext, IContext } from 'turtle/builders/ios/context';
import buildSimulator from 'turtle/builders/ios/simulator';
import { logErrorOnce } from 'turtle/builders/utils/common';
import prepareAdHocBuildCredentials from 'turtle/builders/utils/ios/adhocBuild';
import { uploadBuildToS3 } from 'turtle/builders/utils/uploader';
import { ensureCanBuildSdkVersion } from 'turtle/builders/utils/version';
import config from 'turtle/config';
import { IOS, PLATFORMS } from 'turtle/constants/index';
import { IOS } from 'turtle/constants/index';
import { IJob, IJobResult } from 'turtle/job';
const { BUILD_TYPES } = IOS;
let SUPPORTED_SDK_VERSIONS: number[];
export default async function iosBuilder(job: IJob, logger: any): Promise<IJobResult> {
if (job.config.buildType !== BUILD_TYPES.CLIENT) {
await ensureCanBuildSdkVersion(job);
}
export default async function iosBuilder(job: IJob): Promise<IJobResult> {
await ensureCanBuildSdkVersion(job);
const ctx = createBuilderContext(job);
const ctx = createBuilderContext(job, logger);
try {
await initBuilder(ctx);
const { buildType } = job.config;
if (buildType === BUILD_TYPES.SIMULATOR) {
await buildSimulator(ctx, job);
} else if (buildType === BUILD_TYPES.ARCHIVE) {
if (buildType === BUILD_TYPES.CLIENT) {
await prepareAdHocBuildCredentials(job, ctx.logger);
}
if ([BUILD_TYPES.ARCHIVE, BUILD_TYPES.CLIENT].includes(buildType!)) {
await buildArchive(ctx, job);
} else if (buildType === BUILD_TYPES.SIMULATOR) {
await buildSimulator(ctx, job);
} else {
throw new Error(`Unsupported iOS build type: ${buildType}`);
}
@@ -37,7 +41,7 @@ export default async function iosBuilder(job: IJob): Promise<IJobResult> {
const artifactUrl = await uploadBuildToS3(ctx);
return { artifactUrl };
} catch (err) {
logErrorOnce(err);
logErrorOnce(err, ctx.logger);
throw err;
} finally {
await cleanup(ctx);
@@ -60,25 +64,3 @@ async function cleanup(ctx: IContext) {
const getTemporaryDirs = (ctx: IContext) =>
Object.values(pick(ctx, ['appDir', 'provisioningProfileDir']));
async function ensureCanBuildSdkVersion(job: IJob) {
if (config.builder.useLocalWorkingDir) {
return;
}
if (!SUPPORTED_SDK_VERSIONS) {
SUPPORTED_SDK_VERSIONS = await findSupportedSdkVersions();
}
const targetMajorSdkVersion = ExponentTools.parseSdkMajorVersion(job.sdkVersion || job.manifest.sdkVersion);
if (!SUPPORTED_SDK_VERSIONS.includes(targetMajorSdkVersion)) {
throw new Error(`Unsupported SDK Version!`);
}
}
async function findSupportedSdkVersions(): Promise<number[]> {
const SDK_DIR_PREFIX = 'sdk';
const files = await fs.readdir(path.join(config.directories.workingDir, PLATFORMS.IOS));
const sdks = files.filter((file) => file.startsWith(SDK_DIR_PREFIX));
return sdks.map((sdk) => parseInt(sdk.substr(SDK_DIR_PREFIX.length), 10));
}
+3 -4
View File
@@ -1,11 +1,10 @@
import { IContext } from 'turtle/builders/ios/context';
import runShellAppBuilder from 'turtle/builders/utils/ios/shellAppBuilder';
import { IJob } from 'turtle/job';
import logger from 'turtle/logger';
export default async function buildSimulator(ctx: IContext, job: IJob) {
const l = logger.withFields({ buildPhase: 'running simulator builder' });
l.info('running simulator build');
ctx.logger = ctx.logger.child({ buildPhase: 'running simulator builder' });
ctx.logger.info('running simulator build');
await runShellAppBuilder(ctx, job);
l.info('build complete');
ctx.logger.info('build complete');
}
+12 -11
View File
@@ -2,17 +2,16 @@ import path from 'path';
import fs from 'fs-extra';
import uuidv4 from 'uuid/v4';
import { AndroidKeystore } from 'xdl';
import { Credentials } from 'xdl';
import * as sqs from 'turtle/aws/sqs';
import { UPDATE_CREDENTIALS } from 'turtle/constants/build';
import { IAndroidCredentials, IJob } from 'turtle/job';
import logger from 'turtle/logger';
import { isOffline } from 'turtle/turtleContext';
async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredentials> {
async function getOrCreateCredentials(jobData: IJob, logger: any): Promise<IAndroidCredentials> {
if (!jobData.credentials) {
const l = logger.withFields({ buildPhase: 'generating keystore' });
const l = logger.child({ buildPhase: 'generating keystore' });
const credentials: any = {};
l.info('Creating keystore');
const keystoreFilename = `/tmp/keystore-${jobData.id}.tmp.jks`;
@@ -21,13 +20,15 @@ async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredential
credentials.keystoreAlias = Buffer.from(jobData.experienceName).toString('base64');
const androidPackage = jobData.manifest.android.package;
await AndroidKeystore.createKeystore({
keystorePassword: credentials.keystorePassword,
keyPassword: credentials.keyPassword,
keystoreFilename,
keystoreAlias: credentials.keystoreAlias,
await Credentials.Android.createKeystore(
{
keystorePath: keystoreFilename,
keystorePassword: credentials.keystorePassword,
keyPassword: credentials.keyPassword,
keyAlias: credentials.keystoreAlias,
},
androidPackage,
});
);
credentials.keystore = (await fs.readFile(keystoreFilename)).toString('base64');
l.info('Keystore created successfully');
@@ -42,7 +43,7 @@ async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredential
} else {
await sqs.sendMessage(jobData.id, UPDATE_CREDENTIALS, {
...credentials,
});
}, l);
l.info('Keystore sent to storage successfully');
}
await fs.unlink(keystoreFilename);
+2 -1
View File
@@ -2,10 +2,11 @@ import path from 'path';
import { ExponentTools } from 'xdl';
import { IShellAppDirectoryConfig } from 'turtle/builders/utils/workingdir';
import config from 'turtle/config';
import { PLATFORMS } from 'turtle/constants';
export function formatShellAppDirectory(sdkVersion: string) {
export function formatShellAppDirectory({ sdkVersion }: IShellAppDirectoryConfig) {
const majorSdkVersion = ExponentTools.parseSdkMajorVersion(sdkVersion);
return config.builder.useLocalWorkingDir
? path.join(config.directories.workingDir, 'local')
+1 -2
View File
@@ -2,7 +2,6 @@ import * as url from 'url';
import config from 'turtle/config';
import { IJob } from 'turtle/job';
import logger from 'turtle/logger';
export function getExperienceUrl(job: IJob) {
const { experienceName, config: jobConfig } = job;
@@ -18,7 +17,7 @@ export function getExperienceUrl(job: IJob) {
const alreadyLoggedError = Symbol('alreadyLoggedError');
export function logErrorOnce(err: any) {
export function logErrorOnce(err: any, logger: any) {
if (!err[alreadyLoggedError]) {
logger.error(err.stack);
err[alreadyLoggedError] = true;
+4 -4
View File
@@ -18,15 +18,15 @@ export async function resizeIconWithSharpAsync(iconSizePx: number, iconFilename:
await fs.writeFile(filename, buffer);
}
export async function getImageDimensionsWithSharpAsync(basename: string, dirname: string) {
const filename = path.join(dirname, basename);
export async function getImageDimensionsWithSharpAsync(dirname: string, filename: string) {
const filepath = path.join(dirname, filename);
// PLEASE DON'T REMOVE THE FOLLOWING LINE (sharp caches files taking path as the cache key)
sharp.cache(false);
try {
const meta = await sharp(filename).metadata();
return [meta.width, meta.height];
const { width, height } = await sharp(filepath).metadata();
return { width, height };
} catch (e) {
return null;
}
+98
View File
@@ -0,0 +1,98 @@
import path from 'path';
import spawnAsync from '@expo/spawn-async';
import fs from 'fs-extra';
import isEmpty from 'lodash/isEmpty';
import { IosCodeSigning } from 'xdl';
import * as sqs from 'turtle/aws/sqs';
import BuildError, { BuildErrorReason } from 'turtle/builders/BuildError';
import { UPDATE_CREDENTIALS } from 'turtle/constants/build';
import { IJob } from 'turtle/job';
import { isOffline } from 'turtle/turtleContext';
// tslint:disable-next-line:no-var-requires
const travelingFastlane = process.platform === 'darwin' ? require('@expo/traveling-fastlane-darwin')() : null;
async function prepareAdHocBuildCredentials(job: IJob, logger: any) {
if (process.platform !== 'darwin') {
throw new Error('This function should be called only on macOS!');
}
const { bundleIdentifier } = job.config;
const { certP12, certPassword, teamId, appleSession, udids } = job.credentials;
const certSerialNumber = IosCodeSigning.findP12CertSerialNumber(certP12, certPassword);
const args = [
teamId,
udids!.join(','),
bundleIdentifier,
certSerialNumber || '__last__',
];
try {
const credentials = await runFastlaneAction(
travelingFastlane.manageAdHocProvisioningProfile,
args,
{ env: { FASTLANE_SESSION: appleSession } },
);
if (isEmpty(credentials)) {
if (!job.credentials.provisioningProfile) {
// a user has generated the provisioning profile for his app, but he probably deleted it from Expo servers, ...
throw new BuildError(
BuildErrorReason.PROVISIONING_PROFILE_MISSING,
`Provisioning profile is missing on Expo servers.`,
);
}
return;
}
logger.info('New ad hoc provisioning profile successfully created');
job.credentials.provisioningProfile = credentials.provisioningProfile;
if (isOffline()) {
const provisioningProfilePath = path.join(job.projectDir, `adhoc.mobileprovision`);
await fs.writeFile(provisioningProfilePath, Buffer.from(credentials.provisioningProfile, 'base64'));
logger.info(`Saved provisioning profile to ${provisioningProfilePath}`);
} else {
await sqs.sendMessage(job.id, UPDATE_CREDENTIALS, {
provisioningProfileId: credentials.provisioningProfileId,
provisioningProfile: credentials.provisioningProfile,
}, logger);
logger.info('Ad Hoc provisioning profile sent to storage successfully');
}
} catch (e) {
if (e instanceof BuildError) {
logger.error('Apple Session expired! Please authenticate again using expo-cli.');
}
throw e;
}
}
async function runFastlaneAction(action: string, args: string[], { env }: any) {
const { stderr } = await spawnAsync(
action,
args,
{
stdio: ['inherit', 'pipe', 'pipe'],
env: { ...process.env, ...env },
},
);
const { result, ...rest } = JSON.parse(stderr.trim());
if (result === 'failure') {
const { reason, rawDump, type } = rest;
let errorMsg = `Reason: ${reason}`;
if (rawDump) {
errorMsg += `, raw: ${JSON.stringify(rawDump)}`;
}
if (type === 'session-expired') {
throw new BuildError(BuildErrorReason.SESSION_EXPIRED, errorMsg);
} else {
throw new Error(errorMsg);
}
} else {
return rest;
}
}
export default prepareAdHocBuildCredentials;
+6 -7
View File
@@ -2,27 +2,26 @@ import { IosKeychain } from 'xdl';
import { IContext } from 'turtle/builders/ios/context';
import * as fileUtils from 'turtle/builders/utils/file';
import logger from 'turtle/logger';
interface IKeychain {
path: string;
}
export async function create(ctx: IContext): Promise<IKeychain> {
const l = logger.withFields({ buildPhase: 'creating keychain' });
const l = ctx.logger.child({ buildPhase: 'creating keychain' });
try {
l.info('creating keychain...');
const keychainInfo = await IosKeychain.createKeychain(ctx.appUUID, false);
l.info('done creating keychain');
return keychainInfo;
} catch (err) {
l.error('unable to create keychain');
l.error({ err }, 'unable to create keychain');
throw err;
}
}
export async function remove(ctx: IContext, keychainPath: string) {
const l = logger.withFields({ buildPhase: 'deleting keychain' });
const l = ctx.logger.child({ buildPhase: 'deleting keychain' });
try {
l.info('delete keychain...');
const keychainInfo = await IosKeychain.deleteKeychain({
@@ -32,7 +31,7 @@ export async function remove(ctx: IContext, keychainPath: string) {
l.info('done deleting keychain');
return keychainInfo;
} catch (err) {
l.error('unable to delete keychain');
l.error({ err }, 'unable to delete keychain');
throw err;
}
}
@@ -49,7 +48,7 @@ export async function importCert(
certPassword,
}: { keychainPath: string; certP12: string; certPassword: string },
) {
const l = logger.withFields({ buildPhase: 'importing certificate into keychain' });
const l = ctx.logger.child({ buildPhase: 'importing certificate into keychain' });
try {
l.info('importing distribution certificate into keychain...');
const { tempCertPath: certPath } = ctx;
@@ -57,7 +56,7 @@ export async function importCert(
await IosKeychain.importIntoKeychain({ keychainPath, certPath, certPassword });
l.info('done importing distribution certificate into keychain');
} catch (err) {
l.error('unable to import distribution certificate into keychain');
l.error({ err }, 'unable to import distribution certificate into keychain');
throw err;
}
}
+19 -18
View File
@@ -8,9 +8,8 @@ import { ImageUtils, IosShellApp } from 'xdl';
import { IContext } from 'turtle/builders/ios/context';
import * as commonUtils from 'turtle/builders/utils/common';
import * as imageHelpers from 'turtle/builders/utils/image';
import { PLATFORMS } from 'turtle/constants';
import { IOS } from 'turtle/constants/index';
import { IJob } from 'turtle/job';
import logger from 'turtle/logger/index';
const copyAsync = util.promisify(copy);
@@ -18,35 +17,37 @@ export default async function runShellAppBuilder(ctx: IContext, job: IJob): Prom
const { config: jobConfig, manifest, sdkVersion: sdkVersionFromJob } = job;
const { buildType, releaseChannel } = jobConfig;
const sdkVersion = _.get(manifest, 'sdkVersion', sdkVersionFromJob);
const applicationFilesSrc = path.join(
ctx.workingDir,
'shellAppBase-builds',
buildType as string,
'**',
'*',
);
await copyAsync(applicationFilesSrc, ctx.baseArchiveDir);
logger.info(
await copyAsync(ctx.applicationFilesSrc, ctx.baseArchiveDir);
ctx.logger.info(
{ buildPhase: 'icons setup' },
'ImageUtils: setting image functions to alternative sharp implementations',
);
ImageUtils.setResizeImageFunction(imageHelpers.resizeIconWithSharpAsync);
ImageUtils.setGetImageDimensionsFunction(imageHelpers.getImageDimensionsWithSharpAsync);
const shellAppParams = {
url: commonUtils.getExperienceUrl(job),
sdkVersion,
action: 'configure',
type: buildType,
releaseChannel,
archivePath: ctx.archiveDir,
privateConfigData: job.config,
verbose: true,
output: ctx.outputPath,
expoSourcePath: path.join(ctx.workingDir, PLATFORMS.IOS),
expoSourcePath: path.join(ctx.workingDir, 'ios'),
manifest,
output: ctx.outputPath,
verbose: true,
};
if (buildType === IOS.BUILD_TYPES.CLIENT) {
Object.assign(shellAppParams, {
workspacePath: path.join(ctx.workingDir, 'ios'),
});
} else {
Object.assign(shellAppParams, {
url: commonUtils.getExperienceUrl(job),
releaseChannel,
sdkVersion,
});
}
logger.info({ buildPhase: 'configuring NSBundle' }, 'configuring NSBundle...');
ctx.logger.info({ buildPhase: 'configuring NSBundle' }, 'configuring NSBundle...');
return await IosShellApp.configureAndCopyArchiveAsync(shellAppParams);
}
+12 -6
View File
@@ -2,12 +2,18 @@ import path from 'path';
import { ExponentTools } from 'xdl';
import { IShellAppDirectoryConfig } from 'turtle/builders/utils/workingdir';
import config from 'turtle/config';
import { PLATFORMS } from 'turtle/constants';
import { IOS, PLATFORMS } from 'turtle/constants';
export function formatShellAppDirectory(sdkVersion: string) {
const majorVersion = ExponentTools.parseSdkMajorVersion(sdkVersion);
return config.builder.useLocalWorkingDir
? path.join(config.directories.workingDir, 'local')
: path.join(config.directories.workingDir, PLATFORMS.IOS, `sdk${majorVersion}`);
export function formatShellAppDirectory({ sdkVersion, buildType }: IShellAppDirectoryConfig) {
const { workingDir } = config.directories;
if (buildType === IOS.BUILD_TYPES.CLIENT) {
return path.join(workingDir, PLATFORMS.IOS, 'client');
} else if (config.builder.useLocalWorkingDir) {
return path.join(workingDir, 'local');
} else {
const majorVersion = ExponentTools.parseSdkMajorVersion(sdkVersion);
return path.join(workingDir, PLATFORMS.IOS, `sdk${majorVersion}`);
}
}
+3 -3
View File
@@ -2,24 +2,24 @@ import * as fs from 'fs-extra';
import { uploadFile } from 'turtle/aws/s3';
import config from 'turtle/config';
import logger from 'turtle/logger';
interface IUploadCtx {
fakeUploadBuildPath?: string;
s3FileKey?: string;
uploadPath: string;
logger: any;
}
export async function uploadBuildToS3(ctx: IUploadCtx) {
if (config.builder.fakeUpload) {
const l = logger.withFields({ buildPhase: 'copying build artifact' });
const l = ctx.logger.child({ buildPhase: 'copying build artifact' });
const { fakeUploadBuildPath, uploadPath } = ctx;
l.info('copying build to fake upload directory');
await fs.copy(uploadPath, fakeUploadBuildPath as string);
l.info(`copied build to ${fakeUploadBuildPath}`);
return fakeUploadBuildPath as string;
} else {
const l = logger.withFields({ buildPhase: 'uploading to S3' });
const l = ctx.logger.child({ buildPhase: 'uploading to S3' });
l.info('uploading build artifact to S3');
const { Location: fileLocation } = await uploadFile({
key: ctx.s3FileKey as string,
+30
View File
@@ -0,0 +1,30 @@
import fs from 'fs-extra';
import path from 'path';
import { ExponentTools } from 'xdl';
import config from 'turtle/config';
import { IJob } from 'turtle/job';
let SUPPORTED_SDK_VERSIONS: number[];
export async function ensureCanBuildSdkVersion(job: IJob) {
if (config.builder.useLocalWorkingDir) {
return;
}
if (!SUPPORTED_SDK_VERSIONS) {
SUPPORTED_SDK_VERSIONS = await findSupportedSdkVersions(job.platform);
}
const targetMajorSdkVersion = ExponentTools.parseSdkMajorVersion(job.sdkVersion || job.manifest.sdkVersion);
if (!SUPPORTED_SDK_VERSIONS.includes(targetMajorSdkVersion)) {
throw new Error(`Unsupported SDK Version!`);
}
}
async function findSupportedSdkVersions(platform: 'android' | 'ios'): Promise<number[]> {
const SDK_DIR_PREFIX = 'sdk';
const files = await fs.readdir(path.join(config.directories.workingDir, platform));
const sdks = files.filter((file) => file.startsWith(SDK_DIR_PREFIX));
return sdks.map((sdk) => parseInt(sdk.substr(SDK_DIR_PREFIX.length), 10));
}
+6
View File
@@ -0,0 +1,6 @@
import { IOS } from 'turtle/constants';
export interface IShellAppDirectoryConfig {
sdkVersion: string;
buildType?: IOS.BUILD_TYPES;
}
+4 -6
View File
@@ -8,7 +8,7 @@ export default {
env: env('NODE_ENV'),
hostname: env('HOSTNAME', os.hostname()),
deploymentEnv: env('ENVIRONMENT'),
platform: env('PLATFORM', PLATFORMS.IOS),
platform: env('PLATFORM', PLATFORMS.IOS) as PLATFORMS,
aws: {
accessKeyId: env('AWS_ACCESS_KEY_ID'),
secretAccessKey: env('AWS_SECRET_ACCESS_KEY'),
@@ -47,11 +47,9 @@ export default {
client: {
level: env('CLIENT_LOGGER_LEVEL', 'info'),
},
loggly: {
token: env('LOGGLY_TOKEN', ''),
subdomain: env('LOGGLY_SUBDOMAIN', ''),
buffer: envNum('LOGGLY_BUFFER', 100),
},
},
google: {
credentials: env('GOOGLE_APPLICATION_CREDENTIALS', ''),
},
sentry: {
dsn: env('SENTRY_DSN', ''),
+4 -4
View File
@@ -1,9 +1,9 @@
import * as BUILD from './build';
import * as IOS from './ios';
export const PLATFORMS = {
IOS: 'ios',
ANDROID: 'android',
};
export enum PLATFORMS {
IOS = 'ios',
ANDROID = 'android',
}
export { IOS, BUILD };
+5 -4
View File
@@ -1,4 +1,5 @@
export const BUILD_TYPES = {
ARCHIVE: 'archive',
SIMULATOR: 'simulator',
};
export enum BUILD_TYPES {
ARCHIVE = 'archive',
CLIENT = 'client',
SIMULATOR = 'simulator',
}
+6 -3
View File
@@ -1,3 +1,5 @@
import { IOS } from 'turtle/constants';
export interface IJob {
platform: 'android' | 'ios';
experienceName: string;
@@ -8,7 +10,7 @@ export interface IJob {
releaseChannel: string;
publicUrl?: string;
// ios
buildType?: string;
buildType?: IOS.BUILD_TYPES;
bundleIdentifier?: string;
};
credentials: {
@@ -16,13 +18,14 @@ export interface IJob {
keystore?: string;
keystoreAlias?: string;
keystorePassword?: string;
// ios
keyPassword?: string;
// ios
certP12?: string;
certPassword?: string;
provisioningProfile?: string;
password?: string;
teamId?: string;
appleSession?: string;
udids?: string[];
};
projectDir: string;
fakeUploadDir?: string;
+42 -38
View File
@@ -1,85 +1,80 @@
import path from 'path';
import fs from 'fs-extra';
import _ from 'lodash';
import * as sqs from 'turtle/aws/sqs';
import builders from 'turtle/builders';
import BuildError from 'turtle/builders/BuildError';
import { logErrorOnce } from 'turtle/builders/utils/common';
import config from 'turtle/config';
import { BUILD } from 'turtle/constants/index';
import logger from 'turtle/logger';
import * as buildDurationMetric from 'turtle/metrics/buildDuration';
import * as buildStatusMetric from 'turtle/metrics/buildStatus';
import { checkShouldExit, setCurrentJobId } from 'turtle/turtleContext';
import { checkShouldExit, setCurrentJobId, turtleVersion } from 'turtle/turtleContext';
import { getPriorities } from 'turtle/utils/priorities';
import * as redis from 'turtle/utils/redis';
import { sanitizeJob } from 'turtle/validator';
const { version: turtleVersion } = JSON.parse(fs.readFileSync(path.join(__dirname, '../package.json'), 'utf-8'));
function _maybeExit() {
function _maybeExit(logger: any) {
if (checkShouldExit()) {
logger.warn('Exiting due to previously received termination signal.');
process.exit();
}
}
export async function doJob() {
const jobData = await getJob();
await processJob(jobData);
_maybeExit();
export async function doJob(logger: any) {
const jobData = await getJob(logger);
await processJob(jobData, logger);
_maybeExit(logger);
}
export async function getJob() {
_maybeExit();
export async function getJob(logger: any) {
_maybeExit(logger);
logger.info('Fetching job');
while (true) {
try {
let job = null;
const priorities = await getPriorities();
const priorities = await getPriorities(logger);
for (const priority of priorities) {
job = await sqs.receiveMessage(priority);
_maybeExit();
job = await sqs.receiveMessage(priority, logger);
_maybeExit(logger);
if (job != null) {
job.priority = priority;
break;
}
}
_maybeExit();
_maybeExit(logger);
if (job !== null) {
return job;
}
} catch (err) {
logger.error('Error at receiving messages', err);
logger.error({ err }, 'Error at receiving messages');
}
}
}
async function processJob(jobData: any) {
async function processJob(jobData: any, logger: any) {
const receiptHandle = jobData.ReceiptHandle;
const { priority } = jobData;
let timeoutId;
try {
const rawJob = JSON.parse(jobData.Body);
timeoutId = failAfterMaxJobTime(priority, receiptHandle, rawJob);
timeoutId = failAfterMaxJobTime(priority, receiptHandle, rawJob, logger);
const job = await sanitizeJob(rawJob);
setCurrentJobId(job.id);
const pingerHandle = sqs.changeMessageVisibilityRecurring(priority, jobData.ReceiptHandle, job.id);
const pingerHandle = sqs.changeMessageVisibilityRecurring(priority, jobData.ReceiptHandle, job.id, logger);
logger.info(`Doing job MessageId=${jobData.MessageId} BuildId=${job.id} ${Date.now()}`);
const cancelled = await redis.checkIfCancelled(job.id);
const cancelled = await redis.checkIfCancelled(job.id, logger);
if (cancelled) {
logger.info('The job has been cancelled');
} else {
redis.registerListener(job.id, () => deleteMessage(priority, receiptHandle));
redis.registerListener(job.id, () => deleteMessage(priority, receiptHandle, logger), logger);
const startTimestamp = Date.now();
let buildFailed = false;
const buildType = _.get(job, 'config.buildType', 'default');
try {
const status = await build(job);
const status = await build(job, logger);
buildStatusMetric.add(buildType, status);
buildFailed = !status;
} catch (err) {
@@ -97,13 +92,13 @@ async function processJob(jobData: any) {
);
buildDurationMetric.addTotalDuration(buildType, totalBuildDurationSecs, !buildFailed);
}
redis.unregisterListeners();
redis.unregisterListeners(logger);
}
logger.info(`Job done MessageId=${jobData.MessageId} BuildId=${job.id} ${Date.now()}`);
}
clearInterval(pingerHandle);
} finally {
await deleteMessage(priority, receiptHandle);
await deleteMessage(priority, receiptHandle, logger);
if (timeoutId) {
logger.info('Clearing job failer timeout...');
clearTimeout(timeoutId);
@@ -111,11 +106,11 @@ async function processJob(jobData: any) {
}
}
function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any) {
function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any, logger: any) {
return setTimeout(async () => {
try {
sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, { turtleVersion });
await deleteMessage(priority, receiptHandle);
sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, { turtleVersion }, logger);
await deleteMessage(priority, receiptHandle, logger);
} finally {
logger.error('Going to terminate turtle agent, just in case...');
process.exit(1);
@@ -123,39 +118,48 @@ function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any)
}, config.builder.maxJobTimeMs);
}
async function build(job: any) {
async function build(job: any, logger: any) {
const startTimestamp = Date.now();
const s3Url = await logger.init(job);
const buildLogger = logger.child({ jobID: job.id, experienceName: job.experienceName });
const calculateBuildDuration = () => Math.ceil((Date.now() - startTimestamp) / 1000);
try {
await sqs.sendMessage(job.id, BUILD.JOB_STATES.IN_PROGRESS, {
logUrl: s3Url,
logFormat: 'json',
});
const result = await (builders as any)[job.platform](job);
}, buildLogger);
const result = await builders[job.platform](job, buildLogger);
const buildDuration = calculateBuildDuration();
sqs.sendMessage(job.id, BUILD.JOB_STATES.FINISHED, {
...result,
turtleVersion,
buildDuration,
});
}, buildLogger);
return true;
} catch (err) {
logErrorOnce(err);
logErrorOnce(err, buildLogger);
const buildDuration = calculateBuildDuration();
sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, { turtleVersion, buildDuration });
let reason;
if (err instanceof BuildError) {
reason = err.reason;
}
sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, {
turtleVersion,
buildDuration,
...reason && { reason },
}, buildLogger);
return false;
} finally {
await logger.cleanup(job);
}
}
async function deleteMessage(priority: string, receiptHandle: string) {
async function deleteMessage(priority: string, receiptHandle: string, logger: any) {
try {
setCurrentJobId(null);
await sqs.deleteMessage(priority, receiptHandle);
} catch (err) {
logger.error('Error at deleting msg', err);
logger.error({ err }, 'Error at deleting msg');
}
}
+6 -4
View File
@@ -12,11 +12,13 @@ export default baseJobSchema.concat(
}),
credentials: Joi.object()
.keys({
certP12: Joi.string(),
certPassword: Joi.string(),
teamId: Joi.string(),
certP12: Joi.string().required(),
certPassword: Joi.string().required(),
teamId: Joi.string().required(),
// it's not required for adhoc Expo Client builds
provisioningProfile: Joi.string(),
password: Joi.string(),
appleSession: Joi.string(),
udids: Joi.array().items(Joi.string()),
})
.when('config.buildType', {
is: IOS.BUILD_TYPES.SIMULATOR,
-48
View File
@@ -1,48 +0,0 @@
import EventEmitter from 'events';
import BunyanLoggly from 'bunyan-loggly';
import config from 'turtle/config';
import * as constants from 'turtle/constants/logger';
import { IJob } from 'turtle/job';
export interface ILogglyConfig {
token: string;
subdomain: string;
tags: string[];
}
interface ILogglyExtraFields {
jobID: string;
experienceName: string;
}
export default class HackyLogglyStream extends EventEmitter {
private logglyStream: BunyanLoggly;
private extraFields: ILogglyExtraFields | null;
constructor(logglyConfig: ILogglyConfig) {
super();
this.logglyStream = new BunyanLoggly(logglyConfig, config.logger.loggly.buffer);
this.extraFields = null;
}
public init(job: IJob) {
this.extraFields = {
jobID: job.id,
experienceName: job.experienceName,
};
}
public cleanup() {
this.extraFields = null;
}
public write(rec: any) {
const { msg: message, hostname: host, level, ...other } = rec;
this.logglyStream.write({
message,
level: constants.LEVELS_MAPPING[level],
host,
...other,
data: this.extraFields || {},
});
}
}
+16 -33
View File
@@ -4,12 +4,15 @@ import { SentryStream } from 'bunyan-sentry-stream';
import { Client as RavenClient } from 'raven';
import config from 'turtle/config';
import * as constants from 'turtle/constants/logger';
import { IJob } from 'turtle/job';
import HackyLogglyStream from 'turtle/logger/hackyLogglyStream';
import S3Stream from 'turtle/logger/s3Stream';
import { isOffline } from 'turtle/turtleContext';
// type error when using import
// https://github.com/googleapis/nodejs-logging-bunyan/issues/241
// tslint:disable-next-line:no-var-requires
const { LoggingBunyan } = require('@google-cloud/logging-bunyan');
interface IStream {
stream: any;
type?: string;
@@ -18,7 +21,6 @@ interface IStream {
}
export const s3logger = new S3Stream();
let logglyStream: HackyLogglyStream;
const streams: IStream[] = [];
@@ -47,53 +49,34 @@ if (config.sentry.dsn && config.deploymentEnv !== 'development') {
});
}
if (config.logger.loggly.token) {
const logglyConfig = {
token: config.logger.loggly.token,
subdomain: config.logger.loggly.subdomain,
// prettier-ignore
tags: [
'app-shell-apps',
`platform.${config.platform}`,
`environment.${config.deploymentEnv}`,
],
if (config.google.credentials) {
const resource = {
type: 'generic_node',
labels: {
node_id: config.hostname,
location: '', // default value
namespace: '', // default value
},
};
logglyStream = new HackyLogglyStream(logglyConfig);
streams.push({
type: 'raw',
stream: logglyStream,
});
streams.push(new LoggingBunyan({ name: 'turtle', resource }).stream('info'));
}
const logger = bunyan.createLogger({
name: 'turtle',
level: config.logger.level,
platform: config.platform,
environment: config.deploymentEnv,
streams,
});
logger.withFields = (extraFields: any) => withFields(logger, extraFields);
logger.init = async (job: IJob) => {
if (logglyStream) {
logglyStream.init(job);
}
return await s3logger.init(job);
};
logger.cleanup = async () => {
// a little hacky, but works
logger.info({ lastBuildLog: true }, 'this is the last log from this build');
if (logglyStream) {
logglyStream.cleanup();
}
await s3logger.waitForLogger();
};
export default logger;
export function withFields(loggerObj: any, extraFields: any) {
return constants.LEVELS.reduce((obj, level) => {
obj[level] = (...args: any[]) => loggerObj[level](extraFields, ...args);
return obj;
}, {} as any);
}
+1 -2
View File
@@ -1,11 +1,10 @@
import * as cloudwatch from 'turtle/aws/cloudwatch';
import { TYPE_DIMENSIONS } from 'turtle/aws/cloudwatch/dimensions';
import logger from 'turtle/logger';
const TURTLE_BUILD_DURATION_METRIC_NAME = 'turtleBuildDuration';
const TOTAL_BUILD_DURATION_METRIC_NAME = 'totalBuildDuration';
export function register() {
export function register(logger: any) {
for (const metricName of [TURTLE_BUILD_DURATION_METRIC_NAME, TOTAL_BUILD_DURATION_METRIC_NAME]) {
logger.info(`Registered AWS Cloudwatch metric: ${metricName}`);
cloudwatch.registerMetric({
+1 -2
View File
@@ -2,11 +2,10 @@ import * as cloudwatch from 'turtle/aws/cloudwatch';
import { TYPE_DIMENSIONS } from 'turtle/aws/cloudwatch/dimensions';
import * as reducers from 'turtle/aws/cloudwatch/reducers';
import config from 'turtle/config';
import logger from 'turtle/logger';
const BUILD_METRIC_NAME = 'build';
export function register() {
export function register(logger: any) {
logger.info(`Registered AWS Cloudwatch metric: ${BUILD_METRIC_NAME}`);
cloudwatch.registerMetric({
metricName: BUILD_METRIC_NAME,
+3 -4
View File
@@ -1,12 +1,11 @@
import * as cloudwatch from 'turtle/aws/cloudwatch';
import logger from 'turtle/logger';
import * as buildDuration from 'turtle/metrics/buildDuration';
import * as buildStatus from 'turtle/metrics/buildStatus';
const metricsToRegister = [buildStatus, buildDuration];
export async function init() {
export async function init(logger: any) {
logger.info('Initializing Cloudwatch metrics');
await cloudwatch.init();
metricsToRegister.forEach((metric) => metric.register());
await cloudwatch.init(logger);
metricsToRegister.forEach((metric) => metric.register(logger));
}
-13
View File
@@ -1,13 +0,0 @@
export = BunyanLoggly;
declare class BunyanLoggly {
constructor(options: BunyanLoggly.IOptions, bufferLength?: number, bufferTimeout?: number, logglyCallback?: any);
public write(data: any): void;
}
declare namespace BunyanLoggly {
interface IOptions {
token: string;
subdomain: string;
}
}
+18 -6
View File
@@ -4,9 +4,12 @@ import config from 'turtle/config';
import { doJob } from 'turtle/jobManager';
import logger from 'turtle/logger';
import setup from 'turtle/setup';
import { checkShouldExit, setShouldExit } from 'turtle/turtleContext';
import { checkShouldExit, setShouldExit, turtleVersion } from 'turtle/turtleContext';
import { getRedisClient, RedisClient } from 'turtle/utils/redis';
process.on('unhandledRejection', (err) => logger.error('Unhandled promise rejection:', err));
const REDIS_TURTLE_VERSION_KEY = 'turtle:version';
process.on('unhandledRejection', (err) => logger.error({ err }, 'Unhandled promise rejection'));
function handleExit() {
if (checkShouldExit()) {
@@ -28,18 +31,27 @@ async function main() {
);
LoggerDetach.configure(logger);
if (setup[config.platform]) {
await setup[config.platform]();
await setup[config.platform](logger);
}
try {
const redis = await getRedisClient(logger, RedisClient.Configuration);
await redis.set(REDIS_TURTLE_VERSION_KEY, turtleVersion);
logger.info(`Register Turtle version ${turtleVersion} in Redis`);
} catch (err) {
logger.error({ err }, 'Failed to register Turtle version in Redis');
}
while (true) {
try {
await doJob();
await doJob(logger);
} catch (err) {
logger.error('Failed to do a job', err);
logger.error({ err }, 'Failed to do a job');
}
}
}
main()
.then(() => logger.error('This should never happen...'))
.catch((err) => logger.error('Something went terribly wrong :(', err))
.catch((err) => logger.error({ err }, 'Something went terribly wrong'))
.then(() => process.exit(1));
+2 -3
View File
@@ -1,8 +1,7 @@
import logger from 'turtle/logger';
import commonSetup from 'turtle/setup/common';
export default async function setup() {
export default async function setup(logger: any) {
logger.info('Setting up environment...');
await commonSetup();
await commonSetup(logger);
logger.info('Finished setting up environment');
}
+2 -2
View File
@@ -3,8 +3,8 @@ import fs from 'fs-extra';
import config from 'turtle/config';
import * as metrics from 'turtle/metrics';
export default async function commonSetup() {
await metrics.init();
export default async function commonSetup(logger: any) {
await metrics.init(logger);
await fs.remove(config.directories.temporaryFilesRoot);
await fs.remove(config.directories.tempS3LogsDir);
}
+2 -3
View File
@@ -3,7 +3,6 @@ import os from 'os';
import path from 'path';
import * as keychain from 'turtle/builders/utils/ios/keychain';
import logger from 'turtle/logger';
import commonSetup from 'turtle/setup/common';
async function deleteProvisioningProfilesFromHomedir() {
@@ -17,10 +16,10 @@ async function deleteProvisioningProfilesFromHomedir() {
}
}
export default async function setup() {
export default async function setup(logger: any) {
logger.info('Setting up environment...');
await deleteProvisioningProfilesFromHomedir();
await keychain.cleanUp();
await commonSetup();
await commonSetup(logger);
logger.info('Finished setting up environment');
}
+5
View File
@@ -1,5 +1,10 @@
import fs from 'fs-extra';
import path from 'path';
import config from 'turtle/config';
const { version: turtleVersion } = JSON.parse(fs.readFileSync(path.join(__dirname, '../package.json'), 'utf-8'));
export { turtleVersion };
let shouldExit = false;
let currentJobId: string | null = null;
+9 -7
View File
@@ -2,7 +2,6 @@ import Joi from 'joi';
import os from 'os';
import config from 'turtle/config';
import logger from 'turtle/logger';
import { getRedisClient, RedisClient } from 'turtle/utils/redis';
export const NORMAL = 'normalPriority';
@@ -40,7 +39,11 @@ export function createDefaultConfigurationKey(platform = config.platform) {
return `${configPrefix(platform)}:default`;
}
export async function getPriorities() {
export async function getPriorities(logger: any) {
if (config.env === 'test') {
return NORMAL_CONFIGURATION;
}
try {
const redis = await getRedisClient(RedisClient.Configuration);
if (redis.getConfig === undefined) {
@@ -56,24 +59,23 @@ export async function getPriorities() {
logger.debug(`Using configuration at index ${configurationsIndex} pulled from redis`);
const { error, value: configValue } = configSchema.validate(configurations[configurationsIndex]);
if (error) {
logger.warn(error);
logger.warn({ err: error });
throw new Error('Received configuration is not valid');
}
return configValue;
} catch (err) {
logger.warn(err);
logger.warn({ err });
const defaultConfig = JSON.parse(await redis.get(createDefaultConfigurationKey()));
const { error, value: defaultConfigValue } = configSchema.validate(defaultConfig);
if (error) {
logger.warn(error);
logger.warn({ err: error });
throw new Error('Received default configuration is not valid');
}
logger.warn('Using default configuration pulled from redis');
return defaultConfigValue;
}
} catch (err) {
logger.warn(err);
logger.warn('Using configuration chosen locally');
logger.warn({ err }, 'Using configuration chosen locally');
if (Math.random() < 0.7) {
return HIGH_CONFIGURATION;
} else {
+8 -9
View File
@@ -1,7 +1,6 @@
import Redis from 'ioredis';
import config from 'turtle/config';
import logger from 'turtle/logger';
const MILLIS_TO_UPLOAD_LOGS = 3000;
export const MILLIS_CONNECTION_TIMEOUT = 10000;
@@ -47,31 +46,31 @@ const redisClients: IRedisClients = {
[RedisClient.Configuration]: null,
};
export async function getRedisClient(type = RedisClient.Default) {
export async function getRedisClient(logger: any, type = RedisClient.Default) {
if (!redisClients[type]) {
try {
redisClients[type] = await connect(MILLIS_CONNECTION_TIMEOUT, type);
} catch (err) {
logger.error(err);
logger.error({ err });
}
}
return redisClients[type];
}
export async function checkIfCancelled(jobId: string) {
export async function checkIfCancelled(jobId: string, logger: any) {
try {
const redis = await getRedisClient();
const redis = await getRedisClient(logger);
return await redis.get(`jobs:cancelled:${jobId}`);
} catch (err) {
if (config.deploymentEnv === 'development') {
logger.warn('Did you turn on redis server? Run `yarn start-docker` in server/www');
}
logger.error(err);
logger.error({ err });
return false;
}
}
export async function registerListener(jobId: string, deleteMessage: any) {
export async function registerListener(jobId: string, deleteMessage: any, logger: any) {
try {
const redis = await getRedisClient(RedisClient.Subscriber);
redis.subscribe('jobs:cancelled');
@@ -88,9 +87,9 @@ export async function registerListener(jobId: string, deleteMessage: any) {
}
}
export async function unregisterListeners() {
export async function unregisterListeners(logger: any) {
try {
const redis = await getRedisClient();
const redis = await getRedisClient(logger);
redis.removeAllListeners('message');
} catch (err) {
logger.error(err);
Executable
+3
View File
@@ -0,0 +1,3 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws "$@"
+3
View File
@@ -0,0 +1,3 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws_bash_completer "$@"
+3
View File
@@ -0,0 +1,3 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../nix" awscli --command aws_completer "$@"
+3
View File
@@ -0,0 +1,3 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../nix" git-crypt --command git-crypt "$@"
+19
View File
@@ -0,0 +1,19 @@
#! /usr/bin/env bash
set -euo pipefail
scriptDir=$(dirname "$0")
nixDir="$scriptDir/../../nix"
nix build -f $nixDir --no-link "$@"
for package in "$@"; do
for b in $(ls $(nix path-info -f $nixDir $package)/bin); do
cat << EOF > "$scriptDir/$b"
#! /usr/bin/env bash
# This file was generated by $(basename "$0")
exec nix run -f "\$(dirname "\$0")/../../nix" $package --command $b "\$@"
EOF
chmod +x "$scriptDir/$b"
done
done
Executable
+3
View File
@@ -0,0 +1,3 @@
#! /usr/bin/env bash
# This file was generated by nix-binstubs
exec nix run -f "$(dirname "$0")/../../nix" nodejs-8_x --command node "$@"
Loaded 100 of 106 files, more files were not shown because too many files have changed in this diff. Show more