improve docs and remove duplicated scripts

.circleci/config.yml

@@ -137,6 +137,23 @@ commands:
           paths:
             - "<< parameters.workingdir >>/<< parameters.platform >>/<< parameters.sdk_version >>"
 
+  deploy_turtle:
+    parameters:
+      platform:
+        type: string
+      env:
+        type: string
+    steps:
+      - checkout
+      - run:
+          name: Install curl & jq
+          command: nix-env -iA nixpkgs.curl nixpkgs.jq
+      - run:
+          name: Schedule a Turtle deploy
+          command: |
+            echo "We're going to deploy << parameters.platform >> Turtles to << parameters.env >>"
+            ./scripts/deploy.sh "<< parameters.platform >>" "<< parameters.env >>"
+
 jobs:
   build:
     executor: nix
@@ -203,6 +220,34 @@ jobs:
           sdk_version: sdk32
           platform: android
 
+  android_staging_deploy:
+    executor: nix
+    steps:
+      - deploy_turtle:
+          platform: android
+          env: staging
+
+  android_production_deploy:
+    executor: nix
+    steps:
+      - deploy_turtle:
+          platform: android
+          env: production
+
+  ios_staging_deploy:
+    executor: nix
+    steps:
+      - deploy_turtle:
+          platform: ios
+          env: staging
+
+  ios_production_deploy:
+    executor: nix
+    steps:
+      - deploy_turtle:
+          platform: ios
+          env: production
+
 workflows:
   checks:
     jobs:
@@ -212,3 +257,25 @@ workflows:
       - unit_tests
       - ios_smoke_tests
       - android_smoke_tests
+  deploy:
+    jobs:
+      - android_staging_approve_deploy:
+          type: approval
+      - android_staging_deploy:
+          requires:
+            - android_staging_approve_deploy
+      - android_production_approve_deploy:
+          type: approval
+      - android_production_deploy:
+          requires:
+            - android_production_approve_deploy
+      - ios_staging_approve_deploy:
+          type: approval
+      - ios_staging_deploy:
+          requires:
+            - ios_staging_approve_deploy
+      - ios_production_approve_deploy:
+          type: approval
+      - ios_production_deploy:
+          requires:
+            - ios_production_approve_deploy

.envrc

@@ -8,16 +8,13 @@ source_local() {
   watch_file $file
 }
 
-source_up || true
 source_local
 
 export NODE_ENV=development
 
 export EXPO_TURTLE_TOOLS_DIR="$(pwd)/tools"
 
-if [ -z "$EXPO_UNIVERSE_DIR" ]; then
-  PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
-  if [ "$CI" = "true" ]; then
-    PATH_add $EXPO_TURTLE_TOOLS_DIR/ci/bin
-  fi
+PATH_add $EXPO_TURTLE_TOOLS_DIR/bin
+if [ "$CI" = "true" ]; then
+  PATH_add $EXPO_TURTLE_TOOLS_DIR/ci/bin
 fi

CONTRIBUTING.md

@@ -16,9 +16,6 @@ This repository contains the codebase for the `turtle-cli` NPM package. The same
 ### Branches
 The `master` branch of the repository should be kept releasable at any time. This way we can continuously release fixes and improvements without costly managing of different branches and issues will be noticed and fixed quickly. This also ensures other contributors can check out the latest version from GitHub and work on it with minimal disruption from other features in progress.
 
-## Setting up the repository for development
-*TODO*
-
 ## Submitting a pull request
 To submit a pull request:
 1. Fork the [repository](https://github.com/expo/turtle) and create a feature branch. (Existing contributors can create feature branches without forking. Prefix the branch name with `@your-github-username/`.)

DEVELOPMENT.md

@@ -0,0 +1,36 @@
+# Turtle Development
+
+## Prerequisites
+
+- Working `www` service, see [this README](https://github.com/expo/universe/tree/master/server/www/README.md) for more details.
+- Own AWS SQS queues. See [this README](https://github.com/expo/universe/tree/master/terraform/turtle-queues/README.md) to learn more.
+
+## Setup
+
+- Run `update-local-secrets` to populate the `.secrets*` files. You'll need to run this any time any secret changes (it shouldn't be very often).
+- Run `EXPO_UNIVERSE_DIR=/path/to/universe/dir yarn secrets:init-private YOUR_SQS_NICKNAME`. Alternatively, you can create `.secrets.local-queue` file with the SQS queues' urls yourself (`AWS_SQS_OUT_QUEUE_URL`, `AWS_SQS_ANDROID_QUEUE_URL`, `AWS_SQS_ANDROID_PRIORITY_QUEUE_URL`, `AWS_SQS_IOS_QUEUE_URL`, `AWS_SQS_IOS_PRIORITY_QUEUE_URL`)
+- Run `yarn set-workingdir:local` or `yarn set-workingdir:remote`:
+  - `:local` creates symlinks to the shell apps in `expo` repository (_unfortunately this is currently broken_)
+  - `:remote` downloads precompiled shell apps
+- Run `yarn` to install node modules.
+- Run `yarn set-priorities:local` to set priorities configuration. This is not mandatory, but you'll get a lot of annoying messages without this.
+- Run `yarn start:ios` or `yarn start:android` to run Turtle.
+
+## Updating shell apps
+
+_TODO_
+
+## Generating recording files for smoke tests
+
+Smoke tests are powered by [Nock](https://www.npmjs.com/package/nock) which uses JSON files containing responses to the HTTP requests fired by the build process.
+
+To record new responses remove old `recording.json` and run tests and then manually schedule a build with `expo-cli` tool.
+
+As a precaution, the file with HTTP requests/responses contains, among other things, iOS certificates used for the test build, is
+encrypted with `git-crypt`. Remember not to revoke them, because then the tests will fail.
+
+## Other notes
+
+Turtle uses Redis to get notifications (from www) about builds cancellations. To make this feature working, you have to use the same Redis instance as www. Therefore, you need to remember about starting www before starting Turtle. If you forget about it, don't worry, Turtle will remind you to run `yarn start-docker` in `server/www` directory.
+
+Turtle uploads build artifacts to S3. This could take some time if you have a poor internet connection. To facilitate local development, there is a feature which allows Turtle to fake artifacts upload. `secrets:init-private` script, which you should have run while setting up Turtle, sets fake upload directory to your user's default `Downloads` directory (we're assuming you're using Mac). If you need to test uploading artifacts to S3, you have to edit `.secrets.local` file and comment or remove `TURTLE_FAKE_UPLOAD` and `TURTLE_FAKE_UPLOAD_DIR` environment variables.

README.md

@@ -15,21 +15,20 @@ Turtle CLI is a command line interface for building Expo standalone apps.
 You can use it both on your CI and your private computer.
 
 - [Changelog](CHANGELOG.md)
-- [Contributing to Turtle CLI](CONTRIBUTING.md)
+- [Contributing](CONTRIBUTING.md)
+- [Development](DEVELOPMENT.md)
 - [Code of Conduct](CODE_OF_CONDUCT.md)
 
-## Glossary
-- **Turtle** - script which takes a shell app, fills it with user's code and produces a standalone app
-- **Shell app** — precompiled (in case of iOS builds) Expo app which doesn't have user's code
-- **Standalone app** - .ipa/.apk file, a user uploads standalone apps to the Google Play Store/Apple App Store
-
 ## Installation
+
 [Installation instructions and documentation here.](https://docs.expo.io/versions/latest/distribution/turtle-cli)
 
 ## Usage example
+
 See the [expo/turtle-cli-example](https://github.com/expo/turtle-cli-example) repository to learn how to use Turtle CLI in order to build Expo standalone apps on your CI.
 
 ## Do you need help?
+
 - Make sure you've read [the Expo Docs](https://docs.expo.io/), especially [Building Standalone Apps on Your CI
 ](https://docs.expo.io/versions/latest/distribution/turtle-cli) and [Building Standalone Apps](https://docs.expo.io/versions/latest/distribution/building-standalone-apps) guides.
 - Search for existing GitHub issues [in this repository](https://github.com/expo/turtle/issues) and in [expo/expo](https://github.com/expo/expo/issues) repository.
@@ -39,5 +38,6 @@ See the [expo/turtle-cli-example](https://github.com/expo/turtle-cli-example) re
   * ask a question on [Expo Developers Slack](http://slack.expo.io).
 
 ## License
+
 The Turtle CLI source code is made available under the [MIT license](LICENSE).
 Some of the dependencies are licensed differently, with the BSD license, for example.

package.json

@@ -17,11 +17,6 @@
     "secrets:init-private": "./scripts/initPrivateSecrets.sh",
     "set-workingdir:remote": "./scripts/setWorkingdirMode.sh remote",
     "set-workingdir:local": "./scripts/setWorkingdirMode.sh local",
-    "docker:pull": "retry5 gcloud docker -- pull gcr.io/exponentjs/turtle-android:latest",
-    "docker:pull:tag": "retry5 gcloud docker -- pull gcr.io/exponentjs/turtle-android:${TAG}",
-    "docker:build": "yarn build:production android && gcloud docker -- build -t gcr.io/exponentjs/turtle-android:latest -t gcr.io/exponentjs/turtle-android:${TAG} .",
-    "docker:push": "retry5 gcloud docker -- push gcr.io/exponentjs/turtle-android:${TAG} && retry5 gcloud docker -- push gcr.io/exponentjs/turtle-android:latest",
-    "deploy:android": "./scripts/android/deploy.sh",
     "test": "yarn jest",
     "test:integration": "yarn test --config jest/integration-config.json",
     "test:unit": "yarn test --config jest/unit-config.json",
@@ -30,9 +25,6 @@
     "test:smoke:ios": "PLATFORM=ios yarn test:smoke --testNamePattern=iOS",
     "release": "./scripts/release.sh",
     "release:beta": "./scripts/releaseBeta.sh",
-    "ssh:android:pods-names": "./scripts/ssh/findAndroidPods.sh",
-    "ssh:android:connect": "./scripts/ssh/connectToAndroidPod.sh",
-    "ssh:ios:ips": "./scripts/ssh/findIosIps.sh",
     "set-priorities": "yarn set-priorities:local",
     "set-priorities:default": "expenv ts-node -r tsconfig-paths/register --files src/scripts/setConfigurations.ts",
     "set-priorities:local": "EXPENV_OVERRIDES=local-common,local,local-queue yarn set-priorities:default",

scripts/deploy.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+SCRIPT_NAME=`basename "$0"`
+DEPLOY_ENDPOINT_URL="https://circleci.com/api/v1.1/project/github/expo/turtle-deploy/tree/master"
+
+if [[ -z "$1" || -z "$2" ]]; then
+  echo "usage: ${bold}$SCRIPT_NAME PLATFORM ENVIRONMENT [FORCE]${normal}"
+  exit 1
+fi
+
+PLATFORM="$1"
+ENVIRONMENT="$2"
+
+if [[ "$3" = "--force" ]]; then
+  FORCE_DEPLOY="true"
+fi
+
+if [[ "$PLATFORM" != "android" && "$PLATFORM" != "ios" ]]; then
+  echo "unsupported PLATFORM passed: ${bold}$PLATFORM${normal}"
+  echo "valid values are: ${bold}android${normal}, ${bold}ios${normal}"
+  exit 1
+fi
+
+if [[ "$ENVIRONMENT" != "staging" && "$ENVIRONMENT" != "production" ]]; then
+  echo "unsupported ENVIRONMENT passed: ${bold}$ENVIRONMENT${normal}"
+  echo "valid values are: ${bold}staging${normal}, ${bold}production${normal}"
+  exit 1
+fi
+
+if [[ ! -z "$CIRCLE_SHA1" ]]; then
+  TURTLE_COMMIT="$CIRCLE_SHA1"
+elif [[ -z "$FORCE_DEPLOY" ]]; then
+  read -p "You're about to schedule a deploy, are you sure you want to deploy the current commit? (y/n) " -n 1 -r
+  echo    # (optional) move to a new line
+  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "exiting..."
+    exit 0
+  else
+    TURTLE_COMMIT=`git log --pretty=format:%H -1`
+  fi
+fi
+
+if [[ -z "$TURTLE_BOT_CIRCLE_API_USER_TOKEN" ]]; then
+  echo "Please set the TURTLE_BOT_CIRCLE_API_USER_TOKEN env variable."
+  exit 1
+fi
+
+if [[ $FORCE_DEPLOY = "true" ]]; then
+  curl -X POST \
+    --header "Content-Type: application/json" \
+    -u $TURTLE_BOT_CIRCLE_API_USER_TOKEN: \
+    -d '
+      {
+        "build_parameters": {
+          "CIRCLE_JOB": "deploy_'$PLATFORM'",
+          "TURTLE_ENV": "'$ENVIRONMENT'",
+          "TURTLE_FORCE_DEPLOY": "true"
+        }
+      }' \
+    $DEPLOY_ENDPOINT_URL | jq -r .build_url
+else
+  curl -X POST \
+    --header "Content-Type: application/json" \
+    -u $TURTLE_BOT_CIRCLE_API_USER_TOKEN: \
+    -d '
+      {
+        "build_parameters": {
+          "CIRCLE_JOB": "deploy_'$PLATFORM'",
+          "TURTLE_ENV": "'$ENVIRONMENT'",
+          "TURTLE_COMMIT": "'$TURTLE_COMMIT'"
+        }
+      }' \
+    $DEPLOY_ENDPOINT_URL | jq -r .build_url
+fi

scripts/initPrivateSecrets.sh

@@ -13,13 +13,17 @@ fi
 TURTLE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
 TERRAFORM_DIR="${EXPO_UNIVERSE_DIR}/terraform"
 
-ANDROID=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_android_in | head -1 | cut -d"=" -f2 | xargs`
-IOS=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_ios_in | head -1 | cut -d"=" -f2 | xargs`
+ANDROID_NORMAL=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_android_in | head -1 | cut -d"=" -f2 | xargs`
+ANDROID_PRIORITY=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_android_in_priority | head -1 | cut -d"=" -f2 | xargs`
+IOS_NORMAL=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_ios_in | head -1 | cut -d"=" -f2 | xargs`
+IOS_PRIORITY=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_ios_in_priority | head -1 | cut -d"=" -f2 | xargs`
 OUT=`cd $TERRAFORM_DIR; terraform state show module.turtle_queues_local_$1.aws_sqs_queue.jobs_out | head -1 | cut -d"=" -f2 | xargs`
 
 cat << EOF > $TURTLE_DIR/.secrets.local-queue
-AWS_SQS_ANDROID_QUEUE_URL="$ANDROID"
-AWS_SQS_IOS_QUEUE_URL="$IOS"
+AWS_SQS_ANDROID_QUEUE_URL="$ANDROID_NORMAL"
+AWS_SQS_ANDROID_PRIORITY_QUEUE_URL="$ANDROID_PRIORITY"
+AWS_SQS_IOS_QUEUE_URL="$IOS_NORMAL"
+AWS_SQS_IOS_PRIORITY_QUEUE_URL="$IOS_PRIORITY"
 AWS_SQS_OUT_QUEUE_URL="$OUT"
 EOF
 

scripts/setWorkingdirMode.sh

@@ -6,42 +6,44 @@ DOWNLOAD_DIR="${EXPO_DOWNLOAD_DIR:-$HOME/Downloads}"
 
 mkdir -p $DOWNLOAD_DIR
 
-if [ ! -f "$ROOT_DIR/.secrets.local-queue" ]; then 
-  echo "You need to run \"yarn secrets:init-private [SQS_NAME]\" or create .secrets.local-queue file yourself with AWS_SQS_ANDROID_QUEUE_URL, AWS_SQS_IOS_QUEUE_URL and AWS_SQS_OUT_QUEUE_URL variables."
+if [ ! -f "$ROOT_DIR/.secrets.local-queue" ]; then
+  echo "You need to run \"yarn secrets:init-private [SQS_NAME]\" or create the .secrets.local-queue file yourself with AWS_SQS_ANDROID_QUEUE_URL, AWS_SQS_ANDROID_PRIORITY_QUEUE_URL, AWS_SQS_IOS_QUEUE_URL, AWS_SQS_IOS_PRIORITY_QUEUE_URL and AWS_SQS_OUT_QUEUE_URL variables."
   exit 1
 fi
 
-setup_local() {
-  ensure_link_exists() {
-    if [ ! -L "$2" ]; then
-      ln -s $1 $2
-      echo "$2 link created"
-    fi
-  }
-  WORKINGDIR="$ROOT_DIR/workingdir/local"
-  
-  if [[ -d $WORKINGDIR ]]; then
-    echo "Local workingdir already prepared"
-    return
-  fi
-
-  if [[ -z "$EXPO_UNIVERSE_DIR" ]]; then
-    echo "EXPO_UNIVERSE_DIR needs to be set"
-    exit -1
-  fi
-
-  cd $EXPO_UNIVERSE_DIR/exponent && yarn
-  cd $EXPO_UNIVERSE_DIR/exponent/tools-public && yarn && ./generate-dynamic-macros-android.sh
-
-  mkdir -p $WORKINGDIR
-  echo "$WORKINGDIR directory created"
-
-  LINKS=("template-files" "package.json" "android" "expokit-npm-package" "packages" "client-builds" "cpp" "exponent-view-template" "ios" "shellAppBase-builds" "shellAppWorkspaces" "node_modules")
-
-  for LINKNAME in ${LINKS[@]}; do
-    ensure_link_exists "$EXPO_UNIVERSE_DIR/exponent/$LINKNAME" "$WORKINGDIR/$LINKNAME"
-  done
-}
+# @dsokal: this is currently broken
+#
+# setup_local() {
+#   ensure_link_exists() {
+#     if [ ! -L "$2" ]; then
+#       ln -s $1 $2
+#       echo "$2 link created"
+#     fi
+#   }
+#   WORKINGDIR="$ROOT_DIR/workingdir/local"
+#
+#   if [[ -d $WORKINGDIR ]]; then
+#     echo "Local workingdir already prepared"
+#     return
+#   fi
+#
+#   if [[ -z "$EXPO_UNIVERSE_DIR" ]]; then
+#     echo "EXPO_UNIVERSE_DIR needs to be set"
+#     exit -1
+#   fi
+#
+#   cd $EXPO_UNIVERSE_DIR/exponent && yarn
+#   cd $EXPO_UNIVERSE_DIR/exponent/tools-public && yarn && ./generate-dynamic-macros-android.sh
+#
+#   mkdir -p $WORKINGDIR
+#   echo "$WORKINGDIR directory created"
+#
+#   LINKS=("template-files" "package.json" "android" "expokit-npm-package" "packages" "client-builds" "cpp" "exponent-view-template" "ios" "shellAppBase-builds" "shellAppWorkspaces" "node_modules")
+#
+#   for LINKNAME in ${LINKS[@]}; do
+#     ensure_link_exists "$EXPO_UNIVERSE_DIR/exponent/$LINKNAME" "$WORKINGDIR/$LINKNAME"
+#   done
+# }
 
 setup_remote() {
   if [[ -d $ROOT_DIR/workingdir/android && -d $ROOT_DIR/workingdir/ios ]]; then
@@ -55,7 +57,7 @@ setup_remote() {
       $ROOT_DIR/scripts/ios/fetchRemoteIosTarball.sh
     fi
   fi
-  
+
   for sdk in $ROOT_DIR/workingdir/android/* ; do
     if [[ -f "$sdk/universe-package.json" ]]; then
       mv $sdk/package.json $sdk/exponent-package.json
@@ -87,8 +89,10 @@ EOF
 }
 
 if [ "$1" == "local" ]; then
-  setup_local
-  switch_local
+  # setup_local
+  # switch_local
+  echo "Setting local workingdir is currently broken, sorry :("
+  exit -1
 elif [ "$1" == "remote" ]; then
   setup_remote
   switch_remote

scripts/ssh/connectToAndroidPod.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-if [ -z "$1" ]; then
-  echo "Please provide an environment name as a first argument"
-  exit 1
-fi
-
-if [ -z "$2" ]; then
-  echo "Please provide a pod name as a second argument"
-  exit 1
-fi
-
-kubectl --namespace $1 exec -it $2 -- /bin/bash

scripts/ssh/findAndroidPods.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-ENVIRONMENT=${1:-production}
-
-kubectl get pods --namespace $ENVIRONMENT | grep "turtle-android-" | sed -e "s/  */ /g" | cut -d" " -f1

scripts/ssh/findIosIps.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-ENVIRONMENT=${1:-production}
-
-govc vm.ip -wait 5s turtle-js-"${ENVIRONMENT}"-*

src/utils/redis.ts

@@ -64,7 +64,7 @@ export async function checkIfCancelled(jobId: string) {
     return await redis.get(`jobs:cancelled:${jobId}`);
   } catch (err) {
     if (config.deploymentEnv === 'development') {
-      logger.warn('Did you turn on redis server? Run `yarn start-docker` in server/www');
+      logger.warn('Did you turn on redis server? Run `yarn start-docker` in $EXPO_UNIVERSE_DIR/server/www');
     }
     logger.error({ err });
     return false;