add script for for setting builder replica count in redis

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Added
 - Turtle CLI logo to the project (created by [@zularizal](https://github.com/zularizal))!
+### Changed
+- Updated Android shell app for SDK 32 to fix AppAuth issue (https://github.com/expo/expo/pull/4115).
+- Default value for build timeout to 15 minutes.
+### Fixed
+- Generating Android keystore in turtle when one is not passed to turtle-cli (https://github.com/expo/turtle/pull/65).
+- Upload logs to s3 if build has been canceled or timed out.
 
 ## [0.7.2] - 2019-05-09
 ### Added

package.json

@@ -36,7 +36,11 @@
     "set-priorities:default": "expenv ts-node -r tsconfig-paths/register --files src/scripts/setConfigurations.ts",
     "set-priorities:local": "EXPENV_OVERRIDES=local-common,local,local-queue yarn set-priorities:default",
     "set-priorities:staging": "yarn set-priorities:default",
-    "set-priorities:production": "EXPENV_OVERRIDES=prod yarn set-priorities:default"
+    "set-priorities:production": "EXPENV_OVERRIDES=prod yarn set-priorities:default",
+    "set-replica-count:default": "expenv ts-node -r tsconfig-paths/register --files src/scripts/setReplicaCount.ts | bunyan -o short -l debug",
+    "set-replica-count:local": "EXPENV_OVERRIDES=local-common,local,local-queue yarn set-replica-count:default",
+    "set-replica-count:staging": "yarn set-replica-count:default",
+    "set-replica-count:production": "EXPENV_OVERRIDES=prod yarn set-replica-count:default"
   },
   "husky": {
     "hooks": {
@@ -130,7 +134,7 @@
     "tar": "^4.4.6",
     "uuid": "^3.2.1",
     "which": "^1.3.1",
-    "xdl": "53.5.4"
+    "xdl": "53.7.3"
   },
   "optionalDependencies": {
     "@expo/traveling-fastlane-darwin": "1.9.4"

shellTarballs/android/sdk32

@@ -1 +1 @@
-s3://exp-artifacts/android-shell-builder-0cab68670fc50ce1dc16b7d86ca7976870062c39.tar.gz
+s3://exp-artifacts/android-shell-builder-c4ba6f73ab341afa378732f863e261a10fc71356.tar.gz

src/bin/commands/build/android.ts

@@ -38,6 +38,7 @@ const buildJobObject = (appJSON: any, { releaseChannel, username, projectDir, pu
   config: {
     ..._.get(appJSON, 'expo.android.config', {}),
     releaseChannel,
+    androidPackage: _.get(appJSON, 'expo.android.package'),
     publicUrl,
   },
   id: uuid.v4(),

src/builders/utils/android/credentials.ts

@@ -1,6 +1,7 @@
 import path from 'path';
 
 import fs from 'fs-extra';
+import get from 'lodash/get';
 import uuidv4 from 'uuid/v4';
 import { Credentials } from 'xdl';
 
@@ -19,7 +20,14 @@ async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredential
     credentials.keystorePassword = uuidv4().replace(/-/g, '');
     credentials.keyPassword = uuidv4().replace(/-/g, '');
     credentials.keystoreAlias = Buffer.from(jobData.experienceName).toString('base64');
-    const androidPackage = jobData.manifest.android.package;
+    const androidPackage =
+      get(jobData, 'manifest.android.package')
+      || get(jobData, 'config.androidPackage');
+    if (!androidPackage) {
+      throw new Error(
+        'Android package name is not set in the app manifest (please update app.json if you\'re using turtle-cli).',
+      );
+    }
 
     await Credentials.Android.createKeystore(
       {

src/config.ts

@@ -64,7 +64,7 @@ export default {
       (val: string) => env('NODE_ENV') === 'development' && val === '1',
     ),
     fakeUpload: envTransform('TURTLE_FAKE_UPLOAD', '0', (val: string) => val === '1'),
-    maxJobTimeMs: envNum('TURTLE_MAX_JOB_TIME_MS', 60 * 60 * 1000),
+    maxJobTimeMs: envNum('TURTLE_MAX_JOB_TIME_MS', 15 * 60 * 1000),
     useLocalWorkingDir: envTransform('TURTLE_USE_LOCAL_WORKING_DIR', '0', (val: string) => val === '1'),
   },
   directories: {

src/job.ts

@@ -12,6 +12,8 @@ export interface IJob {
     // ios
     buildType?: IOS.BUILD_TYPES;
     bundleIdentifier?: string;
+    // android
+    androidPackage?: string;
   };
   credentials: {
     // android

src/jobManager.ts

@@ -113,7 +113,8 @@ function failAfterMaxJobTime(priority: string, receiptHandle: string, job: any)
       sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, { turtleVersion });
       await deleteMessage(priority, receiptHandle);
     } finally {
-      logger.error('Going to terminate turtle agent, just in case...');
+      logger.error('Build timed out. Going to terminate turtle agent.');
+      await logger.cleanup();
       process.exit(1);
     }
   }, config.builder.maxJobTimeMs);

src/scripts/setConfigurations.ts

@@ -28,27 +28,33 @@ const TIMEOUT = 3000;
 
 const AVAILABLE_CONFIGURATIONS_IOS = [
   HIGH_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_ONLY_CONFIGURATION,
+  HIGH_ONLY_CONFIGURATION,
+  HIGH_ONLY_CONFIGURATION,
+  // there are 5 iOS builders at the moment
+  HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
 ];
 
 const AVAILABLE_CONFIGURATIONS_ANDROID = [
   HIGH_CONFIGURATION,
-  HIGH_ONLY_CONFIGURATION,
+  HIGH_CONFIGURATION,
+  HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_ONLY_CONFIGURATION,
-  HIGH_CONFIGURATION,
   HIGH_ONLY_CONFIGURATION,
-  HIGH_CONFIGURATION,
+  HIGH_ONLY_CONFIGURATION,
+  HIGH_ONLY_CONFIGURATION,
+  HIGH_ONLY_CONFIGURATION,
+  HIGH_ONLY_CONFIGURATION,
   HIGH_ONLY_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
-  HIGH_CONFIGURATION,
+  // there are 15 Android builders at the moment
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,
   HIGH_CONFIGURATION,

src/scripts/setReplicaCount.ts

@@ -0,0 +1,41 @@
+import isNaN from 'lodash/isNaN';
+import { PLATFORMS } from 'turtle/constants';
+import logger from 'turtle/logger';
+import { setReplicaCount } from 'turtle/utils/metadata';
+import { connect, RedisClient } from 'turtle/utils/redis';
+
+const REDIS_CONNECTION_TIMEOUT = 10000;
+
+async function run() {
+  const platform = process.env.SCRIPT_PLATFORM;
+  const replicaCountRaw = process.env.SCRIPT_REPLICA_COUNT;
+  const replicaCount = Number(replicaCountRaw);
+
+  if (!platform || !replicaCountRaw) {
+    throw new Error('You must set both SCRIPT_PLATFORM and SCRIPT_REPLICA_COUNT env variables.');
+  }
+
+  if (!Object.values(PLATFORMS).includes(platform)) {
+    throw new Error(`Unsupported platform: ${platform}`);
+  }
+
+  if (isNaN(replicaCount)) {
+    throw new Error(`Wrong value for replica count: ${replicaCountRaw}`);
+  }
+
+  const redis = await connect(REDIS_CONNECTION_TIMEOUT, RedisClient.Configuration);
+  logger.info(`[${platform}] Setting replica count in redis to ${replicaCount}`);
+
+  await setReplicaCount(redis, platform as PLATFORMS, replicaCount);
+
+  redis.disconnect();
+}
+
+if (require.main === module) {
+  run()
+    .then(() => logger.info('All done'))
+    .catch((err) => {
+      logger.error('Error:', err);
+      process.exit(1);
+    });
+}

src/utils/metadata.ts

@@ -0,0 +1,12 @@
+import config from 'turtle/config';
+import { PLATFORMS } from 'turtle/constants';
+
+const createKeyPrefix = (platform: PLATFORMS) => `${config.deploymentEnv}:${platform}`;
+const createReplicaCountKey = (platform: PLATFORMS) => `${createKeyPrefix(platform)}:replicaCount`;
+
+export async function setReplicaCount(redis: any, platform: PLATFORMS, replicaCount: number) {
+  await redis.set(
+    createReplicaCountKey(platform),
+    replicaCount,
+  );
+}

src/utils/redis.ts

@@ -79,6 +79,7 @@ export async function registerListener(jobId: string, deleteMessage: any) {
       if (message === jobId) {
         logger.info({ lastBuildLog: true }, 'Job cancelled - killing process');
         await deleteMessage();
+        await logger.cleanup();
         setTimeout(() => process.exit(1), MILLIS_TO_UPLOAD_LOGS);
       }
     });