fix for breaking changes in xdl

package.json

@@ -99,6 +99,7 @@
     "typescript": "^3.0.1"
   },
   "dependencies": {
+    "@expo/spawn-async": "^1.4.0",
     "async-retry": "^1.2.1",
     "aws-sdk": "^2.226.1",
     "bunyan": "^1.8.12",
@@ -122,6 +123,9 @@
     "tar": "^4.4.6",
     "uuid": "^3.2.1",
     "which": "^1.3.1",
-    "xdl": "52.0.14-alpha.6+371be041"
+    "xdl": "53.1.1"
+  },
+  "optionalDependencies": {
+    "@expo/traveling-fastlane-darwin": "^1.8.0"
   }
 }

shellTarballs/ios/client

@@ -0,0 +1 @@
+s3://exp-artifacts/ios-expo-client-15bb883c22173d5b01df19421580170e776041d0.tar.gz

src/aws/sqs.ts

@@ -62,7 +62,7 @@ export async function changeMessageVisibility(priority: string, receiptHandle: s
 }
 
 // Every VISIBILITY_TIMEOUT_SEC / 3 seconds we are telling AWS SQS
-// that we're still proccessing the message, so it does not
+// that we're still processing the message, so it does not
 // send the build job to another turtle agent
 export function changeMessageVisibilityRecurring(priority: string, receiptHandle: string, jobId: string) {
   return setInterval(() => {

src/bin/setup/utils/common.ts

@@ -6,13 +6,14 @@ import { ExponentTools } from 'xdl';
 
 import download from 'turtle/bin/setup/utils/downloader';
 import { ensureToolsAreInstalled, IToolDefinition } from 'turtle/bin/setup/utils/toolsDetector';
+import { IShellAppDirectoryConfig } from 'turtle/builders/utils/workingdir';
 import config from 'turtle/config';
 import logger from 'turtle/logger';
 
 const l = logger.withFields({ buildPhase: 'setting up environment' });
 
 interface IShellAppFormaters {
-  formatShellAppDirectory: (sdkVersion: string) => string;
+  formatShellAppDirectory: (config: IShellAppDirectoryConfig) => string;
   formatShellAppTarballUriPath: (sdkMajorVersion: string) => string;
 }
 
@@ -27,7 +28,7 @@ export async function ensureShellAppIsPresent(
   formatters: IShellAppFormaters,
   postDownloadAction?: PostDownloadAction,
 ) {
-  const workingdir = formatters.formatShellAppDirectory(sdkVersion);
+  const workingdir = formatters.formatShellAppDirectory({ sdkVersion });
   if (await fs.pathExists(workingdir)) {
     return;
   }

src/builders/BuildError.ts

@@ -0,0 +1,17 @@
+class BuildError extends Error {
+  public reason: BuildErrorReason;
+
+  constructor(reason: BuildErrorReason, message: string) {
+    super(message);
+    this.name = 'BuildError';
+    this.reason = reason;
+  }
+}
+
+enum BuildErrorReason {
+  SESSION_EXPIRED = 'session-expired',
+  PROVISIONING_PROFILE_MISSING = 'provisioning-profile-missing',
+}
+
+export default BuildError;
+export { BuildErrorReason };

src/builders/android.ts

@@ -62,7 +62,7 @@ async function runShellAppBuilder(
 
   const { manifest: { sdkVersion: sdkVersionFromManifest = null } = {}, sdkVersion: sdkVersionFromJob } = jobData;
   const sdkVersion = sdkVersionFromJob || sdkVersionFromManifest;
-  const workingDir = formatShellAppDirectory(sdkVersion);
+  const workingDir = formatShellAppDirectory({ sdkVersion });
 
   try {
     await AndroidShellApp.createAndroidShellAppAsync({

src/builders/ios/archive.ts

@@ -33,7 +33,7 @@ async function buildAndSignIPA(ctx: IContext, job: IJob, keychainPath: string, m
   l.info('building and signing IPA');
 
   const {
-    credentials: { provisioningProfile, certPassword, teamId, password },
+    credentials: { provisioningProfile, certPassword, teamId },
     config: { bundleIdentifier: bundleIdentifierFromConfig },
   } = job;
 
@@ -51,9 +51,8 @@ async function buildAndSignIPA(ctx: IContext, job: IJob, keychainPath: string, m
     certPath: ctx.tempCertPath,
     certPassword,
     teamID: teamId,
-    password,
     bundleIdentifier,
-    workingDir: ctx.workingDir,
+    workspacePath: ctx.workspacePath,
     manifest,
   });
   await ipaBuilder.build();

src/builders/ios/context.ts

@@ -22,9 +22,11 @@ export interface IContext {
   tempCertPath: string;
   uploadPath: string;
   workingDir: string;
+  workspacePath: string;
+  applicationFilesSrc: string;
 }
 
-const { DEFAULT_EXPOKIT_WORKSPACE_NAME } = IosShellApp;
+const { EXPOKIT_APP, EXPONENT_APP } = IosShellApp;
 const { BUILD_TYPES } = IOS;
 
 export function createBuilderContext(job: IJob): IContext {
@@ -39,7 +41,7 @@ export function createBuilderContext(job: IJob): IContext {
   } = job;
 
   const sdkVersion = sdkVersionFromJob || sdkVersionFromManifest;
-  const workingDir = formatShellAppDirectory(sdkVersion);
+  const workingDir = formatShellAppDirectory({ sdkVersion, buildType: buildType! });
 
   const context: any = {
     appDir: join(config.directories.temporaryFilesRoot, appUUID),
@@ -55,24 +57,63 @@ export function createBuilderContext(job: IJob): IContext {
   context.tempCertPath = join(context.appDir, 'cert.p12');
   context.baseArchiveDir = join(context.appDir, 'archive');
 
-  if (buildType === BUILD_TYPES.SIMULATOR) {
-    context.outputPath = join(context.appDir, 'archive.tar.gz');
-    context.uploadPath = join(context.appDir, 'archive.tar.gz');
-    context.archiveDir = join(
-      context.baseArchiveDir,
-      'Release',
-      `${DEFAULT_EXPOKIT_WORKSPACE_NAME}.app`,
+  if (buildType === BUILD_TYPES.CLIENT) {
+    context.applicationFilesSrc = join(
+      workingDir,
+      'expo-client-build',
+      '**',
+      '*',
     );
-  } else if (buildType === BUILD_TYPES.ARCHIVE) {
+  } else {
+    context.applicationFilesSrc = join(
+      workingDir,
+      'shellAppBase-builds',
+      buildType as string,
+      '**',
+      '*',
+    );
+  }
+
+  if (buildType === BUILD_TYPES.ARCHIVE) {
     context.outputPath = join(context.appDir, 'archive.xcarchive');
     context.uploadPath = join(context.buildDir, 'archive.ipa');
     context.archiveDir = join(
       context.baseArchiveDir,
       'Release',
-      `${DEFAULT_EXPOKIT_WORKSPACE_NAME}.xcarchive`,
+      `${EXPOKIT_APP}.xcarchive`,
       'Products',
       'Applications',
-      `${DEFAULT_EXPOKIT_WORKSPACE_NAME}.app`,
+      `${EXPOKIT_APP}.app`,
+    );
+    context.workspacePath = path.join(
+      workingDir,
+      'shellAppWorkspaces',
+      'ios',
+      'default',
+      `${EXPOKIT_APP}.xcworkspace`,
+    );
+  } else if (buildType === BUILD_TYPES.CLIENT) {
+    context.outputPath = join(context.appDir, 'archive.xcarchive');
+    context.uploadPath = join(context.buildDir, 'archive.ipa');
+    context.archiveDir = join(
+      context.baseArchiveDir,
+      `${EXPONENT_APP}.xcarchive`,
+      'Products',
+      'Applications',
+      `${EXPONENT_APP}.app`,
+    );
+    context.workspacePath = path.join(
+      workingDir,
+      'ios',
+      `${EXPONENT_APP}.xcworkspace`,
+    );
+  } else if (buildType === BUILD_TYPES.SIMULATOR) {
+    context.outputPath = join(context.appDir, 'archive.tar.gz');
+    context.uploadPath = join(context.appDir, 'archive.tar.gz');
+    context.archiveDir = join(
+      context.baseArchiveDir,
+      'Release',
+      `${EXPOKIT_APP}.app`,
     );
   }
 

src/builders/ios/index.ts

@@ -8,6 +8,7 @@ import buildArchive from 'turtle/builders/ios/archive';
 import { createBuilderContext, IContext } from 'turtle/builders/ios/context';
 import buildSimulator from 'turtle/builders/ios/simulator';
 import { logErrorOnce } from 'turtle/builders/utils/common';
+import prepareAdHocBuildCredentials from 'turtle/builders/utils/ios/adhocBuild';
 import { uploadBuildToS3 } from 'turtle/builders/utils/uploader';
 import config from 'turtle/config';
 import { IOS, PLATFORMS } from 'turtle/constants/index';
@@ -18,7 +19,9 @@ const { BUILD_TYPES } = IOS;
 let SUPPORTED_SDK_VERSIONS: number[];
 
 export default async function iosBuilder(job: IJob): Promise<IJobResult> {
-  await ensureCanBuildSdkVersion(job);
+  if (job.config.buildType !== BUILD_TYPES.CLIENT) {
+    await ensureCanBuildSdkVersion(job);
+  }
 
   const ctx = createBuilderContext(job);
 
@@ -26,10 +29,15 @@ export default async function iosBuilder(job: IJob): Promise<IJobResult> {
     await initBuilder(ctx);
 
     const { buildType } = job.config;
-    if (buildType === BUILD_TYPES.SIMULATOR) {
-      await buildSimulator(ctx, job);
-    } else if (buildType === BUILD_TYPES.ARCHIVE) {
+
+    if (buildType === BUILD_TYPES.CLIENT) {
+      await prepareAdHocBuildCredentials(job);
+    }
+
+    if ([BUILD_TYPES.ARCHIVE, BUILD_TYPES.CLIENT].includes(buildType!)) {
       await buildArchive(ctx, job);
+    } else if (buildType === BUILD_TYPES.SIMULATOR) {
+      await buildSimulator(ctx, job);
     } else {
       throw new Error(`Unsupported iOS build type: ${buildType}`);
     }

src/builders/utils/android/credentials.ts

@@ -2,7 +2,7 @@ import path from 'path';
 
 import fs from 'fs-extra';
 import uuidv4 from 'uuid/v4';
-import { AndroidKeystore } from 'xdl';
+import { Credentials } from 'xdl';
 
 import * as sqs from 'turtle/aws/sqs';
 import { UPDATE_CREDENTIALS } from 'turtle/constants/build';
@@ -21,13 +21,15 @@ async function getOrCreateCredentials(jobData: IJob): Promise<IAndroidCredential
     credentials.keystoreAlias = Buffer.from(jobData.experienceName).toString('base64');
     const androidPackage = jobData.manifest.android.package;
 
-    await AndroidKeystore.createKeystore({
-      keystorePassword: credentials.keystorePassword,
-      keyPassword: credentials.keyPassword,
-      keystoreFilename,
-      keystoreAlias: credentials.keystoreAlias,
+    await Credentials.Android.createKeystore(
+      {
+        keystorePath: keystoreFilename,
+        keystorePassword: credentials.keystorePassword,
+        keyPassword: credentials.keyPassword,
+        keyAlias: credentials.keystoreAlias,
+      },
       androidPackage,
-    });
+    );
 
     credentials.keystore = (await fs.readFile(keystoreFilename)).toString('base64');
     l.info('Keystore created successfully');

src/builders/utils/android/workingdir.ts

@@ -2,10 +2,11 @@ import path from 'path';
 
 import { ExponentTools } from 'xdl';
 
+import { IShellAppDirectoryConfig } from 'turtle/builders/utils/workingdir';
 import config from 'turtle/config';
 import { PLATFORMS } from 'turtle/constants';
 
-export function formatShellAppDirectory(sdkVersion: string) {
+export function formatShellAppDirectory({ sdkVersion }: IShellAppDirectoryConfig) {
   const majorSdkVersion = ExponentTools.parseSdkMajorVersion(sdkVersion);
   return config.builder.useLocalWorkingDir
     ? path.join(config.directories.workingDir, 'local')

src/builders/utils/image.ts

@@ -18,15 +18,15 @@ export async function resizeIconWithSharpAsync(iconSizePx: number, iconFilename:
   await fs.writeFile(filename, buffer);
 }
 
-export async function getImageDimensionsWithSharpAsync(basename: string, dirname: string) {
-  const filename = path.join(dirname, basename);
+export async function getImageDimensionsWithSharpAsync(dirname: string, filename: string) {
+  const filepath = path.join(dirname, filename);
 
   // PLEASE DON'T REMOVE THE FOLLOWING LINE (sharp caches files taking path as the cache key)
   sharp.cache(false);
 
   try {
-    const meta = await sharp(filename).metadata();
-    return [meta.width, meta.height];
+    const { width, height } = await sharp(filepath).metadata();
+    return { width, height };
   } catch (e) {
     return null;
   }

src/builders/utils/ios/adhocBuild.ts

@@ -0,0 +1,99 @@
+import path from 'path';
+
+import spawnAsync from '@expo/spawn-async';
+import fs from 'fs-extra';
+import isEmpty from 'lodash/isEmpty';
+import { IosCodeSigning } from 'xdl';
+
+import * as sqs from 'turtle/aws/sqs';
+import BuildError, { BuildErrorReason } from 'turtle/builders/BuildError';
+import { UPDATE_CREDENTIALS } from 'turtle/constants/build';
+import { IJob } from 'turtle/job';
+import logger from 'turtle/logger';
+import { isOffline } from 'turtle/turtleContext';
+
+// tslint:disable-next-line:no-var-requires
+const travelingFastlane = process.platform === 'darwin' ? require('@expo/traveling-fastlane-darwin')() : null;
+
+async function prepareAdHocBuildCredentials(job: IJob) {
+  if (process.platform !== 'darwin') {
+    throw new Error('This function should be called only on macOS!');
+  }
+
+  const { bundleIdentifier } = job.config;
+  const { certP12, certPassword, teamId, appleSession, udids } = job.credentials;
+
+  const certSerialNumber = IosCodeSigning.findP12CertSerialNumber(certP12, certPassword);
+  const args = [
+    teamId,
+    udids!.join(','),
+    bundleIdentifier,
+    certSerialNumber || '__last__',
+  ];
+
+  try {
+    const credentials = await runFastlaneAction(
+      travelingFastlane.manageAdHocProvisioningProfile,
+      args,
+      { env: { FASTLANE_SESSION: appleSession } },
+    );
+    if (isEmpty(credentials)) {
+      if (!job.credentials.provisioningProfile) {
+        // a user has generated the provisioning profile for his app, but he probably deleted it from Expo servers, ...
+        throw new BuildError(
+          BuildErrorReason.PROVISIONING_PROFILE_MISSING,
+          `Provisioning profile is missing on Expo servers.`,
+        );
+      }
+      return;
+    }
+
+    logger.info('New ad hoc provisioning profile successfully created');
+    job.credentials.provisioningProfile = credentials.provisioningProfile;
+
+    if (isOffline()) {
+      const provisioningProfilePath = path.join(job.projectDir, `adhoc.mobileprovision`);
+      await fs.writeFile(provisioningProfilePath, Buffer.from(credentials.provisioningProfile, 'base64'));
+      logger.info(`Saved provisioning profile to ${provisioningProfilePath}`);
+    } else {
+      await sqs.sendMessage(job.id, UPDATE_CREDENTIALS, {
+        provisioningProfileId: credentials.provisioningProfileId,
+        provisioningProfile: credentials.provisioningProfile,
+      });
+      logger.info('Ad Hoc provisioning profile sent to storage successfully');
+    }
+  } catch (e) {
+    if (e instanceof BuildError) {
+      logger.error('Apple Session expired! Please authenticate again using expo-cli.');
+    }
+    throw e;
+  }
+}
+
+async function runFastlaneAction(action: string, args: string[], { env }: any) {
+  const { stderr } = await spawnAsync(
+    action,
+    args,
+    {
+      stdio: ['inherit', 'pipe', 'pipe'],
+      env: { ...process.env, ...env },
+    },
+  );
+  const { result, ...rest } = JSON.parse(stderr.trim());
+  if (result === 'failure') {
+    const { reason, rawDump, type } = rest;
+    let errorMsg = `Reason: ${reason}`;
+    if (rawDump) {
+      errorMsg += `, raw: ${JSON.stringify(rawDump)}`;
+    }
+    if (type === 'session-expired') {
+      throw new BuildError(BuildErrorReason.SESSION_EXPIRED, errorMsg);
+    } else {
+      throw new Error(errorMsg);
+    }
+  } else {
+    return rest;
+  }
+}
+
+export default prepareAdHocBuildCredentials;

src/builders/utils/ios/shellAppBuilder.ts

@@ -8,7 +8,7 @@ import { ImageUtils, IosShellApp } from 'xdl';
 import { IContext } from 'turtle/builders/ios/context';
 import * as commonUtils from 'turtle/builders/utils/common';
 import * as imageHelpers from 'turtle/builders/utils/image';
-import { PLATFORMS } from 'turtle/constants';
+import { IOS } from 'turtle/constants/index';
 import { IJob } from 'turtle/job';
 import logger from 'turtle/logger/index';
 
@@ -18,14 +18,8 @@ export default async function runShellAppBuilder(ctx: IContext, job: IJob): Prom
   const { config: jobConfig, manifest, sdkVersion: sdkVersionFromJob } = job;
   const { buildType, releaseChannel } = jobConfig;
   const sdkVersion = _.get(manifest, 'sdkVersion', sdkVersionFromJob);
-  const applicationFilesSrc = path.join(
-    ctx.workingDir,
-    'shellAppBase-builds',
-    buildType as string,
-    '**',
-    '*',
-  );
-  await copyAsync(applicationFilesSrc, ctx.baseArchiveDir);
+
+  await copyAsync(ctx.applicationFilesSrc, ctx.baseArchiveDir);
 
   logger.info(
     { buildPhase: 'icons setup' },
@@ -34,18 +28,26 @@ export default async function runShellAppBuilder(ctx: IContext, job: IJob): Prom
   ImageUtils.setResizeImageFunction(imageHelpers.resizeIconWithSharpAsync);
   ImageUtils.setGetImageDimensionsFunction(imageHelpers.getImageDimensionsWithSharpAsync);
   const shellAppParams = {
-    url: commonUtils.getExperienceUrl(job),
-    sdkVersion,
     action: 'configure',
     type: buildType,
-    releaseChannel,
     archivePath: ctx.archiveDir,
     privateConfigData: job.config,
-    verbose: true,
-    output: ctx.outputPath,
-    expoSourcePath: path.join(ctx.workingDir, PLATFORMS.IOS),
+    expoSourcePath: path.join(ctx.workingDir, 'ios'),
     manifest,
+    output: ctx.outputPath,
+    verbose: true,
   };
+  if (buildType === IOS.BUILD_TYPES.CLIENT) {
+    Object.assign(shellAppParams, {
+      workspacePath: path.join(ctx.workingDir, 'ios'),
+    });
+  } else {
+    Object.assign(shellAppParams, {
+      url: commonUtils.getExperienceUrl(job),
+      releaseChannel,
+      sdkVersion,
+    });
+  }
 
   logger.info({ buildPhase: 'configuring NSBundle' }, 'configuring NSBundle...');
   return await IosShellApp.configureAndCopyArchiveAsync(shellAppParams);

src/builders/utils/ios/workingdir.ts

@@ -2,12 +2,18 @@ import path from 'path';
 
 import { ExponentTools } from 'xdl';
 
+import { IShellAppDirectoryConfig } from 'turtle/builders/utils/workingdir';
 import config from 'turtle/config';
-import { PLATFORMS } from 'turtle/constants';
+import { IOS, PLATFORMS } from 'turtle/constants';
 
-export function formatShellAppDirectory(sdkVersion: string) {
-  const majorVersion = ExponentTools.parseSdkMajorVersion(sdkVersion);
-  return config.builder.useLocalWorkingDir
-    ? path.join(config.directories.workingDir, 'local')
-    : path.join(config.directories.workingDir, PLATFORMS.IOS, `sdk${majorVersion}`);
+export function formatShellAppDirectory({ sdkVersion, buildType }: IShellAppDirectoryConfig) {
+  const { workingDir } = config.directories;
+  if (buildType === IOS.BUILD_TYPES.CLIENT) {
+    return path.join(workingDir, PLATFORMS.IOS, 'client');
+  } else if (config.builder.useLocalWorkingDir) {
+    return path.join(workingDir, 'local');
+  } else {
+    const majorVersion = ExponentTools.parseSdkMajorVersion(sdkVersion);
+    return path.join(workingDir, PLATFORMS.IOS, `sdk${majorVersion}`);
+  }
 }

src/builders/utils/workingdir.ts

@@ -0,0 +1,6 @@
+import { IOS } from 'turtle/constants';
+
+export interface IShellAppDirectoryConfig {
+  sdkVersion: string;
+  buildType?: IOS.BUILD_TYPES;
+}

src/config.ts

@@ -8,7 +8,7 @@ export default {
   env: env('NODE_ENV'),
   hostname: env('HOSTNAME', os.hostname()),
   deploymentEnv: env('ENVIRONMENT'),
-  platform: env('PLATFORM', PLATFORMS.IOS),
+  platform: env('PLATFORM', PLATFORMS.IOS) as PLATFORMS,
   aws: {
     accessKeyId: env('AWS_ACCESS_KEY_ID'),
     secretAccessKey: env('AWS_SECRET_ACCESS_KEY'),

src/constants/index.ts

@@ -1,9 +1,9 @@
 import * as BUILD from './build';
 import * as IOS from './ios';
 
-export const PLATFORMS = {
-  IOS: 'ios',
-  ANDROID: 'android',
-};
+export enum PLATFORMS {
+  IOS = 'ios',
+  ANDROID = 'android',
+}
 
 export { IOS, BUILD };

src/constants/ios.ts

@@ -1,4 +1,5 @@
-export const BUILD_TYPES = {
-  ARCHIVE: 'archive',
-  SIMULATOR: 'simulator',
-};
+export enum BUILD_TYPES {
+  ARCHIVE = 'archive',
+  CLIENT = 'client',
+  SIMULATOR = 'simulator',
+}

src/job.ts

@@ -1,3 +1,5 @@
+import { IOS } from 'turtle/constants';
+
 export interface IJob {
   platform: 'android' | 'ios';
   experienceName: string;
@@ -8,7 +10,7 @@ export interface IJob {
     releaseChannel: string;
     publicUrl?: string;
     // ios
-    buildType?: string;
+    buildType?: IOS.BUILD_TYPES;
     bundleIdentifier?: string;
   };
   credentials: {
@@ -16,13 +18,14 @@ export interface IJob {
     keystore?: string;
     keystoreAlias?: string;
     keystorePassword?: string;
-    // ios
     keyPassword?: string;
+    // ios
     certP12?: string;
     certPassword?: string;
     provisioningProfile?: string;
-    password?: string;
     teamId?: string;
+    appleSession?: string;
+    udids?: string[];
   };
   projectDir: string;
   fakeUploadDir?: string;

src/jobManager.ts

@@ -2,6 +2,7 @@ import _ from 'lodash';
 
 import * as sqs from 'turtle/aws/sqs';
 import builders from 'turtle/builders';
+import BuildError from 'turtle/builders/BuildError';
 import { logErrorOnce } from 'turtle/builders/utils/common';
 import config from 'turtle/config';
 import { BUILD } from 'turtle/constants/index';
@@ -139,7 +140,15 @@ async function build(job: any) {
   } catch (err) {
     logErrorOnce(err);
     const buildDuration = calculateBuildDuration();
-    sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, { turtleVersion, buildDuration });
+    let reason;
+    if (err instanceof BuildError) {
+      reason = err.reason;
+    }
+    sqs.sendMessage(job.id, BUILD.JOB_STATES.ERRORED, {
+      turtleVersion,
+      buildDuration,
+      ...reason && { reason },
+    });
     return false;
   } finally {
     await logger.cleanup(job);

src/jobsSchemas/ios.ts

@@ -12,11 +12,13 @@ export default baseJobSchema.concat(
     }),
     credentials: Joi.object()
       .keys({
-        certP12: Joi.string(),
-        certPassword: Joi.string(),
-        teamId: Joi.string(),
+        certP12: Joi.string().required(),
+        certPassword: Joi.string().required(),
+        teamId: Joi.string().required(),
+        // it's not required for adhoc Expo Client builds
         provisioningProfile: Joi.string(),
-        password: Joi.string(),
+        appleSession: Joi.string(),
+        udids: Joi.array().items(Joi.string()),
       })
       .when('config.buildType', {
         is: IOS.BUILD_TYPES.SIMULATOR,