[worker] Apply log masking to uploaded Xcode build logs

Pass the job's log masks to the Xcode build log upload and mask the file
line by line while streaming it, including values written in the escaped
form Xcode uses for build settings.

.changeset/worker-mask-xcode-build-logs.md

@@ -0,0 +1,5 @@
+---
+'@expo/worker': patch
+---
+
+Apply the job log's secret masking to uploaded Xcode build logs.

packages/worker/src/__unit__/context.test.ts

@@ -39,6 +39,7 @@ describe(createBuildContext.name, () => {
     buildId: 'build-id',
     buildLogger,
     addBuildLogMask: jest.fn(),
+    createLogMaskingStream: jest.fn(),
   };
 
   beforeEach(() => {

packages/worker/src/__unit__/logger.test.ts

@@ -1,4 +1,5 @@
-import { TransformCallback, Writable } from 'node:stream';
+import { Readable, Transform, TransformCallback, Writable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
 import { EnvironmentSecretType } from '@expo/eas-build-job';
 import z from 'zod';
 
@@ -356,3 +357,99 @@ describe('logger', () => {
     expect(outputStream.readableLength).toBe(0);
   });
 });
+
+describe('createLogMaskingStream', () => {
+  async function maskAsync(stream: Transform, chunks: Buffer[]): Promise<Buffer> {
+    const output: Buffer[] = [];
+    await pipeline(
+      Readable.from(chunks),
+      stream,
+      new Writable({
+        write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
+          output.push(chunk);
+          callback();
+        },
+      })
+    );
+    return Buffer.concat(output);
+  }
+
+  it('masks secrets, secrets as Xcode prints them in build settings, and decoded secrets', async () => {
+    const { createLogMaskingStream } = await createBuildLoggerWithSecretsFilter({
+      environmentSecrets: [
+        { name: 'API_KEY', value: 'ab cd=ef$gh', type: EnvironmentSecretType.STRING },
+        {
+          name: 'ANOTHER_SECRET_BASE64',
+          value: 'YW5vdGhlclNlY3JldA==',
+          type: EnvironmentSecretType.STRING,
+        },
+      ],
+    });
+    const log = [
+      '    export API_KEY\\=ab\\ cd\\=ef\\$gh',
+      'token is ab cd=ef$gh',
+      'decoded anotherSecret here',
+      '',
+    ].join('\n');
+
+    const masked = await maskAsync(createLogMaskingStream(), [Buffer.from(log)]);
+
+    expect(masked.toString()).toBe(
+      [
+        `    export API_KEY\\=${'*'.repeat(14)}`,
+        `token is ${'*'.repeat(11)}`,
+        `decoded ${'*'.repeat(13)} here`,
+        '',
+      ].join('\n')
+    );
+  });
+
+  it('masks secrets split across chunks, including masks added after the logger was created', async () => {
+    const { createLogMaskingStream, addBuildLogMask } = await createBuildLoggerWithSecretsFilter(
+      {}
+    );
+    addBuildLogMask('leased-token');
+
+    const masked = await maskAsync(createLogMaskingStream(), [
+      Buffer.from('first line leased-'),
+      Buffer.from('token\nsecond '),
+      Buffer.from('leased-token without a newline'),
+    ]);
+
+    expect(masked.toString()).toBe(
+      `first line ${'*'.repeat(12)}\nsecond ${'*'.repeat(12)} without a newline`
+    );
+  });
+
+  it('keeps everything except secrets byte for byte, including multibyte and invalid UTF-8', async () => {
+    const { createLogMaskingStream } = await createBuildLoggerWithSecretsFilter({
+      environmentSecrets: [
+        { name: 'PASSWORD', value: 'pässwörd', type: EnvironmentSecretType.STRING },
+      ],
+    });
+    const log = Buffer.concat([
+      Buffer.from('✓ Compiling café.swift\r\n'),
+      Buffer.from([0xff, 0xfe, 0x0a]),
+      Buffer.from('password: pässwörd\n'),
+    ]);
+    // Split inside the multibyte characters.
+    const chunks = [
+      log.subarray(0, 1),
+      log.subarray(1, 20),
+      log.subarray(20, 40),
+      log.subarray(40),
+    ];
+
+    const masked = await maskAsync(createLogMaskingStream(), chunks);
+
+    const passwordBytes = Buffer.byteLength('pässwörd');
+    expect(masked).toEqual(
+      Buffer.concat([
+        Buffer.from('✓ Compiling café.swift\r\n'),
+        Buffer.from([0xff, 0xfe, 0x0a]),
+        Buffer.from(`password: ${'*'.repeat(passwordBytes)}\n`),
+      ])
+    );
+    expect(masked.length).toBe(log.length);
+  });
+});

packages/worker/src/__unit__/service.test.ts

@@ -115,6 +115,7 @@ describe('BuildService Datadog setup', () => {
     createBuildLoggerWithSecretsFilterMock.mockResolvedValue({
       logger: buildLogger,
       addBuildLogMask: jest.fn(),
+      createLogMaskingStream: jest.fn(),
       cleanUp: jest.fn(async () => {}),
       logBuffer: { getLogs: () => [], getPhaseLogs: () => [] } as any,
       outputStream: {} as any,

packages/worker/src/__unit__/xcodeLogs.test.ts

@@ -0,0 +1,82 @@
+import { RemoteLoggerStream, uploadWithSignedUrl } from '@expo/build-tools';
+import fs from 'fs-extra';
+import os from 'os';
+import path from 'path';
+import { Readable } from 'stream';
+import zlib from 'zlib';
+
+import config from '../config';
+import { uploadXcodeBuildLogs } from '../ios/xcodeLogs';
+import { createTextMaskingStream } from '../logger';
+
+// These tests use real fs and directories in os.tmpdir(), since memfs read streams can't be closed.
+jest.unmock('fs');
+jest.unmock('node:fs');
+
+const logger = { warn: jest.fn() } as any;
+
+async function readAllAsync(stream: Readable): Promise<Buffer> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of stream) {
+    chunks.push(Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks);
+}
+
+describe(uploadXcodeBuildLogs, () => {
+  const originalGcs = { ...config.loggers.gcs };
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'xcode-logs-test-'));
+    config.loggers.gcs.signedUploadUrlForXcodeBuildLogs = {
+      url: 'https://gcs.expo.test/xcode-logs',
+      headers: {},
+    };
+  });
+
+  afterEach(async () => {
+    Object.assign(config.loggers.gcs, originalGcs);
+    jest.mocked(uploadWithSignedUrl).mockReset();
+    await fs.remove(tmpDir);
+  });
+
+  it.each([
+    ['no compression', null, (body: Buffer) => body],
+    ['gzip', RemoteLoggerStream.CompressionMethod.GZIP, zlib.gunzipSync],
+    ['brotli', RemoteLoggerStream.CompressionMethod.BR, zlib.brotliDecompressSync],
+  ])('uploads the log with secrets masked (%s)', async (_name, compressionMethod, decompress) => {
+    config.loggers.gcs.compressionMethod = compressionMethod as any;
+    const logsPath = path.join(tmpDir, 'xcodebuild.log');
+    await fs.writeFile(logsPath, '    export API_KEY\\=ab\\ cd\\=ef\\$gh\n** BUILD SUCCEEDED **\n');
+    let uploaded: Buffer | undefined;
+    jest.mocked(uploadWithSignedUrl).mockImplementation(async ({ srcGeneratorAsync }) => {
+      uploaded = await readAllAsync((await srcGeneratorAsync()) as Readable);
+      return 'https://gcs.expo.test/xcode-logs';
+    });
+
+    await uploadXcodeBuildLogs(logger, logsPath, () =>
+      createTextMaskingStream(new Set(['ab cd=ef$gh']))
+    );
+
+    expect(decompress(uploaded!).toString()).toBe(
+      `    export API_KEY\\=${'*'.repeat(14)}\n** BUILD SUCCEEDED **\n`
+    );
+  });
+
+  it.each([
+    ['no compression', null],
+    ['gzip', RemoteLoggerStream.CompressionMethod.GZIP],
+  ])('fails the upload when the log cannot be read (%s)', async (_name, compressionMethod) => {
+    config.loggers.gcs.compressionMethod = compressionMethod as any;
+    jest.mocked(uploadWithSignedUrl).mockImplementation(async ({ srcGeneratorAsync }) => {
+      await readAllAsync((await srcGeneratorAsync()) as Readable);
+      return 'https://gcs.expo.test/xcode-logs';
+    });
+
+    // A directory can be stat'ed but not read.
+    await expect(
+      uploadXcodeBuildLogs(logger, tmpDir, () => createTextMaskingStream(new Set(['secret'])))
+    ).rejects.toThrow('EISDIR');
+  });
+});

packages/worker/src/context.ts

@@ -2,6 +2,7 @@ import { BuildContext, BuildContextOptions, LogBuffer, RuntimeSettings } from '@
 import { Job, ManagedArtifactType, Metadata, Platform } from '@expo/eas-build-job';
 import { bunyan } from '@expo/logger';
 import assert from 'assert';
+import { Transform } from 'stream';
 
 import { GCSCacheManager } from './CacheManager';
 import config from './config';
@@ -27,6 +28,7 @@ export async function createBuildContext<TJob extends Job>({
   buildId,
   buildLogger,
   addBuildLogMask,
+  createLogMaskingStream,
 }: {
   job: TJob;
   logBuffer: LogBuffer;
@@ -36,6 +38,7 @@ export async function createBuildContext<TJob extends Job>({
   buildId: string;
   buildLogger: bunyan;
   addBuildLogMask: (secret: string) => void;
+  createLogMaskingStream: () => Transform;
 }): Promise<BuildContext<TJob>> {
   const childLogger = buildLogger.child({ buildId });
   await RuntimeSettings.loadAsync({
@@ -51,7 +54,7 @@ export async function createBuildContext<TJob extends Job>({
     switch (type) {
       case ManagedArtifactType.XCODE_BUILD_LOGS: {
         if (job.platform === Platform.IOS) {
-          await uploadXcodeBuildLogs(logger, paths[0]);
+          await uploadXcodeBuildLogs(logger, paths[0], createLogMaskingStream);
           return { filename: null };
         }
         throw new Error('Uploading Xcode logs in non-iOS builds is not supported');

packages/worker/src/ios/xcodeLogs.ts

@@ -3,14 +3,18 @@ import { bunyan } from '@expo/logger';
 import fs from 'fs-extra';
 import os from 'os';
 import path from 'path';
-import { Readable, pipeline } from 'stream';
+import { Readable, Transform, pipeline } from 'stream';
 import { promisify } from 'util';
 import { v4 as uuid } from 'uuid';
 import zlib from 'zlib';
 
 import config from '../config';
 
-export async function uploadXcodeBuildLogs(logger: bunyan, logsPath: string): Promise<void> {
+export async function uploadXcodeBuildLogs(
+  logger: bunyan,
+  logsPath: string,
+  createLogMaskingStream: () => Transform
+): Promise<void> {
   if (!config.loggers.gcs.signedUploadUrlForXcodeBuildLogs) {
     logger.warn('GCS Presigned URL for Xcode logs was not specified, skipping upload');
   } else {
@@ -18,8 +22,15 @@ export async function uploadXcodeBuildLogs(logger: bunyan, logsPath: string): Pr
       signedUrl: config.loggers.gcs.signedUploadUrlForXcodeBuildLogs,
       srcGeneratorAsync: async () => {
         const stat = await fs.stat(logsPath);
+        // Masking replaces each byte of a secret with one byte, so the size stays the same.
+        // pipeline passes read errors on to the masked stream, which fails the upload.
+        const maskedStream = pipeline(
+          fs.createReadStream(logsPath),
+          createLogMaskingStream(),
+          () => {}
+        );
         const { stream } = await compress(
-          { stream: fs.createReadStream(logsPath), length: stat.size },
+          { stream: maskedStream, length: stat.size },
           config.loggers.gcs.compressionMethod
         );
         return stream;

packages/worker/src/logger.ts

@@ -102,6 +102,40 @@ function createTransformStream(secretPatterns: ReadonlySet<string>): Transform {
   });
 }
 
+// Characters that Xcode escapes with a backslash when it prints build settings.
+const XCODE_ESCAPED_CHARACTERS = /[\t !"#$&'()*;<=>?[\\\]^`{|}~]/g;
+
+/**
+ * Masks secrets in a text file's contents, line by line, including secrets written the way Xcode
+ * prints build settings. Masking works on bytes, so everything else stays unchanged.
+ */
+export function createTextMaskingStream(secretPatterns: ReadonlySet<string>): Transform {
+  const patterns = [
+    ...new Set(
+      [...secretPatterns].flatMap(secret => [
+        secret,
+        secret.replace(XCODE_ESCAPED_CHARACTERS, '\\$&'),
+      ])
+    ),
+  ].map(pattern => Buffer.from(pattern).toString('latin1'));
+  const maskLine = (line: string): string =>
+    patterns.some(pattern => line.includes(pattern)) ? maskSecrets(line, patterns) : line;
+  let partialLine = '';
+  return new Transform({
+    transform(chunk: Buffer, _encoding: BufferEncoding, callback: TransformCallback) {
+      const lines = (partialLine + chunk.toString('latin1')).split('\n');
+      partialLine = lines.pop() ?? '';
+      callback(
+        null,
+        lines.length > 0 ? Buffer.from(`${lines.map(maskLine).join('\n')}\n`, 'latin1') : undefined
+      );
+    },
+    flush(callback: TransformCallback) {
+      callback(null, partialLine ? Buffer.from(maskLine(partialLine), 'latin1') : undefined);
+    },
+  });
+}
+
 function maskSecrets(message: string, secrets: Iterable<string>): string {
   const characters = message.split('');
   for (const secret of secrets) {
@@ -170,6 +204,7 @@ function createDiscardStream(): Writable {
 export async function createBuildLoggerWithSecretsFilter(secrets: Job['secrets']): Promise<{
   logger: bunyan;
   addBuildLogMask: (secret: string) => void;
+  createLogMaskingStream: () => Transform;
   cleanUp: () => Promise<void>;
   logBuffer: WorkerLogBuffer;
   outputStream: Readable;
@@ -239,6 +274,8 @@ export async function createBuildLoggerWithSecretsFilter(secrets: Job['secrets']
   return {
     logger: buildLogger,
     addBuildLogMask,
+    // Reads the masks when the stream is created, so it includes masks added during the job.
+    createLogMaskingStream: () => createTextMaskingStream(secretPatterns),
     cleanUp: async () => {
       await Promise.all([remoteLoggerStream?.cleanUp?.(), httpLogStream?.cleanUp?.()]);
     },

packages/worker/src/service.ts

@@ -275,6 +275,7 @@ export default class BuildService {
       const {
         logger: buildLogger,
         addBuildLogMask,
+        createLogMaskingStream,
         cleanUp,
         logBuffer,
       } = await createBuildLoggerWithSecretsFilter(job.secrets ?? {});
@@ -315,6 +316,7 @@ export default class BuildService {
         buildId: this.buildId,
         buildLogger,
         addBuildLogMask,
+        createLogMaskingStream,
       });
       this.buildContext = ctx;