[build-tools] upload simulator recording thumbnails

packages/build-tools/resources/record-sim/Package.swift

@@ -17,6 +17,7 @@ let package = Package(
             linkerSettings: [
                 .linkedFramework("AVFoundation"),
                 .linkedFramework("CoreGraphics"),
+                .linkedFramework("CoreImage"),
                 .linkedFramework("CoreMedia"),
                 .linkedFramework("CoreVideo"),
                 .linkedFramework("IOSurface"),

packages/build-tools/resources/record-sim/Sources/RecordSim/RecordingModels.swift

@@ -72,6 +72,7 @@ public struct RecordingManifest: Codable, Sendable {
     public let hlsTargetDurationSeconds: Int?
     public let hlsMediaSequence: Int?
     public let recording: String?
+    public let thumbnail: String?
     public let initSegment: String?
     public let segments: [MediaSegmentRecord]
 }

packages/build-tools/resources/record-sim/Sources/RecordSim/RecordingOutputWriter.swift

@@ -1,4 +1,5 @@
 import AVFoundation
+import CoreImage
 import Foundation
 
 final class RecordingOutputWriter: NSObject, AVAssetWriterDelegate {
@@ -11,6 +12,7 @@ final class RecordingOutputWriter: NSObject, AVAssetWriterDelegate {
     private var initSegmentPath: String?
     private var segments: [MediaSegmentRecord] = []
     private var nextMediaSegmentIndex = 0
+    private var thumbnailPath: String?
     private var firstError: Error?
 
     init(rootDirectory: URL, onSegment: ((SegmentOutput) -> Void)?) {
@@ -35,6 +37,27 @@ final class RecordingOutputWriter: NSObject, AVAssetWriterDelegate {
         stateQueue.sync { firstError }
     }
 
+    func writeThumbnail(pixelBuffer: CVPixelBuffer) throws {
+        let relativePath = "thumbnail.png"
+        let image = CIImage(cvPixelBuffer: pixelBuffer)
+        let longestEdge = max(image.extent.width, image.extent.height)
+        guard longestEdge > 0 else {
+            throw RecorderError.make(52, "Cannot create thumbnail from an empty frame")
+        }
+
+        let scale = min(1, 640 / longestEdge)
+        let thumbnail = image.transformed(by: CGAffineTransform(scaleX: scale, y: scale))
+        try CIContext().writePNGRepresentation(
+            of: thumbnail,
+            to: rootDirectory.appendingPathComponent(relativePath),
+            format: .RGBA8,
+            colorSpace: CGColorSpaceCreateDeviceRGB()
+        )
+        stateQueue.sync {
+            thumbnailPath = relativePath
+        }
+    }
+
     func assetWriter(
         _ writer: AVAssetWriter,
         didOutputSegmentData segmentData: Data,
@@ -129,6 +152,7 @@ final class RecordingOutputWriter: NSObject, AVAssetWriterDelegate {
                 hlsTargetDurationSeconds: targetDuration,
                 hlsMediaSequence: segmented ? 0 : nil,
                 recording: segmented ? nil : "recording.mp4",
+                thumbnail: thumbnailPath,
                 initSegment: segmented ? initSegmentPath : nil,
                 segments: segments
             )

packages/build-tools/resources/record-sim/Sources/RecordSim/SimulatorRecorder.swift

@@ -458,7 +458,7 @@ public final class SimulatorRecorder {
         capturedAt: CMTime,
         capturedAtWallClock: Date
     ) throws {
-        guard let writer, let input, let adaptor else {
+        guard let writer, let input, let adaptor, let outputWriter else {
             throw RecorderError.make(43, "AVAssetWriter is not initialized")
         }
         let isFirstFrame = firstAcceptedCaptureTime == nil
@@ -476,6 +476,7 @@ public final class SimulatorRecorder {
             throw writer.error ?? RecorderError.make(44, "Failed to append pixel buffer")
         }
         if isFirstFrame {
+            try? outputWriter.writeThumbnail(pixelBuffer: pixelBuffer)
             firstAcceptedCaptureTime = capturedAt
             firstAcceptedWallClock = capturedAtWallClock
         }

packages/build-tools/src/steps/functions/__tests__/uploadDeviceRunSessionScreenRecordings.test.ts

@@ -20,23 +20,33 @@ describe(createUploadDeviceRunSessionScreenRecordingsBuildFunction, () => {
     jest
       .mocked(uploadDeviceRunSessionArtifactAsync)
       .mockReset()
-      .mockImplementation(async (_ctx, { stream }) => {
-        await new Promise<void>((resolve, reject) => {
-          stream.once('end', resolve);
-          stream.once('error', reject);
-          stream.resume();
-        });
+      .mockImplementation(async (_ctx, { stream, thumbnail }) => {
+        await Promise.all(
+          [stream, thumbnail?.stream]
+            .filter(value => value !== undefined)
+            .map(
+              streamToConsume =>
+                new Promise<void>((resolve, reject) => {
+                  streamToConsume.once('end', resolve);
+                  streamToConsume.once('error', reject);
+                  streamToConsume.resume();
+                })
+            )
+        );
       });
   });
 
   it('uploads simulator device metadata', async () => {
     const recordingDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'ios-recording-test-'));
     const recordingPath = path.join(recordingDirectory, 'recording.mp4');
+    const thumbnailPath = path.join(recordingDirectory, 'thumbnail.png');
     await fs.writeFile(recordingPath, 'recording');
+    await fs.writeFile(thumbnailPath, 'thumbnail');
     await fs.writeFile(
       path.join(recordingDirectory, 'session.json'),
       JSON.stringify({
         recording: path.basename(recordingPath),
+        thumbnail: path.basename(thumbnailPath),
         firstFrameWallClock: { iso8601: '2026-07-10T10:00:00.000Z' },
         width: 1179,
         height: 2556,
@@ -79,6 +89,10 @@ describe(createUploadDeviceRunSessionScreenRecordingsBuildFunction, () => {
             width: 1179,
             height: 2556,
           },
+          thumbnail: expect.objectContaining({
+            filename: 'thumbnail.png',
+            size: 9,
+          }),
         })
       );
     } finally {

packages/build-tools/src/steps/functions/uploadDeviceRunSessionScreenRecordings.ts

@@ -33,6 +33,7 @@ const RecordingManifestSchema = z.object({
   width: z.number().int().positive(),
   height: z.number().int().positive(),
   recording: z.string(),
+  thumbnail: z.string().optional(),
 });
 
 const recordingStartTimeFormatter = new Intl.DateTimeFormat('en-US', {
@@ -94,6 +95,10 @@ export function createUploadDeviceRunSessionScreenRecordingsBuildFunction(
               const displayName = `${recording.deviceName} screen recording (${shortUdid}, started at ${startedAt})`;
               const recordingPath = path.join(recording.directory, metadata.recording);
               const { size } = await stat(recordingPath);
+              const thumbnailPath = metadata.thumbnail
+                ? path.join(recording.directory, metadata.thumbnail)
+                : null;
+              const thumbnailSize = thumbnailPath ? (await stat(thumbnailPath)).size : null;
               const recordingId = path.basename(recording.directory);
               logger.info(
                 `Uploading screen recording for ${recording.deviceName} (${formatBytes(size)}).`
@@ -116,6 +121,15 @@ export function createUploadDeviceRunSessionScreenRecordingsBuildFunction(
                 },
                 size,
                 stream: createReadStream(recordingPath),
+                ...(thumbnailPath && thumbnailSize !== null
+                  ? {
+                      thumbnail: {
+                        filename: path.basename(thumbnailPath),
+                        size: thumbnailSize,
+                        stream: createReadStream(thumbnailPath),
+                      },
+                    }
+                  : {}),
               });
             } catch (err) {
               const error = err instanceof Error ? err : new Error(String(err));

packages/build-tools/src/steps/utils/__tests__/deviceRunSessionArtifacts.test.ts

@@ -15,6 +15,7 @@ describe(uploadDeviceRunSessionArtifactAsync, () => {
 
   it('streams an artifact through a signed upload URL', async () => {
     const stream = Readable.from(Buffer.from('artifact-data'));
+    const thumbnailStream = Readable.from(Buffer.from('thumbnail-data'));
     const reportedSize = 1024;
     const mutation = jest.fn().mockReturnValue({
       toPromise: async () => ({
@@ -28,6 +29,13 @@ describe(uploadDeviceRunSessionArtifactAsync, () => {
                   'Content-Type': 'application/octet-stream',
                 },
               },
+              thumbnailUploadSession: {
+                url: 'https://uploads.expo.test/thumbnail',
+                headers: {
+                  'Content-Length': '14',
+                  'Content-Type': 'application/octet-stream',
+                },
+              },
             },
           },
         },
@@ -39,7 +47,10 @@ describe(uploadDeviceRunSessionArtifactAsync, () => {
       },
     } as unknown as CustomBuildContext;
 
-    jest.mocked(fetch).mockResolvedValueOnce(new Response('', { status: 200 }));
+    jest
+      .mocked(fetch)
+      .mockResolvedValueOnce(new Response('', { status: 200 }))
+      .mockResolvedValueOnce(new Response('', { status: 200 }));
 
     await uploadDeviceRunSessionArtifactAsync(ctx, {
       deviceRunSessionId: 'drs-id',
@@ -50,6 +61,11 @@ describe(uploadDeviceRunSessionArtifactAsync, () => {
       metadata: { firstFrameRecordAt: 'test-time' },
       size: reportedSize,
       stream,
+      thumbnail: {
+        filename: 'thumbnail.png',
+        size: 14,
+        stream: thumbnailStream,
+      },
     });
 
     expect(mutation).toHaveBeenCalledWith(
@@ -62,6 +78,10 @@ describe(uploadDeviceRunSessionArtifactAsync, () => {
           kind: 'agent-device-test-report',
           metadata: { firstFrameRecordAt: 'test-time' },
           size: reportedSize,
+          thumbnail: {
+            filename: 'thumbnail.png',
+            size: 14,
+          },
         },
       })
     );
@@ -72,5 +92,12 @@ describe(uploadDeviceRunSessionArtifactAsync, () => {
         body: stream,
       })
     );
+    expect(jest.mocked(fetch)).toHaveBeenCalledWith(
+      'https://uploads.expo.test/thumbnail',
+      expect.objectContaining({
+        method: 'PUT',
+        body: thumbnailStream,
+      })
+    );
   });
 });

packages/build-tools/src/steps/utils/deviceRunSessionArtifacts.ts

@@ -1,10 +1,41 @@
 import { SystemError } from '@expo/eas-build-job';
-import { graphql } from 'gql.tada';
+import { TypedDocumentNode, gql } from '@urql/core';
 import fetch, { Headers } from 'node-fetch';
 
 import { CustomBuildContext } from '../../customBuildContext';
 
-const CREATE_DEVICE_RUN_SESSION_ARTIFACT_UPLOAD_SESSION_MUTATION = graphql(`
+type ArtifactUploadSession = {
+  url: string;
+  headers: Record<string, string>;
+};
+
+type CreateArtifactUploadSessionData = {
+  deviceRunSession: {
+    createArtifactUploadSession: {
+      uploadSession: ArtifactUploadSession;
+      thumbnailUploadSession: ArtifactUploadSession | null;
+    };
+  };
+};
+
+type CreateArtifactUploadSessionVariables = {
+  deviceRunSessionId: string;
+  input: {
+    name: string;
+    filename: string;
+    kind?: string;
+    metadata?: Record<string, unknown>;
+    size: number;
+    thumbnail?: {
+      filename: string;
+      size: number;
+    };
+  };
+};
+
+// This uses an explicitly typed document so build-tools can roll out after the API change without
+// requiring the new schema to already be deployed while installing dependencies.
+const CREATE_DEVICE_RUN_SESSION_ARTIFACT_UPLOAD_SESSION_MUTATION = gql`
   mutation CreateDeviceRunSessionArtifactUploadSession(
     $deviceRunSessionId: ID!
     $input: CreateDeviceRunSessionArtifactUploadSessionInput!
@@ -15,10 +46,14 @@ const CREATE_DEVICE_RUN_SESSION_ARTIFACT_UPLOAD_SESSION_MUTATION = graphql(`
           url
           headers
         }
+        thumbnailUploadSession {
+          url
+          headers
+        }
       }
     }
   }
-`);
+` as TypedDocumentNode<CreateArtifactUploadSessionData, CreateArtifactUploadSessionVariables>;
 
 export async function uploadDeviceRunSessionArtifactAsync(
   ctx: CustomBuildContext,
@@ -31,6 +66,7 @@ export async function uploadDeviceRunSessionArtifactAsync(
     metadata,
     size,
     stream,
+    thumbnail,
   }: {
     deviceRunSessionId: string;
     artifactId: string;
@@ -40,17 +76,43 @@ export async function uploadDeviceRunSessionArtifactAsync(
     metadata?: Record<string, unknown>;
     size: number;
     stream: NodeJS.ReadableStream;
+    thumbnail?: {
+      filename: string;
+      size: number;
+      stream: NodeJS.ReadableStream;
+    };
   }
 ): Promise<void> {
-  const uploadSession = await createDeviceRunSessionArtifactUploadSessionAsync(ctx, {
-    deviceRunSessionId,
-    artifactId,
-    name,
-    filename,
-    kind,
-    metadata,
-    size,
-  });
+  const { uploadSession, thumbnailUploadSession } =
+    await createDeviceRunSessionArtifactUploadSessionAsync(ctx, {
+      deviceRunSessionId,
+      artifactId,
+      name,
+      filename,
+      kind,
+      metadata,
+      size,
+      thumbnail,
+    });
+  if (thumbnail && !thumbnailUploadSession) {
+    throw new SystemError(
+      `Failed to create thumbnail upload session for device run session artifact ${artifactId}`
+    );
+  }
+
+  await Promise.all([
+    uploadStreamAsync(uploadSession, stream, artifactId),
+    ...(thumbnail && thumbnailUploadSession
+      ? [uploadStreamAsync(thumbnailUploadSession, thumbnail.stream, `${artifactId} thumbnail`)]
+      : []),
+  ]);
+}
+
+async function uploadStreamAsync(
+  uploadSession: ArtifactUploadSession,
+  stream: NodeJS.ReadableStream,
+  artifactId: string
+): Promise<void> {
   const response = await fetch(uploadSession.url, {
     method: 'PUT',
     headers: new Headers(uploadSession.headers as Record<string, string>),
@@ -74,6 +136,7 @@ async function createDeviceRunSessionArtifactUploadSessionAsync(
     kind,
     metadata,
     size,
+    thumbnail,
   }: {
     deviceRunSessionId: string;
     artifactId: string;
@@ -82,6 +145,10 @@ async function createDeviceRunSessionArtifactUploadSessionAsync(
     kind: string | undefined;
     metadata?: Record<string, unknown>;
     size: number;
+    thumbnail?: {
+      filename: string;
+      size: number;
+    };
   }
 ) {
   const result = await ctx.graphqlClient
@@ -93,6 +160,9 @@ async function createDeviceRunSessionArtifactUploadSessionAsync(
         ...(kind !== undefined ? { kind } : {}),
         ...(metadata !== undefined ? { metadata } : {}),
         size,
+        ...(thumbnail !== undefined
+          ? { thumbnail: { filename: thumbnail.filename, size: thumbnail.size } }
+          : {}),
       },
     })
     .toPromise();
@@ -102,5 +172,5 @@ async function createDeviceRunSessionArtifactUploadSessionAsync(
       { cause: result.error }
     );
   }
-  return result.data!.deviceRunSession.createArtifactUploadSession.uploadSession;
+  return result.data!.deviceRunSession.createArtifactUploadSession;
 }