libretro/RetroArch
1
0 Fork 0
mirror of https://github.com/libretro/RetroArch.git synced 2026-11-07 08:36:02 -10:00
Code Issues Packages Projects Releases 1 Wiki Activity

camera/avfoundation: cache per-frame scratch buffers; quiet per-frame logs

Browse Source 
captureOutput:didOutputSampleBuffer:fromConnection: allocated and freed
four full-frame scratch buffers on every callback: an intermediate
colour-converted image, a rotated image, an optional mirrored image,
and a scaled image. For a 720p BGRA camera at 30 fps that was roughly
3.5 MB × 4 allocations × 30 Hz of allocator churn on the main thread
(the delegate queue is dispatch_get_main_queue() - see
setSampleBufferDelegate:queue: in setupCameraSession).

Cache the buffers on the AVCameraManager singleton via private ivars
in a class extension. Grow on demand (realloc only when the required
size exceeds the current capacity), otherwise reuse. Camera source
resolution is effectively constant for the life of a session, so
after the first frame no allocator work happens. Scratch buffers are
freed in avfoundation_free via a new teardownScratchBuffers method,
so a driver teardown/re-init with a different target resolution does
not leak stale capacity.

The original code swapped rotatedBuffer.data = mirroredBuffer on
successful mirror; with stable per-frame buffer identities that trick
no longer fits. Replace it with a local 'scaleSource' pointer that
selects between the rotated and mirrored buffer, preserving the same
fallback semantics (mirror failure scales from the rotated buffer).

Also silence two per-frame RARCH_LOG statements (the aspect-fill
scaling dimensions and the 270-degree rotation notice) by putting
them behind #ifdef DEBUG. They duplicate information already logged
once at init and produced one log line per camera frame in release
builds.

ARC / MRC compatibility: the file is compiled under MRC in the top-
level Makefile and the RetroArch*.xcodeproj / RetroArch_OSX107 / iOS13
projects, and under ARC in the iOS6-11 projects and the iOS Theos
build. The new code uses only plain-C pointers for the scratch
buffers (invisible to ARC's memory management), no explicit
retain/release/autorelease calls, and a method name
(teardownScratchBuffers) that avoids the release/init/copy/new/
mutableCopy selector families that ARC treats specially. No
-dealloc override is added - it would require [super dealloc] under
MRC (banned under ARC), and the singleton is never dealloc'd in
practice; teardownScratchBuffers from avfoundation_free is the real
cleanup path.

Thread-safety: unchanged. The capture delegate is invoked on the
main queue and the consumer of frameBuffer reads on the main thread
as well; no new synchronization introduced. The scratch-buffer
realloc happens in the same callback that uses them.
This commit is contained in:
U-DESKTOP-SPFP6AQ\twistedtechre committed 2026-04-23 18:25:46 +02:00
1 parent 93449d3ecc
commit 141d2a6219
1 file changed
+109 -51
Download Patch File Download Diff File Expand all files Collapse all files
camera/drivers/avfoundation.m
+109 -51
View File
@@ -45,6 +45,31 @@
@property (assign) size_t height;
- (bool)setupCameraSession;
- (void)teardownScratchBuffers;
@end
/* Private class extension: cached per-frame scratch buffers.
* captureOutput:didOutputSampleBuffer: used to malloc + free four full-
* frame buffers per callback (intermediate colour-converted, rotated,
* optional mirrored, and scaled). For a typical 720p BGRA camera that
* was ~3.5 MB × 4 allocations × 30 fps = hundreds of MB/s of allocator
* churn on the main thread (the capture delegate queue is the main
* queue — see setSampleBufferDelegate:queue: below). Cache the
* buffers on the manager and grow only when the required size
* exceeds the current capacity; free them in teardownScratchBuffers
* on driver teardown. Ivars are plain C pointers so the file remains
* identically correct under MRC and ARC. */
@interface AVCameraManager ()
{
void *_intermediateBuf;
size_t _intermediateCap;
void *_rotatedBuf;
size_t _rotatedCap;
void *_mirroredBuf;
size_t _mirroredCap;
void *_scaledBuf;
size_t _scaledCap;
}
@end
@implementation AVCameraManager
@@ -123,13 +148,21 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
#ifdef DEBUG
RARCH_LOG("[Camera] Processing frame %zux%zu format: %u.\n", sourceWidth, sourceHeight, (unsigned int)pixelFormat);
#endif
// Create intermediate buffer for full-size converted image
uint32_t *intermediateBuffer = (uint32_t*)malloc(sourceWidth * sourceHeight * 4);
if (!intermediateBuffer) {
RARCH_ERR("[Camera] Failed to allocate intermediate buffer.\n");
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return;
// Intermediate buffer for full-size converted image. Cached on
// the manager - grown only when the required size exceeds the
// current capacity. See class extension above for rationale.
size_t intermediateSize = sourceWidth * sourceHeight * 4;
if (intermediateSize > _intermediateCap) {
void *tmp = realloc(_intermediateBuf, intermediateSize);
if (!tmp) {
RARCH_ERR("[Camera] Failed to allocate intermediate buffer.\n");
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return;
}
_intermediateBuf = tmp;
_intermediateCap = intermediateSize;
}
uint32_t *intermediateBuffer = (uint32_t*)_intermediateBuf;
vImage_Buffer srcBuffer = {}, intermediateVBuffer = {}, dstBuffer = {};
vImage_Error err = kvImageNoError;
@@ -201,14 +234,12 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
default:
RARCH_ERR("[Camera] Unsupported pixel format: %u.\n", (unsigned int)pixelFormat);
free(intermediateBuffer);
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return;
}
if (err != kvImageNoError) {
RARCH_ERR("[Camera] Error converting color format: %ld.\n", err);
free(intermediateBuffer);
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return;
}
@@ -232,20 +263,27 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
// TODO: Add an API to retroarch to allow for mirroring of front camera
shouldMirror = true; // Mirror front camera
#endif
#ifdef DEBUG
RARCH_LOG("[Camera] Using 270-degree rotation with mirroring for front camera in portrait mode.\n");
#endif
}
}
#endif
// Rotate image
vImage_Buffer rotatedBuffer = {};
rotatedBuffer.data = malloc(sourceWidth * sourceHeight * 4);
if (!rotatedBuffer.data) {
RARCH_ERR("[Camera] Failed to allocate rotation buffer.\n");
free(intermediateBuffer);
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return;
// Rotate image (cached scratch buffer)
size_t rotatedSize = sourceWidth * sourceHeight * 4;
if (rotatedSize > _rotatedCap) {
void *tmp = realloc(_rotatedBuf, rotatedSize);
if (!tmp) {
RARCH_ERR("[Camera] Failed to allocate rotation buffer.\n");
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return;
}
_rotatedBuf = tmp;
_rotatedCap = rotatedSize;
}
vImage_Buffer rotatedBuffer = {};
rotatedBuffer.data = _rotatedBuf;
// Set dimensions based on rotation angle
if (rotationDegrees == 90 || rotationDegrees == 270) {
@@ -267,24 +305,30 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
if (err != kvImageNoError) {
RARCH_ERR("[Camera] Error rotating image: %ld.\n", err);
free(rotatedBuffer.data);
free(intermediateBuffer);
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return;
}
// Mirror the image if needed
if (shouldMirror) {
vImage_Buffer mirroredBuffer = {};
mirroredBuffer.data = malloc(rotatedBuffer.height * rotatedBuffer.rowBytes);
if (!mirroredBuffer.data) {
RARCH_ERR("[Camera] Failed to allocate mirror buffer.\n");
free(rotatedBuffer.data);
free(intermediateBuffer);
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return;
}
// Mirror the image if needed. On success, the scale step below
// reads from the mirrored buffer; on failure it falls back to
// rotated. Unlike the original code we don't swap pointers —
// each scratch has a stable identity across frames.
vImage_Buffer *scaleSource = &rotatedBuffer;
vImage_Buffer mirroredBuffer = {};
if (shouldMirror) {
size_t mirroredSize = rotatedBuffer.height * rotatedBuffer.rowBytes;
if (mirroredSize > _mirroredCap) {
void *tmp = realloc(_mirroredBuf, mirroredSize);
if (!tmp) {
RARCH_ERR("[Camera] Failed to allocate mirror buffer.\n");
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return;
}
_mirroredBuf = tmp;
_mirroredCap = mirroredSize;
}
mirroredBuffer.data = _mirroredBuf;
mirroredBuffer.width = rotatedBuffer.width;
mirroredBuffer.height = rotatedBuffer.height;
mirroredBuffer.rowBytes = rotatedBuffer.rowBytes;
@@ -292,17 +336,15 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
err = vImageHorizontalReflect_ARGB8888(&rotatedBuffer, &mirroredBuffer, kvImageNoFlags);
if (err == kvImageNoError) {
// Free rotated buffer and use mirrored buffer for scaling
free(rotatedBuffer.data);
rotatedBuffer = mirroredBuffer;
scaleSource = &mirroredBuffer;
} else {
RARCH_ERR("[Camera] Error mirroring image: %ld.\n", err);
free(mirroredBuffer.data);
/* scaleSource stays pointed at rotatedBuffer */
}
}
// Calculate aspect fill scaling
float sourceAspect = (float)rotatedBuffer.width / rotatedBuffer.height;
float sourceAspect = (float)scaleSource->width / scaleSource->height;
float targetAspect = (float)self.width / self.height;
vImage_Buffer scaledBuffer = {};
@@ -318,30 +360,32 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
scaledHeight = (size_t)(self.width / sourceAspect);
}
#ifdef DEBUG
RARCH_LOG("[Camera] Aspect fill scaling from %zux%zu to %zux%zu.\n",
rotatedBuffer.width, rotatedBuffer.height, scaledWidth, scaledHeight);
scaleSource->width, scaleSource->height, scaledWidth, scaledHeight);
#endif
scaledBuffer.data = malloc(scaledWidth * scaledHeight * 4);
if (!scaledBuffer.data) {
RARCH_ERR("[Camera] Failed to allocate scaled buffer.\n");
free(rotatedBuffer.data);
free(intermediateBuffer);
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return;
size_t scaledSize = scaledWidth * scaledHeight * 4;
if (scaledSize > _scaledCap) {
void *tmp = realloc(_scaledBuf, scaledSize);
if (!tmp) {
RARCH_ERR("[Camera] Failed to allocate scaled buffer.\n");
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return;
}
_scaledBuf = tmp;
_scaledCap = scaledSize;
}
scaledBuffer.data = _scaledBuf;
scaledBuffer.width = scaledWidth;
scaledBuffer.height = scaledHeight;
scaledBuffer.rowBytes = scaledWidth * 4;
// Scale maintaining aspect ratio
err = vImageScale_ARGB8888(&rotatedBuffer, &scaledBuffer, NULL, kvImageHighQualityResampling);
err = vImageScale_ARGB8888(scaleSource, &scaledBuffer, NULL, kvImageHighQualityResampling);
if (err != kvImageNoError) {
RARCH_ERR("[Camera] Error scaling image: %ld.\n", err);
free(scaledBuffer.data);
free(rotatedBuffer.data);
free(intermediateBuffer);
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
return;
}
@@ -360,14 +404,22 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
self.width * 4);
}
// Clean up
free(scaledBuffer.data);
free(rotatedBuffer.data);
free(intermediateBuffer);
/* Scratch buffers retained on the manager; freed on teardown. */
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
} // End of autorelease pool
}
- (void)teardownScratchBuffers {
if (_intermediateBuf) { free(_intermediateBuf); _intermediateBuf = NULL; }
_intermediateCap = 0;
if (_rotatedBuf) { free(_rotatedBuf); _rotatedBuf = NULL; }
_rotatedCap = 0;
if (_mirroredBuf) { free(_mirroredBuf); _mirroredBuf = NULL; }
_mirroredCap = 0;
if (_scaledBuf) { free(_scaledBuf); _scaledBuf = NULL; }
_scaledCap = 0;
}
- (AVCaptureDevice *)selectCameraDevice {
RARCH_LOG("[Camera] Selecting camera device...\n");
@@ -630,6 +682,12 @@ static void avfoundation_free(void *data)
avf->manager.frameBuffer = NULL;
}
/* The manager is a singleton; its scratch buffers from the per-
* frame conversion/rotation/mirror/scale pipeline persist across
* driver instances. Free them here so a subsequent init starts
* clean and stale capacity from a prior session does not linger. */
[avf->manager teardownScratchBuffers];
free(avf);
RARCH_LOG("[Camera] AVFoundation camera freed.\n");
}