Files
RetroArch/camera/drivers/ffmpeg.c
Codex 53280a4f5a camera: honor FFmpeg frame color range and matrix
Configure the cached scaler from each decoded frame's color metadata, with decoder metadata fallback and format defaults for unlabelled RGB, gray and YUVJ. Keep direct producer-buffer conversion. Cover full-to-limited range and BT.709-to-BT.601 transitions, plus unlabelled grayscale, checking every scaled output pixel.
2026-10-14 18:06:48 +02:00

956 lines
31 KiB
C

/* RetroArch - A frontend for libretro.
* Copyright (C) 2010-2023 - Hans-Kristian Arntzen
* Copyright (C) 2023 - Jesse Talavera-Greenberg
*
* RetroArch is free software: you can redistribute it and/or modify it under the terms
* of the GNU General Public License as published by the Free Software Found-
* ation, either version 3 of the License, or (at your option) any later version.
*
* RetroArch is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with RetroArch.
* If not, see <http://www.gnu.org/licenses/>.
*/
#include <libavformat/version.h>
#if LIBAVFORMAT_VERSION_MAJOR >= 58
#include <libretro.h>
#include <memalign.h>
#include <retro_common_api.h>
#include <retro_assert.h>
#include <rthreads/rthreads.h>
#include <rthreads/retro_eventcount.h>
#include <retro_atomic.h>
#include <queues/retro_triple_buffer.h>
#include <lists/string_list.h>
#include <string/stdstring.h>
#ifdef __cplusplus
extern "C" {
#endif
#include <libavdevice/avdevice.h>
#include <libavcodec/avcodec.h>
#include <libavutil/imgutils.h>
#include <libavutil/pixdesc.h>
#include <libswscale/swscale.h>
#ifdef __cplusplus
}
#endif
#include "../camera_driver.h"
#include "../../configuration.h"
#include "../../verbosity.h"
/* The libavdevice input format the camera comes from. A build may name
* it already - a test harness points it at lavfi, which gives the
* whole pipeline a synthetic camera and needs no device. */
#ifndef FFMPEG_CAMERA_DEFAULT_BACKEND
#ifdef ANDROID
#define FFMPEG_CAMERA_DEFAULT_BACKEND "android_camera"
#elif defined(__linux__)
#define FFMPEG_CAMERA_DEFAULT_BACKEND "v4l2"
#elif defined(__APPLE__)
#define FFMPEG_CAMERA_DEFAULT_BACKEND "avfoundation"
#elif defined(__WIN32__)
#define FFMPEG_CAMERA_DEFAULT_BACKEND "dshow"
#elif defined(__FreeBSD__) || defined(__OpenBSD__) || defined (__NetBSD__)
#define FFMPEG_CAMERA_DEFAULT_BACKEND "bktr"
#else
#define FFMPEG_CAMERA_DEFAULT_BACKEND "lavfi"
#endif
#endif
/* lavf 59 (FFmpeg 5.0) made the demuxer/codec discovery API const-correct:
* av_find_input_format() returns const AVInputFormat*, avformat_open_input()
* accepts one, and av_find_best_stream() takes const AVCodec**. Older
* versions use mutable pointers throughout, so a single const-qualified
* declaration cannot satisfy both. */
#if LIBAVFORMAT_VERSION_MAJOR >= 59
typedef const AVInputFormat ffmpeg_camera_input_format_t;
typedef const AVCodec ffmpeg_camera_codec_t;
#else
typedef AVInputFormat ffmpeg_camera_input_format_t;
typedef AVCodec ffmpeg_camera_codec_t;
#endif
typedef struct ffmpeg_camera
{
sthread_t *poll_thread;
AVFormatContext *format_context;
AVCodecContext *decoder_context;
ffmpeg_camera_codec_t *decoder; /* owned by ffmpeg, don't free it */
ffmpeg_camera_input_format_t *input_format; /* owned by ffmpeg, don't free it */
AVDictionary *options;
AVPacket *packet;
AVFrame *camera_frame;
int video_stream_index;
unsigned requested_dims; /* VIDEO_SCALE_PACK */
uint8_t *target_planes[4];
int target_linesizes[4];
unsigned target_dims; /* VIDEO_SCALE_PACK */
struct SwsContext *scale_context;
/* "name" for the camera device.
* Not just the reported name, there may be a bit of extra syntax.
* See https://ffmpeg.org/ffmpeg-devices.html#Input-Devices for details. */
char url[512];
/* Converted frames, from the poll thread to the core's poll: the
* thread fills one, the core reads the newest finished one, and
* neither waits for the other. */
uint8_t *target_buffers[3];
retro_triple_buffer_t frames;
size_t target_buffer_length;
/* The poll thread's loop condition: the main thread sets it in
* stop(), the thread reads it every turn. */
retro_atomic_int_t done;
retro_atomic_int_t poll_epoch;
retro_eventcount_t read_retry;
bool read_retry_init;
} ffmpeg_camera_t;
static void ffmpeg_camera_free(void *data);
static int ffmpeg_camera_get_initial_options(
const AVInputFormat *backend,
AVDictionary **options,
uint64_t caps,
unsigned dims
)
{
int ret = 0;
#ifdef __APPLE__
/* For AVFoundation, we need to handle options differently
* AVFoundation expects these options to be passed to avformat_open_input.
* We'll set them later in ffmpeg_camera_open_device */
if (backend && string_is_equal(backend->name, "avfoundation"))
return 0;
#endif
/* If the core is letting the frontend pick the size... */
if (VIDEO_SCALE_W(dims) != 0 && VIDEO_SCALE_H(dims) != 0)
{
char dimensions[128];
snprintf(dimensions, sizeof(dimensions), "%ux%u",
VIDEO_SCALE_W(dims), VIDEO_SCALE_H(dims));
ret = av_dict_set(options, "video_size", dimensions, 0);
if (ret < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to set option: %s.\n", msg);
av_dict_free(options);
return ret;
}
}
/* I wanted to list supported formats and pick the most appropriate size
* if the requested size isn't available,
* but ffmpeg doesn't seem to offer a way to do that.
*/
if (!options)
{
RARCH_DBG("[FFMPEG] No options set, not allocating a dict (this isn't an error).");
}
return ret;
}
/* Device URL syntax varies by backend.
* See https://ffmpeg.org/ffmpeg-devices.html for details. */
static void ffmpeg_camera_get_source_url(ffmpeg_camera_t *ffmpeg, const AVDeviceInfo *device)
{
#ifdef __WIN32__
if (string_is_equal(ffmpeg->input_format->name, "dshow"))
{
snprintf(ffmpeg->url, sizeof(ffmpeg->url), "video=%s", device->device_description);
return;
}
#elif defined(__APPLE__)
if (string_is_equal(ffmpeg->input_format->name, "avfoundation"))
{
/* we only want video, not audio */
/* Use "0:none" for the first video device, no audio */
snprintf(ffmpeg->url, sizeof(ffmpeg->url), "0:none");
return;
}
#endif
/* Other backends that support listing available sources use the name as-is;
* some advanced backends have extra syntax (e.g. gdigrab)
* but they don't support listing available sources,
* so players will have to enter them manually.
*/
strlcpy(ffmpeg->url, device->device_description, sizeof(ffmpeg->url));
}
static int ffmpeg_camera_interrupt(void *opaque)
{
ffmpeg_camera_t *ffmpeg = (ffmpeg_camera_t*)opaque;
return retro_atomic_load_acquire_int(&ffmpeg->done);
}
static int ffmpeg_camera_open_input(ffmpeg_camera_t *ffmpeg, AVDictionary **options)
{
ffmpeg->format_context = avformat_alloc_context();
if (!ffmpeg->format_context)
return AVERROR(ENOMEM);
ffmpeg->format_context->interrupt_callback.callback = ffmpeg_camera_interrupt;
ffmpeg->format_context->interrupt_callback.opaque = ffmpeg;
return avformat_open_input(&ffmpeg->format_context, ffmpeg->url,
ffmpeg->input_format, options);
}
static int ffmpeg_camera_open_device(ffmpeg_camera_t *ffmpeg)
{
AVDictionaryEntry *e = NULL;
AVDictionary *options = NULL;
int ret = ffmpeg->options ? av_dict_copy(&options, ffmpeg->options, 0) : 0;
/* copy the options dict so that other steps in start() can use it,
* as avformat_open_input clears it and adds unrecognized settings */
if (ret < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to copy options: %s.\n", msg);
goto done;
}
#ifdef __APPLE__
/* For AVFoundation, set the options that the device expects */
if (ffmpeg->input_format && string_is_equal(ffmpeg->input_format->name, "avfoundation"))
{
char dimensions[128];
/* Set video size if requested */
if ( VIDEO_SCALE_W(ffmpeg->requested_dims) != 0
&& VIDEO_SCALE_H(ffmpeg->requested_dims) != 0)
{
/* Use a resolution that the device likely supports */
/* Common resolutions: 640x480, 1280x720, 1920x1080 */
unsigned width = VIDEO_SCALE_W(ffmpeg->requested_dims);
unsigned height = VIDEO_SCALE_H(ffmpeg->requested_dims);
/* If the requested resolution is too small, use a minimum supported size */
if (width < 640 || height < 480)
{
width = 640;
height = 480;
RARCH_LOG("[FFMPEG] Requested resolution %ux%u too small, using %ux%u instead.\n",
VIDEO_SCALE_W(ffmpeg->requested_dims),
VIDEO_SCALE_H(ffmpeg->requested_dims), width, height);
}
snprintf(dimensions, sizeof(dimensions), "%ux%u", width, height);
av_dict_set(&options, "video_size", dimensions, 0);
}
/* Set framerate */
av_dict_set(&options, "framerate", "30", 0);
/* Set pixel format to a widely supported format */
av_dict_set(&options, "pixel_format", "uyvy422", 0);
}
#endif
ret = ffmpeg_camera_open_input(ffmpeg, &options);
if (ret < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_WARN("[FFMPEG] Failed to open video input device \"%s\": %s.\n", ffmpeg->url, msg);
#ifdef __APPLE__
/* For AVFoundation, try with default settings if custom settings fail */
if (ffmpeg->input_format && string_is_equal(ffmpeg->input_format->name, "avfoundation"))
{
RARCH_LOG("[FFMPEG] Trying with default AVFoundation settings...\n");
av_dict_free(&options);
/* Try with just the basic framerate option */
av_dict_set(&options, "framerate", "30", 0);
ret = ffmpeg_camera_open_input(ffmpeg, &options);
if (ret < 0)
{
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_WARN("[FFMPEG] Failed with basic options, trying with no options: %s.\n", msg);
av_dict_free(&options);
/* Last resort: try with no options at all */
ret = ffmpeg_camera_open_input(ffmpeg, NULL);
if (ret < 0)
{
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to open device even with default settings: %s.\n", msg);
goto done;
}
RARCH_LOG("[FFMPEG] Successfully opened device with default settings.\n");
}
else
{
RARCH_LOG("[FFMPEG] Successfully opened device with basic settings.\n");
}
}
else
#endif
/* If we're not already requesting the default format... */
if (ffmpeg->options)
{
ret = ffmpeg_camera_open_input(ffmpeg, NULL);
if (ret < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to open the same device in its default format: %s.\n", msg);
goto done;
}
}
}
done:
if (options)
{
e = NULL;
while ((e = av_dict_get(options, "", e, AV_DICT_IGNORE_SUFFIX)))
{
/* av_dict_iterate isn't always available, so we use av_dict_get's legacy behavior instead */
RARCH_WARN("[FFMPEG] Unrecognized option on video input device: %s=%s.\n", e->key, e->value);
}
}
av_dict_free(&options); /* noop if NULL */
if (ret == 0)
{
RARCH_LOG("[FFMPEG] Opened video input device \"%s\".\n", ffmpeg->url);
}
return ret;
}
static void *ffmpeg_camera_init(const char *device, uint64_t caps, unsigned dims)
{
ffmpeg_camera_t *ffmpeg = NULL;
AVDeviceInfoList *device_list = NULL;
int ret = 0;
int num_sources = 0;
/* If the core didn't ask for raw framebuffers... */
if ((caps & (UINT64_C(1) << RETRO_CAMERA_BUFFER_RAW_FRAMEBUFFER)) == 0)
{
RARCH_ERR("[FFMPEG] Camera driver only supports raw framebuffer output.\n");
return NULL;
}
ffmpeg = (ffmpeg_camera_t*)calloc(1, sizeof(*ffmpeg));
if (!ffmpeg)
{
RARCH_ERR("[FFMPEG] Failed to allocate memory for camera driver.\n");
return NULL;
}
ffmpeg->requested_dims = dims;
ffmpeg->read_retry_init = retro_eventcount_init(&ffmpeg->read_retry);
if (!ffmpeg->read_retry_init)
{
RARCH_ERR("[FFMPEG] Failed to initialize capture wakeup.\n");
goto error;
}
avdevice_register_all();
RARCH_LOG("[FFMPEG] Initialized libavdevice.\n");
ffmpeg->input_format = av_find_input_format(FFMPEG_CAMERA_DEFAULT_BACKEND);
if (!ffmpeg->input_format)
{
RARCH_ERR("[FFMPEG] No suitable video input backend found.\n");
goto error;
}
RARCH_LOG("[FFMPEG] Using camera backend: %s (%s, flags=0x%x).\n", ffmpeg->input_format->name, ffmpeg->input_format->long_name, ffmpeg->input_format->flags);
ret = ffmpeg_camera_get_initial_options(ffmpeg->input_format, &ffmpeg->options, caps, dims);
if (ret < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to get initial options: %s.\n", msg);
goto error;
}
/* A device the frontend named is the device, and enumeration is
* skipped: what the setting says is a source url for the backend
* in use, which is the only thing this driver could do with it.
* It used to be ignored outright - the argument was taken and
* never read, so a user who picked a camera got whichever one
* enumerated first. */
if (!string_is_empty(device))
{
strlcpy(ffmpeg->url, device, sizeof(ffmpeg->url));
RARCH_LOG("[FFMPEG] Using the device the frontend asked for: %s.\n",
ffmpeg->url);
goto have_device;
}
num_sources = avdevice_list_input_sources(ffmpeg->input_format, NULL, ffmpeg->options, &device_list);
#ifdef __APPLE__
/* AVFoundation device listing may fail or return -38 (function not implemented) */
/* but we can still use the device with index 0 */
if (num_sources < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, num_sources);
RARCH_WARN("[FFMPEG] Device enumeration not fully supported on AVFoundation: %s. Using default device.\n", msg);
/* Continue with default device */
ffmpeg_camera_get_source_url(ffmpeg, NULL);
}
else if (num_sources == 0)
{
RARCH_ERR("[FFMPEG] No video input sources found.\n");
goto error;
}
else
{
ffmpeg_camera_get_source_url(ffmpeg, device_list->devices[0]);
}
#else
if (num_sources == 0)
{
RARCH_ERR("[FFMPEG] No video input sources found.\n");
goto error;
}
if (num_sources < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, num_sources);
RARCH_ERR("[FFMPEG] Failed to list video input sources: %s.\n", msg);
goto error;
}
ffmpeg_camera_get_source_url(ffmpeg, device_list->devices[0]);
#endif
have_device:
RARCH_LOG("[FFMPEG] Using video input device: %s (%s, flags=0x%x).\n", ffmpeg->input_format->name, ffmpeg->input_format->long_name, ffmpeg->input_format->flags);
avdevice_free_list_devices(&device_list);
return ffmpeg;
error:
avdevice_free_list_devices(&device_list);
ffmpeg_camera_free(ffmpeg);
return NULL;
}
static void ffmpeg_camera_stop(void *data);
static void ffmpeg_camera_free(void *data)
{
ffmpeg_camera_t *ffmpeg = (ffmpeg_camera_t*)data;
if (!ffmpeg)
return;
ffmpeg_camera_stop(ffmpeg);
av_dict_free(&ffmpeg->options);
retro_eventcount_free(&ffmpeg->read_retry);
free(ffmpeg);
}
static void ffmpeg_camera_poll_thread(void *data);
static bool ffmpeg_camera_start(void *data)
{
ffmpeg_camera_t *ffmpeg = (ffmpeg_camera_t*)data;
int ret = 0;
AVStream *stream = NULL;
AVDictionary *options = NULL;
const AVDictionaryEntry *e = NULL;
int target_buffer_length = 0;
unsigned i;
/* TODO: Check the actual format context, not just the pointer */
if (ffmpeg->format_context)
{
RARCH_LOG("[FFMPEG] Camera %s is already started, no action needed.\n", ffmpeg->format_context->url);
return true;
}
retro_atomic_store_release_int(&ffmpeg->done, 0);
ret = ffmpeg_camera_open_device(ffmpeg);
if (ret < 0)
goto error;
ret = av_dict_copy(&options, ffmpeg->options, 0);
if (ret < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to copy options: %s.\n", msg);
goto error;
}
ret = avformat_find_stream_info(ffmpeg->format_context, NULL);
if (ret < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to find stream info: %s.\n", msg);
goto error;
}
while ((e = av_dict_get(options, "", e, AV_DICT_IGNORE_SUFFIX)))
{
RARCH_WARN("[FFMPEG] Unrecognized option on video input device: %s=%s.\n", e->key, e->value);
}
ret = av_find_best_stream(ffmpeg->format_context,
AVMEDIA_TYPE_VIDEO, -1, -1, &ffmpeg->decoder, 0);
if (ret < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to find video stream: %s.\n", msg);
goto error;
}
stream = ffmpeg->format_context->streams[ret];
ffmpeg->video_stream_index = stream->index;
RARCH_LOG("[FFMPEG] Using video stream #%d with decoder \"%s\" (%s).\n", ret, ffmpeg->decoder->name, ffmpeg->decoder->long_name);
ffmpeg->decoder_context = avcodec_alloc_context3(ffmpeg->decoder);
if (!ffmpeg->decoder_context)
{
RARCH_ERR("[FFMPEG] Failed to allocate decoder context.\n");
goto error;
}
ret = avcodec_parameters_to_context(ffmpeg->decoder_context, stream->codecpar);
if (ret < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to copy codec parameters to decoder context: %s.\n", msg);
goto error;
}
if (!sws_isSupportedInput(ffmpeg->decoder_context->pix_fmt))
{
RARCH_ERR("[FFMPEG] Unsupported scaler input pixel format: %s.\n", av_get_pix_fmt_name(ffmpeg->decoder_context->pix_fmt));
goto error;
}
ret = av_dict_copy(&ffmpeg->options, options, 0);
if (ret < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to copy options: %s.\n", msg);
goto error;
}
ret = avcodec_open2(ffmpeg->decoder_context, ffmpeg->decoder, NULL);
if (ret < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to open decoder: %s.\n", msg);
goto error;
}
while ((e = av_dict_get(options, "", e, AV_DICT_IGNORE_SUFFIX)))
{
RARCH_WARN("[FFMPEG] Unrecognized option on video input device: %s=%s.\n", e->key, e->value);
}
av_dict_free(&options);
ffmpeg->packet = av_packet_alloc();
if (!ffmpeg->packet)
{
RARCH_ERR("[FFMPEG] Failed to allocate packet.\n");
goto error;
}
ffmpeg->camera_frame = av_frame_alloc();
if (!ffmpeg->camera_frame)
{
RARCH_ERR("[FFMPEG] Failed to allocate camera frame.\n");
goto error;
}
ffmpeg->target_dims = VIDEO_SCALE_PACK(
VIDEO_SCALE_W(ffmpeg->requested_dims)
? VIDEO_SCALE_W(ffmpeg->requested_dims)
: (unsigned)ffmpeg->decoder_context->width,
VIDEO_SCALE_H(ffmpeg->requested_dims)
? VIDEO_SCALE_H(ffmpeg->requested_dims)
: (unsigned)ffmpeg->decoder_context->height);
target_buffer_length = av_image_get_buffer_size(
AV_PIX_FMT_BGRA,
VIDEO_SCALE_W(ffmpeg->target_dims),
VIDEO_SCALE_H(ffmpeg->target_dims),
1
);
if (target_buffer_length < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, target_buffer_length);
RARCH_ERR("[FFMPEG] Failed to calculate target size: %s.\n", msg);
goto error;
}
/* Aligned and padded for swscale's direct output. */
ffmpeg->target_buffer_length = target_buffer_length;
ffmpeg->target_linesizes[0] = VIDEO_SCALE_W(ffmpeg->target_dims) * 4;
for (i = 0; i < 3; i++)
if ((ffmpeg->target_buffers[i] = (uint8_t*)memalign_alloc(32,
(size_t)target_buffer_length + AV_INPUT_BUFFER_PADDING_SIZE)))
/* black until the first frame is in */
memset(ffmpeg->target_buffers[i], 0,
(size_t)target_buffer_length + AV_INPUT_BUFFER_PADDING_SIZE);
if (!ffmpeg->target_buffers[0] || !ffmpeg->target_buffers[1] || !ffmpeg->target_buffers[2])
{
RARCH_ERR("[FFMPEG] Failed to allocate target %d-byte buffer for %dx%d %s-formatted video data.\n",
target_buffer_length,
ffmpeg->decoder_context->width,
ffmpeg->decoder_context->height,
av_get_pix_fmt_name(AV_PIX_FMT_BGRA)
);
goto error;
}
RARCH_LOG("[FFMPEG] Allocated %d bytes for %dx%d %s-formatted video data.\n",
target_buffer_length,
ffmpeg->decoder_context->width,
ffmpeg->decoder_context->height,
av_get_pix_fmt_name(ffmpeg->decoder_context->pix_fmt)
);
ffmpeg->scale_context = sws_getContext(
ffmpeg->decoder_context->width,
ffmpeg->decoder_context->height,
ffmpeg->decoder_context->pix_fmt,
VIDEO_SCALE_W(ffmpeg->target_dims),
VIDEO_SCALE_H(ffmpeg->target_dims),
AV_PIX_FMT_BGRA,
SWS_BILINEAR,
NULL, NULL, NULL
);
if (!ffmpeg->scale_context)
{
RARCH_ERR("[FFMPEG] Failed to create scale context.\n");
goto error;
}
retro_triple_buffer_init(&ffmpeg->frames, ffmpeg->target_buffers[0],
ffmpeg->target_buffers[1], ffmpeg->target_buffers[2]);
ffmpeg->poll_thread = sthread_create(ffmpeg_camera_poll_thread, ffmpeg);
if (!ffmpeg->poll_thread)
{
RARCH_ERR("[FFMPEG] Failed to create poll thread.\n");
goto error;
}
return true;
error:
av_dict_free(&options);
ffmpeg_camera_stop(ffmpeg);
return false;
}
static void ffmpeg_camera_stop(void *data)
{
ffmpeg_camera_t *ffmpeg = (ffmpeg_camera_t*)data;
unsigned i;
/* The thread first. It is the other user of the decoder - it sends
* every packet it reads and receives every frame - and an
* AVCodecContext takes one user at a time, so the flush below
* cannot happen while the thread is still in there. This used to
* flush and then join. */
retro_atomic_store_release_int(&ffmpeg->done, 1);
if (ffmpeg->read_retry_init)
retro_eventcount_notify(&ffmpeg->read_retry);
sthread_join(ffmpeg->poll_thread); /* wait for the thread to finish, then free it */
if (!ffmpeg->format_context)
{
RARCH_LOG("[FFMPEG] Camera %s is already stopped, no flush needed.\n", ffmpeg->url);
}
else if (ffmpeg->decoder_context)
{
int ret = avcodec_send_packet(ffmpeg->decoder_context, NULL);
if (ret < 0 && ret != AVERROR_EOF)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to flush decoder: %s.\n", msg);
}
}
/* these functions are noops for NULL pointers */
ffmpeg->poll_thread = NULL;
sws_freeContext(ffmpeg->scale_context);
ffmpeg->scale_context = NULL;
for (i = 0; i < 3; i++)
{
memalign_free(ffmpeg->target_buffers[i]);
ffmpeg->target_buffers[i] = NULL;
}
/* a poll between stop and start is handed no frame */
retro_triple_buffer_init(&ffmpeg->frames, NULL, NULL, NULL);
memset(ffmpeg->target_planes, 0, sizeof(ffmpeg->target_planes));
ffmpeg->target_buffer_length = 0;
ffmpeg->target_dims = 0;
av_frame_free(&ffmpeg->camera_frame);
memset(ffmpeg->target_linesizes, 0, sizeof(ffmpeg->target_linesizes));
av_packet_free(&ffmpeg->packet);
avcodec_free_context(&ffmpeg->decoder_context);
avformat_close_input(&ffmpeg->format_context);
RARCH_LOG("[FFMPEG] Closed video input device %s.\n", ffmpeg->url);
}
static int ffmpeg_camera_frame_colorspace(ffmpeg_camera_t *ffmpeg)
{
const AVFrame *frame = ffmpeg->camera_frame;
const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(
(enum AVPixelFormat)frame->format);
enum AVColorSpace space = frame->colorspace;
enum AVColorRange range = frame->color_range;
const int *coefficients;
int full_range;
if (space == AVCOL_SPC_UNSPECIFIED)
space = ffmpeg->decoder_context->colorspace;
if (range == AVCOL_RANGE_UNSPECIFIED)
range = ffmpeg->decoder_context->color_range;
full_range = range == AVCOL_RANGE_JPEG;
if (range == AVCOL_RANGE_UNSPECIFIED)
{
/* Preserve the scaler's full-range defaults for RGB, gray and YUVJ. */
full_range = desc && ((desc->flags & AV_PIX_FMT_FLAG_RGB)
|| desc->nb_components <= 2);
switch ((enum AVPixelFormat)frame->format)
{
case AV_PIX_FMT_YUVJ420P:
case AV_PIX_FMT_YUVJ422P:
case AV_PIX_FMT_YUVJ444P:
case AV_PIX_FMT_YUVJ440P:
case AV_PIX_FMT_YUVJ411P:
full_range = 1;
break;
default:
break;
}
}
coefficients = sws_getCoefficients(space);
return sws_setColorspaceDetails(ffmpeg->scale_context,
coefficients, full_range, coefficients, 1, 0, 1 << 16, 1 << 16);
}
static void ffmpeg_camera_poll_thread(void *data)
{
ffmpeg_camera_t *ffmpeg = (ffmpeg_camera_t*)data;
if (!ffmpeg)
return;
while (!retro_atomic_load_acquire_int(&ffmpeg->done))
{
int poll_epoch = retro_atomic_load_acquire_int(&ffmpeg->poll_epoch);
int ret = av_read_frame(ffmpeg->format_context, ffmpeg->packet);
bool draining = ret == AVERROR_EOF;
bool retry_packet = false;
bool progress = false;
if (ret == AVERROR_EXIT && retro_atomic_load_acquire_int(&ffmpeg->done))
break;
if (ret == AVERROR(EAGAIN) || ret == AVERROR(EINTR))
{
/* Retry unavailable input only when the frontend requests a frame. */
while (!retro_atomic_load_acquire_int(&ffmpeg->done)
&& retro_atomic_load_acquire_int(&ffmpeg->poll_epoch) == poll_epoch)
{
int key = retro_eventcount_prepare_wait(&ffmpeg->read_retry);
if (retro_atomic_load_acquire_int(&ffmpeg->done)
|| retro_atomic_load_acquire_int(&ffmpeg->poll_epoch) != poll_epoch)
retro_eventcount_cancel_wait(&ffmpeg->read_retry);
else
retro_eventcount_commit_wait(&ffmpeg->read_retry, key);
}
continue;
}
/* Read the raw data from the camera. If that fails... */
if (ret < 0 && !draining)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to read frame: %s.\n", msg);
break;
}
if (!draining && ffmpeg->packet->stream_index != ffmpeg->video_stream_index)
goto done_loop;
retry_send:
retry_packet = false;
progress = false;
ret = avcodec_send_packet(ffmpeg->decoder_context,
draining ? NULL : ffmpeg->packet);
if (ret == AVERROR(EAGAIN))
retry_packet = true;
else if (ret < 0)
{ /* Send the raw data to the decoder. If that fails... */
if (ret == AVERROR_EOF)
{
RARCH_DBG("[FFMPEG] Video capture device closed.\n");
}
else
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to send packet to decoder: %s.\n", msg);
}
goto done_loop;
}
receive_frame:
if (retro_atomic_load_acquire_int(&ffmpeg->done))
goto done_loop;
ret = avcodec_receive_frame(ffmpeg->decoder_context, ffmpeg->camera_frame);
/* Send the decoded data to the camera frame. If that fails... */
if (ret < 0)
{
if (ret == AVERROR(EAGAIN) && retry_packet && progress)
goto retry_send;
/* These error codes mean no new frame, but not necessarily a problem */
if (!(ret == AVERROR_EOF || ret == AVERROR(EAGAIN)))
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to receive camera frame from decoder: %s.\n", msg);
}
goto done_loop;
}
progress = true;
retro_assert(ffmpeg->decoder->type == AVMEDIA_TYPE_VIDEO);
/* Decoded dimensions and format can change after opening the device. */
ffmpeg->scale_context = sws_getCachedContext(ffmpeg->scale_context,
ffmpeg->camera_frame->width, ffmpeg->camera_frame->height,
(enum AVPixelFormat)ffmpeg->camera_frame->format,
VIDEO_SCALE_W(ffmpeg->target_dims),
VIDEO_SCALE_H(ffmpeg->target_dims), AV_PIX_FMT_BGRA,
SWS_BILINEAR, NULL, NULL, NULL);
if (!ffmpeg->scale_context)
{
RARCH_ERR("[FFMPEG] Failed to configure frame scaler.\n");
goto done_loop;
}
if (ffmpeg_camera_frame_colorspace(ffmpeg) < 0)
{
RARCH_ERR("[FFMPEG] Failed to configure frame colorspace.\n");
goto done_loop;
}
/* sws_scale_frame is tidier but isn't as widely available */
ffmpeg->target_planes[0] = (uint8_t*)retro_triple_buffer_back(&ffmpeg->frames);
ret = sws_scale(
ffmpeg->scale_context,
(const uint8_t *const *)ffmpeg->camera_frame->data,
ffmpeg->camera_frame->linesize,
0,
ffmpeg->camera_frame->height,
ffmpeg->target_planes,
ffmpeg->target_linesizes
);
/* Scale and convert the frame to the target format. If that fails... */
if (ret < 0)
{
char msg[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
RARCH_ERR("[FFMPEG] Failed to scale frame: %s.\n", msg);
goto done_loop;
}
retro_triple_buffer_publish(&ffmpeg->frames);
av_frame_unref(ffmpeg->camera_frame);
goto receive_frame;
done_loop:
/* av_read_frame() fills the packet afresh each turn and does not
* release what it held */
av_packet_unref(ffmpeg->packet);
av_frame_unref(ffmpeg->camera_frame);
if (draining)
break;
}
/* every operation in this function needs this packet */
av_packet_unref(ffmpeg->packet);
}
static bool ffmpeg_camera_poll(void *data,
retro_camera_frame_raw_framebuffer_t frame_raw_cb,
retro_camera_frame_opengl_texture_t frame_gl_cb,
retro_camera_frame_planar_t frame_planar_cb)
{
ffmpeg_camera_t *ffmpeg = (ffmpeg_camera_t*)data;
(void)frame_planar_cb;
if (!ffmpeg->format_context)
{
RARCH_ERR("[FFMPEG] Camera is not started, cannot poll.\n");
return false;
}
if (!frame_raw_cb)
{
RARCH_ERR("[FFMPEG] No callback provided, cannot poll.\n");
return false;
}
retro_atomic_fetch_add_int(&ffmpeg->poll_epoch, 1);
retro_eventcount_notify(&ffmpeg->read_retry);
frame_raw_cb((uint32_t*)retro_triple_buffer_front(&ffmpeg->frames),
VIDEO_SCALE_W(ffmpeg->target_dims), VIDEO_SCALE_H(ffmpeg->target_dims),
ffmpeg->target_linesizes[0]);
return true;
}
camera_driver_t camera_ffmpeg = {
ffmpeg_camera_init,
ffmpeg_camera_free,
ffmpeg_camera_start,
ffmpeg_camera_stop,
ffmpeg_camera_poll,
"ffmpeg",
};
#endif /* LIBAVFORMAT_VERSION_MAJOR >= 58 */