mirror of
https://github.com/libretro/RetroArch.git
synced 2026-11-22 01:33:23 -10:00
Configure the cached scaler from each decoded frame's color metadata, with decoder metadata fallback and format defaults for unlabelled RGB, gray and YUVJ. Keep direct producer-buffer conversion. Cover full-to-limited range and BT.709-to-BT.601 transitions, plus unlabelled grayscale, checking every scaled output pixel.
956 lines
31 KiB
C
956 lines
31 KiB
C
/* RetroArch - A frontend for libretro.
|
|
* Copyright (C) 2010-2023 - Hans-Kristian Arntzen
|
|
* Copyright (C) 2023 - Jesse Talavera-Greenberg
|
|
*
|
|
* RetroArch is free software: you can redistribute it and/or modify it under the terms
|
|
* of the GNU General Public License as published by the Free Software Found-
|
|
* ation, either version 3 of the License, or (at your option) any later version.
|
|
*
|
|
* RetroArch is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
|
|
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
|
|
* PURPOSE. See the GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License along with RetroArch.
|
|
* If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#include <libavformat/version.h>
|
|
#if LIBAVFORMAT_VERSION_MAJOR >= 58
|
|
|
|
#include <libretro.h>
|
|
|
|
#include <memalign.h>
|
|
#include <retro_common_api.h>
|
|
#include <retro_assert.h>
|
|
#include <rthreads/rthreads.h>
|
|
#include <rthreads/retro_eventcount.h>
|
|
#include <retro_atomic.h>
|
|
#include <queues/retro_triple_buffer.h>
|
|
#include <lists/string_list.h>
|
|
#include <string/stdstring.h>
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
#include <libavdevice/avdevice.h>
|
|
#include <libavcodec/avcodec.h>
|
|
#include <libavutil/imgutils.h>
|
|
#include <libavutil/pixdesc.h>
|
|
#include <libswscale/swscale.h>
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#include "../camera_driver.h"
|
|
|
|
#include "../../configuration.h"
|
|
#include "../../verbosity.h"
|
|
|
|
/* The libavdevice input format the camera comes from. A build may name
|
|
* it already - a test harness points it at lavfi, which gives the
|
|
* whole pipeline a synthetic camera and needs no device. */
|
|
#ifndef FFMPEG_CAMERA_DEFAULT_BACKEND
|
|
#ifdef ANDROID
|
|
#define FFMPEG_CAMERA_DEFAULT_BACKEND "android_camera"
|
|
#elif defined(__linux__)
|
|
#define FFMPEG_CAMERA_DEFAULT_BACKEND "v4l2"
|
|
#elif defined(__APPLE__)
|
|
#define FFMPEG_CAMERA_DEFAULT_BACKEND "avfoundation"
|
|
#elif defined(__WIN32__)
|
|
#define FFMPEG_CAMERA_DEFAULT_BACKEND "dshow"
|
|
#elif defined(__FreeBSD__) || defined(__OpenBSD__) || defined (__NetBSD__)
|
|
#define FFMPEG_CAMERA_DEFAULT_BACKEND "bktr"
|
|
#else
|
|
#define FFMPEG_CAMERA_DEFAULT_BACKEND "lavfi"
|
|
#endif
|
|
#endif
|
|
|
|
/* lavf 59 (FFmpeg 5.0) made the demuxer/codec discovery API const-correct:
|
|
* av_find_input_format() returns const AVInputFormat*, avformat_open_input()
|
|
* accepts one, and av_find_best_stream() takes const AVCodec**. Older
|
|
* versions use mutable pointers throughout, so a single const-qualified
|
|
* declaration cannot satisfy both. */
|
|
#if LIBAVFORMAT_VERSION_MAJOR >= 59
|
|
typedef const AVInputFormat ffmpeg_camera_input_format_t;
|
|
typedef const AVCodec ffmpeg_camera_codec_t;
|
|
#else
|
|
typedef AVInputFormat ffmpeg_camera_input_format_t;
|
|
typedef AVCodec ffmpeg_camera_codec_t;
|
|
#endif
|
|
|
|
typedef struct ffmpeg_camera
|
|
{
|
|
sthread_t *poll_thread;
|
|
AVFormatContext *format_context;
|
|
AVCodecContext *decoder_context;
|
|
ffmpeg_camera_codec_t *decoder; /* owned by ffmpeg, don't free it */
|
|
ffmpeg_camera_input_format_t *input_format; /* owned by ffmpeg, don't free it */
|
|
AVDictionary *options;
|
|
AVPacket *packet;
|
|
AVFrame *camera_frame;
|
|
int video_stream_index;
|
|
unsigned requested_dims; /* VIDEO_SCALE_PACK */
|
|
uint8_t *target_planes[4];
|
|
int target_linesizes[4];
|
|
unsigned target_dims; /* VIDEO_SCALE_PACK */
|
|
struct SwsContext *scale_context;
|
|
|
|
/* "name" for the camera device.
|
|
* Not just the reported name, there may be a bit of extra syntax.
|
|
* See https://ffmpeg.org/ffmpeg-devices.html#Input-Devices for details. */
|
|
char url[512];
|
|
|
|
/* Converted frames, from the poll thread to the core's poll: the
|
|
* thread fills one, the core reads the newest finished one, and
|
|
* neither waits for the other. */
|
|
uint8_t *target_buffers[3];
|
|
retro_triple_buffer_t frames;
|
|
size_t target_buffer_length;
|
|
/* The poll thread's loop condition: the main thread sets it in
|
|
* stop(), the thread reads it every turn. */
|
|
retro_atomic_int_t done;
|
|
retro_atomic_int_t poll_epoch;
|
|
retro_eventcount_t read_retry;
|
|
bool read_retry_init;
|
|
} ffmpeg_camera_t;
|
|
|
|
static void ffmpeg_camera_free(void *data);
|
|
|
|
static int ffmpeg_camera_get_initial_options(
|
|
const AVInputFormat *backend,
|
|
AVDictionary **options,
|
|
uint64_t caps,
|
|
unsigned dims
|
|
)
|
|
{
|
|
int ret = 0;
|
|
|
|
#ifdef __APPLE__
|
|
/* For AVFoundation, we need to handle options differently
|
|
* AVFoundation expects these options to be passed to avformat_open_input.
|
|
* We'll set them later in ffmpeg_camera_open_device */
|
|
if (backend && string_is_equal(backend->name, "avfoundation"))
|
|
return 0;
|
|
#endif
|
|
|
|
/* If the core is letting the frontend pick the size... */
|
|
if (VIDEO_SCALE_W(dims) != 0 && VIDEO_SCALE_H(dims) != 0)
|
|
{
|
|
char dimensions[128];
|
|
snprintf(dimensions, sizeof(dimensions), "%ux%u",
|
|
VIDEO_SCALE_W(dims), VIDEO_SCALE_H(dims));
|
|
|
|
ret = av_dict_set(options, "video_size", dimensions, 0);
|
|
|
|
if (ret < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to set option: %s.\n", msg);
|
|
av_dict_free(options);
|
|
return ret;
|
|
}
|
|
}
|
|
|
|
/* I wanted to list supported formats and pick the most appropriate size
|
|
* if the requested size isn't available,
|
|
* but ffmpeg doesn't seem to offer a way to do that.
|
|
*/
|
|
if (!options)
|
|
{
|
|
RARCH_DBG("[FFMPEG] No options set, not allocating a dict (this isn't an error).");
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
/* Device URL syntax varies by backend.
|
|
* See https://ffmpeg.org/ffmpeg-devices.html for details. */
|
|
static void ffmpeg_camera_get_source_url(ffmpeg_camera_t *ffmpeg, const AVDeviceInfo *device)
|
|
{
|
|
#ifdef __WIN32__
|
|
if (string_is_equal(ffmpeg->input_format->name, "dshow"))
|
|
{
|
|
snprintf(ffmpeg->url, sizeof(ffmpeg->url), "video=%s", device->device_description);
|
|
return;
|
|
}
|
|
#elif defined(__APPLE__)
|
|
if (string_is_equal(ffmpeg->input_format->name, "avfoundation"))
|
|
{
|
|
/* we only want video, not audio */
|
|
/* Use "0:none" for the first video device, no audio */
|
|
snprintf(ffmpeg->url, sizeof(ffmpeg->url), "0:none");
|
|
return;
|
|
}
|
|
#endif
|
|
|
|
/* Other backends that support listing available sources use the name as-is;
|
|
* some advanced backends have extra syntax (e.g. gdigrab)
|
|
* but they don't support listing available sources,
|
|
* so players will have to enter them manually.
|
|
*/
|
|
strlcpy(ffmpeg->url, device->device_description, sizeof(ffmpeg->url));
|
|
}
|
|
|
|
static int ffmpeg_camera_interrupt(void *opaque)
|
|
{
|
|
ffmpeg_camera_t *ffmpeg = (ffmpeg_camera_t*)opaque;
|
|
return retro_atomic_load_acquire_int(&ffmpeg->done);
|
|
}
|
|
|
|
static int ffmpeg_camera_open_input(ffmpeg_camera_t *ffmpeg, AVDictionary **options)
|
|
{
|
|
ffmpeg->format_context = avformat_alloc_context();
|
|
if (!ffmpeg->format_context)
|
|
return AVERROR(ENOMEM);
|
|
ffmpeg->format_context->interrupt_callback.callback = ffmpeg_camera_interrupt;
|
|
ffmpeg->format_context->interrupt_callback.opaque = ffmpeg;
|
|
return avformat_open_input(&ffmpeg->format_context, ffmpeg->url,
|
|
ffmpeg->input_format, options);
|
|
}
|
|
|
|
static int ffmpeg_camera_open_device(ffmpeg_camera_t *ffmpeg)
|
|
{
|
|
AVDictionaryEntry *e = NULL;
|
|
AVDictionary *options = NULL;
|
|
int ret = ffmpeg->options ? av_dict_copy(&options, ffmpeg->options, 0) : 0;
|
|
/* copy the options dict so that other steps in start() can use it,
|
|
* as avformat_open_input clears it and adds unrecognized settings */
|
|
|
|
if (ret < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to copy options: %s.\n", msg);
|
|
goto done;
|
|
}
|
|
|
|
#ifdef __APPLE__
|
|
/* For AVFoundation, set the options that the device expects */
|
|
if (ffmpeg->input_format && string_is_equal(ffmpeg->input_format->name, "avfoundation"))
|
|
{
|
|
char dimensions[128];
|
|
|
|
/* Set video size if requested */
|
|
if ( VIDEO_SCALE_W(ffmpeg->requested_dims) != 0
|
|
&& VIDEO_SCALE_H(ffmpeg->requested_dims) != 0)
|
|
{
|
|
/* Use a resolution that the device likely supports */
|
|
/* Common resolutions: 640x480, 1280x720, 1920x1080 */
|
|
unsigned width = VIDEO_SCALE_W(ffmpeg->requested_dims);
|
|
unsigned height = VIDEO_SCALE_H(ffmpeg->requested_dims);
|
|
|
|
/* If the requested resolution is too small, use a minimum supported size */
|
|
if (width < 640 || height < 480)
|
|
{
|
|
width = 640;
|
|
height = 480;
|
|
RARCH_LOG("[FFMPEG] Requested resolution %ux%u too small, using %ux%u instead.\n",
|
|
VIDEO_SCALE_W(ffmpeg->requested_dims),
|
|
VIDEO_SCALE_H(ffmpeg->requested_dims), width, height);
|
|
}
|
|
|
|
snprintf(dimensions, sizeof(dimensions), "%ux%u", width, height);
|
|
av_dict_set(&options, "video_size", dimensions, 0);
|
|
}
|
|
|
|
/* Set framerate */
|
|
av_dict_set(&options, "framerate", "30", 0);
|
|
|
|
/* Set pixel format to a widely supported format */
|
|
av_dict_set(&options, "pixel_format", "uyvy422", 0);
|
|
}
|
|
#endif
|
|
|
|
ret = ffmpeg_camera_open_input(ffmpeg, &options);
|
|
if (ret < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_WARN("[FFMPEG] Failed to open video input device \"%s\": %s.\n", ffmpeg->url, msg);
|
|
|
|
#ifdef __APPLE__
|
|
/* For AVFoundation, try with default settings if custom settings fail */
|
|
if (ffmpeg->input_format && string_is_equal(ffmpeg->input_format->name, "avfoundation"))
|
|
{
|
|
RARCH_LOG("[FFMPEG] Trying with default AVFoundation settings...\n");
|
|
av_dict_free(&options);
|
|
|
|
/* Try with just the basic framerate option */
|
|
av_dict_set(&options, "framerate", "30", 0);
|
|
|
|
ret = ffmpeg_camera_open_input(ffmpeg, &options);
|
|
if (ret < 0)
|
|
{
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_WARN("[FFMPEG] Failed with basic options, trying with no options: %s.\n", msg);
|
|
av_dict_free(&options);
|
|
|
|
/* Last resort: try with no options at all */
|
|
ret = ffmpeg_camera_open_input(ffmpeg, NULL);
|
|
if (ret < 0)
|
|
{
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to open device even with default settings: %s.\n", msg);
|
|
goto done;
|
|
}
|
|
|
|
RARCH_LOG("[FFMPEG] Successfully opened device with default settings.\n");
|
|
}
|
|
else
|
|
{
|
|
RARCH_LOG("[FFMPEG] Successfully opened device with basic settings.\n");
|
|
}
|
|
}
|
|
else
|
|
#endif
|
|
/* If we're not already requesting the default format... */
|
|
if (ffmpeg->options)
|
|
{
|
|
ret = ffmpeg_camera_open_input(ffmpeg, NULL);
|
|
if (ret < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to open the same device in its default format: %s.\n", msg);
|
|
goto done;
|
|
}
|
|
}
|
|
}
|
|
|
|
done:
|
|
if (options)
|
|
{
|
|
e = NULL;
|
|
while ((e = av_dict_get(options, "", e, AV_DICT_IGNORE_SUFFIX)))
|
|
{
|
|
/* av_dict_iterate isn't always available, so we use av_dict_get's legacy behavior instead */
|
|
RARCH_WARN("[FFMPEG] Unrecognized option on video input device: %s=%s.\n", e->key, e->value);
|
|
}
|
|
}
|
|
|
|
av_dict_free(&options); /* noop if NULL */
|
|
if (ret == 0)
|
|
{
|
|
RARCH_LOG("[FFMPEG] Opened video input device \"%s\".\n", ffmpeg->url);
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static void *ffmpeg_camera_init(const char *device, uint64_t caps, unsigned dims)
|
|
{
|
|
ffmpeg_camera_t *ffmpeg = NULL;
|
|
AVDeviceInfoList *device_list = NULL;
|
|
int ret = 0;
|
|
int num_sources = 0;
|
|
|
|
/* If the core didn't ask for raw framebuffers... */
|
|
if ((caps & (UINT64_C(1) << RETRO_CAMERA_BUFFER_RAW_FRAMEBUFFER)) == 0)
|
|
{
|
|
RARCH_ERR("[FFMPEG] Camera driver only supports raw framebuffer output.\n");
|
|
return NULL;
|
|
}
|
|
|
|
ffmpeg = (ffmpeg_camera_t*)calloc(1, sizeof(*ffmpeg));
|
|
if (!ffmpeg)
|
|
{
|
|
RARCH_ERR("[FFMPEG] Failed to allocate memory for camera driver.\n");
|
|
return NULL;
|
|
}
|
|
|
|
ffmpeg->requested_dims = dims;
|
|
ffmpeg->read_retry_init = retro_eventcount_init(&ffmpeg->read_retry);
|
|
if (!ffmpeg->read_retry_init)
|
|
{
|
|
RARCH_ERR("[FFMPEG] Failed to initialize capture wakeup.\n");
|
|
goto error;
|
|
}
|
|
|
|
avdevice_register_all();
|
|
RARCH_LOG("[FFMPEG] Initialized libavdevice.\n");
|
|
|
|
ffmpeg->input_format = av_find_input_format(FFMPEG_CAMERA_DEFAULT_BACKEND);
|
|
if (!ffmpeg->input_format)
|
|
{
|
|
RARCH_ERR("[FFMPEG] No suitable video input backend found.\n");
|
|
goto error;
|
|
}
|
|
|
|
RARCH_LOG("[FFMPEG] Using camera backend: %s (%s, flags=0x%x).\n", ffmpeg->input_format->name, ffmpeg->input_format->long_name, ffmpeg->input_format->flags);
|
|
|
|
ret = ffmpeg_camera_get_initial_options(ffmpeg->input_format, &ffmpeg->options, caps, dims);
|
|
if (ret < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to get initial options: %s.\n", msg);
|
|
goto error;
|
|
}
|
|
|
|
/* A device the frontend named is the device, and enumeration is
|
|
* skipped: what the setting says is a source url for the backend
|
|
* in use, which is the only thing this driver could do with it.
|
|
* It used to be ignored outright - the argument was taken and
|
|
* never read, so a user who picked a camera got whichever one
|
|
* enumerated first. */
|
|
if (!string_is_empty(device))
|
|
{
|
|
strlcpy(ffmpeg->url, device, sizeof(ffmpeg->url));
|
|
RARCH_LOG("[FFMPEG] Using the device the frontend asked for: %s.\n",
|
|
ffmpeg->url);
|
|
goto have_device;
|
|
}
|
|
|
|
num_sources = avdevice_list_input_sources(ffmpeg->input_format, NULL, ffmpeg->options, &device_list);
|
|
|
|
#ifdef __APPLE__
|
|
/* AVFoundation device listing may fail or return -38 (function not implemented) */
|
|
/* but we can still use the device with index 0 */
|
|
if (num_sources < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, num_sources);
|
|
RARCH_WARN("[FFMPEG] Device enumeration not fully supported on AVFoundation: %s. Using default device.\n", msg);
|
|
/* Continue with default device */
|
|
ffmpeg_camera_get_source_url(ffmpeg, NULL);
|
|
}
|
|
else if (num_sources == 0)
|
|
{
|
|
RARCH_ERR("[FFMPEG] No video input sources found.\n");
|
|
goto error;
|
|
}
|
|
else
|
|
{
|
|
ffmpeg_camera_get_source_url(ffmpeg, device_list->devices[0]);
|
|
}
|
|
#else
|
|
if (num_sources == 0)
|
|
{
|
|
RARCH_ERR("[FFMPEG] No video input sources found.\n");
|
|
goto error;
|
|
}
|
|
|
|
if (num_sources < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, num_sources);
|
|
RARCH_ERR("[FFMPEG] Failed to list video input sources: %s.\n", msg);
|
|
goto error;
|
|
}
|
|
|
|
ffmpeg_camera_get_source_url(ffmpeg, device_list->devices[0]);
|
|
#endif
|
|
have_device:
|
|
RARCH_LOG("[FFMPEG] Using video input device: %s (%s, flags=0x%x).\n", ffmpeg->input_format->name, ffmpeg->input_format->long_name, ffmpeg->input_format->flags);
|
|
|
|
avdevice_free_list_devices(&device_list);
|
|
return ffmpeg;
|
|
error:
|
|
avdevice_free_list_devices(&device_list);
|
|
ffmpeg_camera_free(ffmpeg);
|
|
return NULL;
|
|
}
|
|
|
|
static void ffmpeg_camera_stop(void *data);
|
|
static void ffmpeg_camera_free(void *data)
|
|
{
|
|
ffmpeg_camera_t *ffmpeg = (ffmpeg_camera_t*)data;
|
|
|
|
if (!ffmpeg)
|
|
return;
|
|
|
|
ffmpeg_camera_stop(ffmpeg);
|
|
|
|
av_dict_free(&ffmpeg->options);
|
|
retro_eventcount_free(&ffmpeg->read_retry);
|
|
|
|
free(ffmpeg);
|
|
}
|
|
|
|
static void ffmpeg_camera_poll_thread(void *data);
|
|
|
|
static bool ffmpeg_camera_start(void *data)
|
|
{
|
|
ffmpeg_camera_t *ffmpeg = (ffmpeg_camera_t*)data;
|
|
int ret = 0;
|
|
AVStream *stream = NULL;
|
|
AVDictionary *options = NULL;
|
|
const AVDictionaryEntry *e = NULL;
|
|
int target_buffer_length = 0;
|
|
unsigned i;
|
|
|
|
/* TODO: Check the actual format context, not just the pointer */
|
|
if (ffmpeg->format_context)
|
|
{
|
|
RARCH_LOG("[FFMPEG] Camera %s is already started, no action needed.\n", ffmpeg->format_context->url);
|
|
return true;
|
|
}
|
|
|
|
retro_atomic_store_release_int(&ffmpeg->done, 0);
|
|
ret = ffmpeg_camera_open_device(ffmpeg);
|
|
if (ret < 0)
|
|
goto error;
|
|
|
|
ret = av_dict_copy(&options, ffmpeg->options, 0);
|
|
if (ret < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to copy options: %s.\n", msg);
|
|
goto error;
|
|
}
|
|
|
|
ret = avformat_find_stream_info(ffmpeg->format_context, NULL);
|
|
if (ret < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to find stream info: %s.\n", msg);
|
|
goto error;
|
|
}
|
|
|
|
while ((e = av_dict_get(options, "", e, AV_DICT_IGNORE_SUFFIX)))
|
|
{
|
|
RARCH_WARN("[FFMPEG] Unrecognized option on video input device: %s=%s.\n", e->key, e->value);
|
|
}
|
|
|
|
ret = av_find_best_stream(ffmpeg->format_context,
|
|
AVMEDIA_TYPE_VIDEO, -1, -1, &ffmpeg->decoder, 0);
|
|
if (ret < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to find video stream: %s.\n", msg);
|
|
goto error;
|
|
}
|
|
stream = ffmpeg->format_context->streams[ret];
|
|
ffmpeg->video_stream_index = stream->index;
|
|
|
|
RARCH_LOG("[FFMPEG] Using video stream #%d with decoder \"%s\" (%s).\n", ret, ffmpeg->decoder->name, ffmpeg->decoder->long_name);
|
|
|
|
ffmpeg->decoder_context = avcodec_alloc_context3(ffmpeg->decoder);
|
|
if (!ffmpeg->decoder_context)
|
|
{
|
|
RARCH_ERR("[FFMPEG] Failed to allocate decoder context.\n");
|
|
goto error;
|
|
}
|
|
|
|
ret = avcodec_parameters_to_context(ffmpeg->decoder_context, stream->codecpar);
|
|
if (ret < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to copy codec parameters to decoder context: %s.\n", msg);
|
|
goto error;
|
|
}
|
|
|
|
if (!sws_isSupportedInput(ffmpeg->decoder_context->pix_fmt))
|
|
{
|
|
RARCH_ERR("[FFMPEG] Unsupported scaler input pixel format: %s.\n", av_get_pix_fmt_name(ffmpeg->decoder_context->pix_fmt));
|
|
goto error;
|
|
}
|
|
|
|
ret = av_dict_copy(&ffmpeg->options, options, 0);
|
|
if (ret < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to copy options: %s.\n", msg);
|
|
goto error;
|
|
}
|
|
|
|
ret = avcodec_open2(ffmpeg->decoder_context, ffmpeg->decoder, NULL);
|
|
if (ret < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to open decoder: %s.\n", msg);
|
|
goto error;
|
|
}
|
|
|
|
while ((e = av_dict_get(options, "", e, AV_DICT_IGNORE_SUFFIX)))
|
|
{
|
|
RARCH_WARN("[FFMPEG] Unrecognized option on video input device: %s=%s.\n", e->key, e->value);
|
|
}
|
|
|
|
av_dict_free(&options);
|
|
|
|
ffmpeg->packet = av_packet_alloc();
|
|
if (!ffmpeg->packet)
|
|
{
|
|
RARCH_ERR("[FFMPEG] Failed to allocate packet.\n");
|
|
goto error;
|
|
}
|
|
|
|
ffmpeg->camera_frame = av_frame_alloc();
|
|
if (!ffmpeg->camera_frame)
|
|
{
|
|
RARCH_ERR("[FFMPEG] Failed to allocate camera frame.\n");
|
|
goto error;
|
|
}
|
|
|
|
ffmpeg->target_dims = VIDEO_SCALE_PACK(
|
|
VIDEO_SCALE_W(ffmpeg->requested_dims)
|
|
? VIDEO_SCALE_W(ffmpeg->requested_dims)
|
|
: (unsigned)ffmpeg->decoder_context->width,
|
|
VIDEO_SCALE_H(ffmpeg->requested_dims)
|
|
? VIDEO_SCALE_H(ffmpeg->requested_dims)
|
|
: (unsigned)ffmpeg->decoder_context->height);
|
|
|
|
target_buffer_length = av_image_get_buffer_size(
|
|
AV_PIX_FMT_BGRA,
|
|
VIDEO_SCALE_W(ffmpeg->target_dims),
|
|
VIDEO_SCALE_H(ffmpeg->target_dims),
|
|
1
|
|
);
|
|
if (target_buffer_length < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, target_buffer_length);
|
|
RARCH_ERR("[FFMPEG] Failed to calculate target size: %s.\n", msg);
|
|
goto error;
|
|
}
|
|
|
|
/* Aligned and padded for swscale's direct output. */
|
|
ffmpeg->target_buffer_length = target_buffer_length;
|
|
ffmpeg->target_linesizes[0] = VIDEO_SCALE_W(ffmpeg->target_dims) * 4;
|
|
for (i = 0; i < 3; i++)
|
|
if ((ffmpeg->target_buffers[i] = (uint8_t*)memalign_alloc(32,
|
|
(size_t)target_buffer_length + AV_INPUT_BUFFER_PADDING_SIZE)))
|
|
/* black until the first frame is in */
|
|
memset(ffmpeg->target_buffers[i], 0,
|
|
(size_t)target_buffer_length + AV_INPUT_BUFFER_PADDING_SIZE);
|
|
if (!ffmpeg->target_buffers[0] || !ffmpeg->target_buffers[1] || !ffmpeg->target_buffers[2])
|
|
{
|
|
RARCH_ERR("[FFMPEG] Failed to allocate target %d-byte buffer for %dx%d %s-formatted video data.\n",
|
|
target_buffer_length,
|
|
ffmpeg->decoder_context->width,
|
|
ffmpeg->decoder_context->height,
|
|
av_get_pix_fmt_name(AV_PIX_FMT_BGRA)
|
|
);
|
|
goto error;
|
|
}
|
|
|
|
RARCH_LOG("[FFMPEG] Allocated %d bytes for %dx%d %s-formatted video data.\n",
|
|
target_buffer_length,
|
|
ffmpeg->decoder_context->width,
|
|
ffmpeg->decoder_context->height,
|
|
av_get_pix_fmt_name(ffmpeg->decoder_context->pix_fmt)
|
|
);
|
|
|
|
ffmpeg->scale_context = sws_getContext(
|
|
ffmpeg->decoder_context->width,
|
|
ffmpeg->decoder_context->height,
|
|
ffmpeg->decoder_context->pix_fmt,
|
|
VIDEO_SCALE_W(ffmpeg->target_dims),
|
|
VIDEO_SCALE_H(ffmpeg->target_dims),
|
|
AV_PIX_FMT_BGRA,
|
|
SWS_BILINEAR,
|
|
NULL, NULL, NULL
|
|
);
|
|
if (!ffmpeg->scale_context)
|
|
{
|
|
RARCH_ERR("[FFMPEG] Failed to create scale context.\n");
|
|
goto error;
|
|
}
|
|
|
|
retro_triple_buffer_init(&ffmpeg->frames, ffmpeg->target_buffers[0],
|
|
ffmpeg->target_buffers[1], ffmpeg->target_buffers[2]);
|
|
|
|
ffmpeg->poll_thread = sthread_create(ffmpeg_camera_poll_thread, ffmpeg);
|
|
if (!ffmpeg->poll_thread)
|
|
{
|
|
RARCH_ERR("[FFMPEG] Failed to create poll thread.\n");
|
|
goto error;
|
|
}
|
|
|
|
return true;
|
|
error:
|
|
av_dict_free(&options);
|
|
ffmpeg_camera_stop(ffmpeg);
|
|
|
|
return false;
|
|
}
|
|
|
|
static void ffmpeg_camera_stop(void *data)
|
|
{
|
|
ffmpeg_camera_t *ffmpeg = (ffmpeg_camera_t*)data;
|
|
unsigned i;
|
|
|
|
/* The thread first. It is the other user of the decoder - it sends
|
|
* every packet it reads and receives every frame - and an
|
|
* AVCodecContext takes one user at a time, so the flush below
|
|
* cannot happen while the thread is still in there. This used to
|
|
* flush and then join. */
|
|
retro_atomic_store_release_int(&ffmpeg->done, 1);
|
|
if (ffmpeg->read_retry_init)
|
|
retro_eventcount_notify(&ffmpeg->read_retry);
|
|
sthread_join(ffmpeg->poll_thread); /* wait for the thread to finish, then free it */
|
|
|
|
if (!ffmpeg->format_context)
|
|
{
|
|
RARCH_LOG("[FFMPEG] Camera %s is already stopped, no flush needed.\n", ffmpeg->url);
|
|
}
|
|
else if (ffmpeg->decoder_context)
|
|
{
|
|
int ret = avcodec_send_packet(ffmpeg->decoder_context, NULL);
|
|
if (ret < 0 && ret != AVERROR_EOF)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to flush decoder: %s.\n", msg);
|
|
}
|
|
}
|
|
|
|
/* these functions are noops for NULL pointers */
|
|
ffmpeg->poll_thread = NULL;
|
|
|
|
sws_freeContext(ffmpeg->scale_context);
|
|
ffmpeg->scale_context = NULL;
|
|
|
|
for (i = 0; i < 3; i++)
|
|
{
|
|
memalign_free(ffmpeg->target_buffers[i]);
|
|
ffmpeg->target_buffers[i] = NULL;
|
|
}
|
|
/* a poll between stop and start is handed no frame */
|
|
retro_triple_buffer_init(&ffmpeg->frames, NULL, NULL, NULL);
|
|
memset(ffmpeg->target_planes, 0, sizeof(ffmpeg->target_planes));
|
|
ffmpeg->target_buffer_length = 0;
|
|
ffmpeg->target_dims = 0;
|
|
|
|
av_frame_free(&ffmpeg->camera_frame);
|
|
memset(ffmpeg->target_linesizes, 0, sizeof(ffmpeg->target_linesizes));
|
|
|
|
av_packet_free(&ffmpeg->packet);
|
|
avcodec_free_context(&ffmpeg->decoder_context);
|
|
avformat_close_input(&ffmpeg->format_context);
|
|
RARCH_LOG("[FFMPEG] Closed video input device %s.\n", ffmpeg->url);
|
|
}
|
|
|
|
static int ffmpeg_camera_frame_colorspace(ffmpeg_camera_t *ffmpeg)
|
|
{
|
|
const AVFrame *frame = ffmpeg->camera_frame;
|
|
const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(
|
|
(enum AVPixelFormat)frame->format);
|
|
enum AVColorSpace space = frame->colorspace;
|
|
enum AVColorRange range = frame->color_range;
|
|
const int *coefficients;
|
|
int full_range;
|
|
|
|
if (space == AVCOL_SPC_UNSPECIFIED)
|
|
space = ffmpeg->decoder_context->colorspace;
|
|
if (range == AVCOL_RANGE_UNSPECIFIED)
|
|
range = ffmpeg->decoder_context->color_range;
|
|
full_range = range == AVCOL_RANGE_JPEG;
|
|
if (range == AVCOL_RANGE_UNSPECIFIED)
|
|
{
|
|
/* Preserve the scaler's full-range defaults for RGB, gray and YUVJ. */
|
|
full_range = desc && ((desc->flags & AV_PIX_FMT_FLAG_RGB)
|
|
|| desc->nb_components <= 2);
|
|
switch ((enum AVPixelFormat)frame->format)
|
|
{
|
|
case AV_PIX_FMT_YUVJ420P:
|
|
case AV_PIX_FMT_YUVJ422P:
|
|
case AV_PIX_FMT_YUVJ444P:
|
|
case AV_PIX_FMT_YUVJ440P:
|
|
case AV_PIX_FMT_YUVJ411P:
|
|
full_range = 1;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
coefficients = sws_getCoefficients(space);
|
|
return sws_setColorspaceDetails(ffmpeg->scale_context,
|
|
coefficients, full_range, coefficients, 1, 0, 1 << 16, 1 << 16);
|
|
}
|
|
|
|
static void ffmpeg_camera_poll_thread(void *data)
|
|
{
|
|
ffmpeg_camera_t *ffmpeg = (ffmpeg_camera_t*)data;
|
|
|
|
if (!ffmpeg)
|
|
return;
|
|
|
|
while (!retro_atomic_load_acquire_int(&ffmpeg->done))
|
|
{
|
|
int poll_epoch = retro_atomic_load_acquire_int(&ffmpeg->poll_epoch);
|
|
int ret = av_read_frame(ffmpeg->format_context, ffmpeg->packet);
|
|
bool draining = ret == AVERROR_EOF;
|
|
bool retry_packet = false;
|
|
bool progress = false;
|
|
if (ret == AVERROR_EXIT && retro_atomic_load_acquire_int(&ffmpeg->done))
|
|
break;
|
|
if (ret == AVERROR(EAGAIN) || ret == AVERROR(EINTR))
|
|
{
|
|
/* Retry unavailable input only when the frontend requests a frame. */
|
|
while (!retro_atomic_load_acquire_int(&ffmpeg->done)
|
|
&& retro_atomic_load_acquire_int(&ffmpeg->poll_epoch) == poll_epoch)
|
|
{
|
|
int key = retro_eventcount_prepare_wait(&ffmpeg->read_retry);
|
|
if (retro_atomic_load_acquire_int(&ffmpeg->done)
|
|
|| retro_atomic_load_acquire_int(&ffmpeg->poll_epoch) != poll_epoch)
|
|
retro_eventcount_cancel_wait(&ffmpeg->read_retry);
|
|
else
|
|
retro_eventcount_commit_wait(&ffmpeg->read_retry, key);
|
|
}
|
|
continue;
|
|
}
|
|
/* Read the raw data from the camera. If that fails... */
|
|
if (ret < 0 && !draining)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to read frame: %s.\n", msg);
|
|
break;
|
|
}
|
|
|
|
if (!draining && ffmpeg->packet->stream_index != ffmpeg->video_stream_index)
|
|
goto done_loop;
|
|
|
|
retry_send:
|
|
retry_packet = false;
|
|
progress = false;
|
|
ret = avcodec_send_packet(ffmpeg->decoder_context,
|
|
draining ? NULL : ffmpeg->packet);
|
|
if (ret == AVERROR(EAGAIN))
|
|
retry_packet = true;
|
|
else if (ret < 0)
|
|
{ /* Send the raw data to the decoder. If that fails... */
|
|
if (ret == AVERROR_EOF)
|
|
{
|
|
RARCH_DBG("[FFMPEG] Video capture device closed.\n");
|
|
}
|
|
else
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to send packet to decoder: %s.\n", msg);
|
|
}
|
|
|
|
goto done_loop;
|
|
}
|
|
|
|
receive_frame:
|
|
if (retro_atomic_load_acquire_int(&ffmpeg->done))
|
|
goto done_loop;
|
|
ret = avcodec_receive_frame(ffmpeg->decoder_context, ffmpeg->camera_frame);
|
|
|
|
/* Send the decoded data to the camera frame. If that fails... */
|
|
if (ret < 0)
|
|
{
|
|
if (ret == AVERROR(EAGAIN) && retry_packet && progress)
|
|
goto retry_send;
|
|
/* These error codes mean no new frame, but not necessarily a problem */
|
|
if (!(ret == AVERROR_EOF || ret == AVERROR(EAGAIN)))
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to receive camera frame from decoder: %s.\n", msg);
|
|
}
|
|
|
|
goto done_loop;
|
|
}
|
|
progress = true;
|
|
|
|
retro_assert(ffmpeg->decoder->type == AVMEDIA_TYPE_VIDEO);
|
|
|
|
/* Decoded dimensions and format can change after opening the device. */
|
|
ffmpeg->scale_context = sws_getCachedContext(ffmpeg->scale_context,
|
|
ffmpeg->camera_frame->width, ffmpeg->camera_frame->height,
|
|
(enum AVPixelFormat)ffmpeg->camera_frame->format,
|
|
VIDEO_SCALE_W(ffmpeg->target_dims),
|
|
VIDEO_SCALE_H(ffmpeg->target_dims), AV_PIX_FMT_BGRA,
|
|
SWS_BILINEAR, NULL, NULL, NULL);
|
|
if (!ffmpeg->scale_context)
|
|
{
|
|
RARCH_ERR("[FFMPEG] Failed to configure frame scaler.\n");
|
|
goto done_loop;
|
|
}
|
|
|
|
if (ffmpeg_camera_frame_colorspace(ffmpeg) < 0)
|
|
{
|
|
RARCH_ERR("[FFMPEG] Failed to configure frame colorspace.\n");
|
|
goto done_loop;
|
|
}
|
|
|
|
/* sws_scale_frame is tidier but isn't as widely available */
|
|
ffmpeg->target_planes[0] = (uint8_t*)retro_triple_buffer_back(&ffmpeg->frames);
|
|
ret = sws_scale(
|
|
ffmpeg->scale_context,
|
|
(const uint8_t *const *)ffmpeg->camera_frame->data,
|
|
ffmpeg->camera_frame->linesize,
|
|
0,
|
|
ffmpeg->camera_frame->height,
|
|
ffmpeg->target_planes,
|
|
ffmpeg->target_linesizes
|
|
);
|
|
|
|
/* Scale and convert the frame to the target format. If that fails... */
|
|
if (ret < 0)
|
|
{
|
|
char msg[AV_ERROR_MAX_STRING_SIZE];
|
|
av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret);
|
|
RARCH_ERR("[FFMPEG] Failed to scale frame: %s.\n", msg);
|
|
goto done_loop;
|
|
}
|
|
|
|
retro_triple_buffer_publish(&ffmpeg->frames);
|
|
av_frame_unref(ffmpeg->camera_frame);
|
|
goto receive_frame;
|
|
done_loop:
|
|
/* av_read_frame() fills the packet afresh each turn and does not
|
|
* release what it held */
|
|
av_packet_unref(ffmpeg->packet);
|
|
av_frame_unref(ffmpeg->camera_frame);
|
|
if (draining)
|
|
break;
|
|
}
|
|
|
|
/* every operation in this function needs this packet */
|
|
av_packet_unref(ffmpeg->packet);
|
|
}
|
|
|
|
static bool ffmpeg_camera_poll(void *data,
|
|
retro_camera_frame_raw_framebuffer_t frame_raw_cb,
|
|
retro_camera_frame_opengl_texture_t frame_gl_cb,
|
|
retro_camera_frame_planar_t frame_planar_cb)
|
|
{
|
|
ffmpeg_camera_t *ffmpeg = (ffmpeg_camera_t*)data;
|
|
(void)frame_planar_cb;
|
|
|
|
if (!ffmpeg->format_context)
|
|
{
|
|
RARCH_ERR("[FFMPEG] Camera is not started, cannot poll.\n");
|
|
return false;
|
|
}
|
|
|
|
if (!frame_raw_cb)
|
|
{
|
|
RARCH_ERR("[FFMPEG] No callback provided, cannot poll.\n");
|
|
return false;
|
|
}
|
|
|
|
retro_atomic_fetch_add_int(&ffmpeg->poll_epoch, 1);
|
|
retro_eventcount_notify(&ffmpeg->read_retry);
|
|
frame_raw_cb((uint32_t*)retro_triple_buffer_front(&ffmpeg->frames),
|
|
VIDEO_SCALE_W(ffmpeg->target_dims), VIDEO_SCALE_H(ffmpeg->target_dims),
|
|
ffmpeg->target_linesizes[0]);
|
|
|
|
return true;
|
|
}
|
|
|
|
camera_driver_t camera_ffmpeg = {
|
|
ffmpeg_camera_init,
|
|
ffmpeg_camera_free,
|
|
ffmpeg_camera_start,
|
|
ffmpeg_camera_stop,
|
|
ffmpeg_camera_poll,
|
|
"ffmpeg",
|
|
};
|
|
|
|
#endif /* LIBAVFORMAT_VERSION_MAJOR >= 58 */
|