#include #include "cutscene/ffmpeg/FFMPEGDecoder.h" #include "cfile/cfile.h" #include "libs/ffmpeg/FFmpegContext.h" #include "cutscene/ffmpeg/internal.h" #include "cutscene/ffmpeg/AudioDecoder.h" #include "cutscene/ffmpeg/VideoDecoder.h" #include "cutscene/ffmpeg/SubtitleDecoder.h" #include "FFMPEGDecoder.h" #include "localization/localize.h" using namespace libs::ffmpeg; namespace { using namespace cutscene; using namespace cutscene::ffmpeg; class AVPacketScope { AVPacket* _packet; public: explicit AVPacketScope(AVPacket* av_packet) : _packet(av_packet) { } AVPacketScope(const AVPacketScope&) = delete; AVPacketScope& operator=(const AVPacketScope&) = delete; ~AVPacketScope() { av_packet_unref(_packet); } }; const char* CHECKED_EXTENSIONS[] = {"webm", "mp4", "ogg", "png", // This is designed to be used with APNG animations "mve"}; const char* CHECKED_SUBT_EXTENSIONS[] = {"srt"}; double getFrameRate(AVStream* stream, AVCodecContext* codecCtx) { #if LIBAVCODEC_VERSION_INT > AV_VERSION_INT(58, 3, 102) auto fps = av_q2d(stream->r_frame_rate); #else auto fps = av_q2d(av_stream_get_r_frame_rate(stream)); #endif if (fps < 0.000001) { fps = av_q2d(stream->avg_frame_rate); } if (fps < 0.000001) { fps = 1.0 / av_q2d(codecCtx->time_base); } return fps; } CodecContextParameters getCodecParameters(AVStream* stream) { CodecContextParameters paras; #if LIBAVCODEC_VERSION_INT > AV_VERSION_INT(57, 24, 255) paras.width = stream->codecpar->width; paras.height = stream->codecpar->height; paras.pixel_format = (AVPixelFormat)stream->codecpar->format; #if LIBAVCODEC_VERSION_INT > AV_VERSION_INT(59, 36, 255) paras.channel_layout = stream->codecpar->ch_layout.u.mask; #else paras.channel_layout = stream->codecpar->channel_layout; #endif paras.sample_rate = stream->codecpar->sample_rate; paras.audio_format = (AVSampleFormat)stream->codecpar->format; paras.codec_id = stream->codecpar->codec_id; #else paras.width = stream->codec->width; paras.height = stream->codec->height; paras.pixel_format = stream->codec->pix_fmt; paras.channel_layout = stream->codec->channel_layout; paras.sample_rate = stream->codec->sample_rate; paras.audio_format = stream->codec->sample_fmt; paras.codec_id = stream->codec->codec_id; #endif return paras; } SCP_string normalizeLanguage(const char* langauge_name) { if (!stricmp(langauge_name, "eng")) { return "English"; } if (!stricmp(langauge_name, "ger")) { return "German"; } if (!stricmp(langauge_name, "spa")) { return "Spanish"; } // Print to log so that we can find the actual value more easily later mprintf(("FFmpeg log: Found unknown language value '%s'!\n", langauge_name)); // Default to english for everything else return "English"; } /** * @brief Given a FFmpeg pixel format returns the format it should be converted to * @param in_fmt The input format * @return The output format */ AVPixelFormat getConversionFormat(AVPixelFormat in_fmt) { switch (in_fmt) { case AV_PIX_FMT_RGB24: case AV_PIX_FMT_BGR24: case AV_PIX_FMT_BGR8: case AV_PIX_FMT_BGR4: case AV_PIX_FMT_BGR4_BYTE: case AV_PIX_FMT_RGB8: case AV_PIX_FMT_RGB4: case AV_PIX_FMT_RGB4_BYTE: case AV_PIX_FMT_RGB48BE: case AV_PIX_FMT_RGB48LE: case AV_PIX_FMT_RGB565BE: case AV_PIX_FMT_RGB565LE: case AV_PIX_FMT_RGB555BE: case AV_PIX_FMT_RGB555LE: case AV_PIX_FMT_BGR565BE: case AV_PIX_FMT_BGR565LE: case AV_PIX_FMT_BGR555BE: case AV_PIX_FMT_BGR555LE: case AV_PIX_FMT_RGB444LE: case AV_PIX_FMT_RGB444BE: case AV_PIX_FMT_BGR444LE: case AV_PIX_FMT_BGR444BE: case AV_PIX_FMT_BGR48BE: case AV_PIX_FMT_BGR48LE: case AV_PIX_FMT_0RGB: case AV_PIX_FMT_RGB0: case AV_PIX_FMT_0BGR: case AV_PIX_FMT_BGR0: return AV_PIX_FMT_BGR24; case AV_PIX_FMT_RGBA64BE: case AV_PIX_FMT_RGBA64LE: case AV_PIX_FMT_BGRA64BE: case AV_PIX_FMT_BGRA64LE: case AV_PIX_FMT_ARGB: case AV_PIX_FMT_RGBA: case AV_PIX_FMT_ABGR: case AV_PIX_FMT_BGRA: return AV_PIX_FMT_BGRA; default: // Assume that everything else is converted to YUV 420 return AV_PIX_FMT_YUV420P; } } FramePixelFormat getPixelFormat(AVPixelFormat fmt) { switch (fmt) { case AV_PIX_FMT_BGR24: return FramePixelFormat::BGR; case AV_PIX_FMT_BGRA: return FramePixelFormat::BGRA; default: return FramePixelFormat::YUV420; } } } namespace cutscene { namespace ffmpeg { struct InputStream { std::unique_ptr m_ctx; }; FFMPEGDecoder::FFMPEGDecoder() { } FFMPEGDecoder::~FFMPEGDecoder() { } namespace { std::unique_ptr openStream(const SCP_string& name) { // Only check the root and movies folders int dirType; if (cf_exists_full(name.c_str(), CF_TYPE_ROOT)) { dirType = CF_TYPE_ROOT; } else if (cf_exists_full(name.c_str(), CF_TYPE_MOVIES)) { dirType = CF_TYPE_MOVIES; } else { return nullptr; } std::unique_ptr input(new InputStream()); try { // Create the FFmpeg context input->m_ctx = FFmpegContext::createContext(name, dirType); // This may fail, in that case return null if (!input->m_ctx) { return nullptr; } } catch (const FFmpegException& e) { mprintf(("Error opening %s: %s\n", name.c_str(), e.what())); return nullptr; } return input; } std::unique_ptr initializeStatus(std::unique_ptr& stream, std::unique_ptr& subt, const PlaybackProperties& properties) { if (subt && properties.looping) { mprintf(("FFmpeg: External subtitles and looping movies are not supported!\n")); return nullptr; } std::unique_ptr status(new DecoderStatus()); auto ctx = stream->m_ctx->ctx(); auto videoStream = av_find_best_stream(ctx, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); if (videoStream < 0) { if (videoStream == AVERROR_STREAM_NOT_FOUND) { mprintf(("FFmpeg: No video stream found in file!\n")); } else if (videoStream == AVERROR_DECODER_NOT_FOUND) { mprintf(("FFmpeg: Codec for video stream could not be found!\n")); } else { mprintf(("FFmpeg: Unknown error while finding video stream!\n")); } return nullptr; } status->videoStreamIndex = videoStream; status->videoStream = ctx->streams[videoStream]; status->videoCodecPars = getCodecParameters(status->videoStream); status->videoCodec = avcodec_find_decoder(status->videoCodecPars.codec_id); if ( !status->videoCodec ) { mprintf(("FFmpeg: Cannot find decoder for video stream!\n")); return nullptr; } if (properties.looping && status->videoCodecPars.codec_id == AV_CODEC_ID_INTERPLAY_VIDEO) { mprintf(("FFmpeg: Looping is not supported for inteplay (MVE) movies!\n")); return nullptr; } int audioStream = -1; if (properties.with_audio) { audioStream = av_find_best_stream(ctx, AVMEDIA_TYPE_AUDIO, -1, videoStream, nullptr, 0); if (audioStream < 0) { if (audioStream == AVERROR_STREAM_NOT_FOUND) { mprintf(("FFmpeg: No audio stream found in file!\n")); } else if (audioStream == AVERROR_DECODER_NOT_FOUND) { mprintf(("FFmpeg: Codec for audio stream could not be found!\n")); } else { mprintf(("FFmpeg: Unknown error while finding audio stream!\n")); } } else { status->audioStreamIndex = audioStream; status->audioStream = ctx->streams[audioStream]; status->audioCodecPars = getCodecParameters(status->audioStream); status->audioCodec = avcodec_find_decoder(status->audioCodecPars.codec_id); if ( !status->audioCodec ) { mprintf(("FFmpeg: Cannot find decoder for audio stream!\n")); status->audioStreamIndex = -1; audioStream = -1; } } } if (subt) { auto subtCtx = subt->m_ctx->ctx(); auto subtStream = av_find_best_stream(subtCtx, AVMEDIA_TYPE_SUBTITLE, -1, -1, nullptr, 0); if (subtStream < 0) { if (subtStream == AVERROR_STREAM_NOT_FOUND) { mprintf(("FFmpeg: No subtitle stream found in subtitle file!\n")); } else if (subtStream == AVERROR_DECODER_NOT_FOUND) { mprintf(("FFmpeg: Codec for subtitle stream could not be found!\n")); } else { mprintf(("FFmpeg: Unknown error while finding subtitle stream!\n")); } } else { status->subtitleStreamIndex = subtStream; status->subtitleStream = subtCtx->streams[subtStream]; status->externalSubtitles = true; // Not really sure if this makes sense for subtitles but it shouldn't break anything... status->subtitleCodecPars = getCodecParameters(status->subtitleStream); status->subtitleCodec = avcodec_find_decoder(status->subtitleCodecPars.codec_id); } } if (status->subtitleStream == nullptr) { // We don't have an external subtitle stream or loading from that file has failed auto& current_language = Lcl_languages[lcl_get_current_lang_index()]; for (uint32_t i = 0; i < ctx->nb_streams; ++i) { auto test_stream = ctx->streams[i]; #if LIBAVCODEC_VERSION_INT > AV_VERSION_INT(57, 24, 255) auto& pars = test_stream->codecpar; if (!(pars->codec_type == AVMEDIA_TYPE_SUBTITLE)) { continue; } #endif AVDictionaryEntry* tag = nullptr; if ((tag = av_dict_get(test_stream->metadata, "language", nullptr, 0)) == nullptr) { continue; } auto normalized_language = normalizeLanguage(tag->value); if (!stricmp(normalized_language.c_str(), current_language.lang_name)) { status->subtitleStreamIndex = i; status->subtitleStream = test_stream; break; } } } int err; #if LIBAVCODEC_VERSION_INT > AV_VERSION_INT(57, 24, 255) status->videoCodecCtx = avcodec_alloc_context3(status->videoCodec); // Copy codec parameters from input stream to output codec context err = avcodec_parameters_to_context(status->videoCodecCtx, status->videoStream->codecpar); if (err < 0) { char errorStr[512]; av_strerror(err, errorStr, sizeof(errorStr)); mprintf(("FFMPEG: Failed to copy context parameters! Error: %s\n", errorStr)); return nullptr; } #else status->videoCodecCtx = status->videoStream->codec; #endif err = avcodec_open2(status->videoCodecCtx, status->videoCodec, nullptr); if (err < 0) { char errorStr[512]; av_strerror(err, errorStr, sizeof(errorStr)); mprintf(("FFMPEG: Failed to open video codec! Error: %s\n", errorStr)); return nullptr; } mprintf(("FFmpeg: Using video codec %s (%s).\n", status->videoCodec->long_name ? status->videoCodec->long_name : "", status->videoCodec->name ? status->videoCodec->name : "")); // Now initialize audio, if this fails it's not a fatal error if (audioStream >= 0) { #if LIBAVCODEC_VERSION_INT > AV_VERSION_INT(57, 24, 255) status->audioCodecCtx = avcodec_alloc_context3(status->audioCodec); err = avcodec_parameters_to_context(status->audioCodecCtx, status->audioStream->codecpar); if (err < 0) { char errorStr[512]; av_strerror(err, errorStr, sizeof(errorStr)); mprintf(("FFMPEG: Failed to copy context parameters! Error: %s\n", errorStr)); return nullptr; } #else status->audioCodecCtx = status->audioStream->codec; #endif err = avcodec_open2(status->audioCodecCtx, status->audioCodec, nullptr); if (err < 0) { char errorStr[512]; av_strerror(err, errorStr, sizeof(errorStr)); mprintf(("FFMPEG: Failed to open audio codec! Error: %s\n", errorStr)); } mprintf(("FFmpeg: Using audio codec %s (%s).\n", status->audioCodec->long_name ? status->audioCodec->long_name : "", status->audioCodec->name ? status->audioCodec->name : "")); } if (status->subtitleStream != nullptr) { #if LIBAVCODEC_VERSION_INT > AV_VERSION_INT(57, 24, 255) status->subtitleCodecCtx = avcodec_alloc_context3(status->subtitleCodec); err = avcodec_parameters_to_context(status->subtitleCodecCtx, status->subtitleStream->codecpar); if (err < 0) { char errorStr[512]; av_strerror(err, errorStr, sizeof(errorStr)); mprintf(("FFMPEG: Failed to copy context parameters! Error: %s\n", errorStr)); return nullptr; } #else status->subtitleCodecCtx = status->subtitleStream->codec; #endif err = avcodec_open2(status->subtitleCodecCtx, status->subtitleCodec, nullptr); if (err < 0) { char errorStr[512]; av_strerror(err, errorStr, sizeof(errorStr)); mprintf(("FFMPEG: Failed to open subtitle codec! Error: %s\n", errorStr)); } mprintf(("FFmpeg: Using subtitle codec %s (%s).\n", status->subtitleCodec->long_name ? status->subtitleCodec->long_name : "", status->subtitleCodec->name ? status->subtitleCodec->name : "")); } return status; } std::unique_ptr openInputStream(const SCP_string& name) { // Check a list of extensions we might use // The actual format of the file may be whatever FFmpeg supports for (auto ext : CHECKED_EXTENSIONS) { auto fileName = name + "." + ext; auto input = openStream(fileName); if (input) { return input; } } return nullptr; } std::unique_ptr openSubtitleStream(const SCP_string& name) { auto& current_language = Lcl_languages[lcl_get_current_lang_index()]; for (auto ext : CHECKED_SUBT_EXTENSIONS) { SCP_string fileName; if (strlen(current_language.lang_ext) == 0) { fileName = name + "." + ext; } else { fileName = name + "-" + current_language.lang_ext + "." + ext; } auto input = openStream(fileName); if (input) { return input; } } return nullptr; } } bool FFMPEGDecoder::initialize(const SCP_string& fileName, const PlaybackProperties& properties) { SCP_string movieName = fileName; // First make the file name lower case SCP_tolower(movieName); // Then remove the extension size_t dotPos = movieName.find('.'); if (dotPos != SCP_string::npos) { movieName.resize(dotPos); } // Try to open the input stream auto input = openInputStream(movieName); if (!input) { return false; } auto subt = openSubtitleStream(movieName); // We now have a valid input stream, try to find the correct streams auto status = initializeStatus(input, subt, properties); if (!status) { return false; } // Buffer ~ 2 seconds of video and audio initializeQueues(static_cast(ceil(getFrameRate(status->videoStream, status->videoCodecCtx))) * 2); // We're done, now just put the pointer into this std::swap(m_input, input); std::swap(m_subtitleInput, subt); std::swap(m_status, status); m_properties = properties; return true; } MovieProperties FFMPEGDecoder::getProperties() const { MovieProperties props; props.size.width = static_cast(m_status->videoCodecPars.width); props.size.height = static_cast(m_status->videoCodecPars.height); props.fps = static_cast(getFrameRate(m_status->videoStream, m_status->videoCodecCtx)); props.duration = static_cast(m_status->videoStream->duration * av_q2d(m_status->videoStream->time_base)); props.pixelFormat = getPixelFormat(getConversionFormat(m_status->videoCodecPars.pixel_format)); return props; } void FFMPEGDecoder::startDecoding() { std::unique_ptr videoDecoder(new VideoDecoder(m_status.get(), getConversionFormat(m_status->videoCodecPars.pixel_format))); std::unique_ptr audioDecoder; std::unique_ptr subtitleDecoder; if (hasAudio()) { audioDecoder.reset(new AudioDecoder(m_status.get())); } if (hasSubtitles()) { subtitleDecoder.reset(new SubtitleDecoder(m_status.get())); } std::unique_ptr subtitle_thread; if (hasExternalSubtitle()) { // Start a new thread for the subtitles since otherwise we would need to make sure that neither of the input // streames starves while the other waits for more space in the queues subtitle_thread.reset( new std::thread([this, &subtitleDecoder]() { runSubtitleDecoder(subtitleDecoder.get()); })); } auto ctx = m_input->m_ctx->ctx(); AVPacket packet; do { if (m_properties.looping) { // Only seek if we are looping the movie. The MVE code doesn't support seeking to this function may not be // called there auto seek_err = av_seek_frame(ctx, -1, 0, AVSEEK_FLAG_BACKWARD); if (seek_err < 0) { char errorStr[512]; av_strerror(seek_err, errorStr, sizeof(errorStr)); mprintf(("FFMPEG: Failed to seek to start of movie file! Error: %s\n", errorStr)); break; } } videoDecoder->flushBuffers(); if (audioDecoder) { audioDecoder->flushBuffers(); } if (subtitleDecoder) { subtitleDecoder->flushBuffers(); } while (isDecoding()) { auto read_err = av_read_frame(ctx, &packet); AVPacketScope scope(&packet); if (read_err < 0) { if (read_err == AVERROR_EOF) { // Finished reading -> break out of loop break; } else if (avio_feof(ctx->pb) != 0) { // Also EOF! break; } else { // Some kind of other error, try to continue reading char errorStr[512]; av_strerror(read_err, errorStr, sizeof(errorStr)); mprintf(("FFMPEG: Failed to read frame! Error: %s\n", errorStr)); // Skip packet continue; } } if (packet.stream_index == m_status->videoStreamIndex) { videoDecoder->decodePacket(&packet); VideoFramePtr ptr; while ((ptr = videoDecoder->getFrame()) != nullptr) { pushFrameData(std::move(ptr)); } } else if (audioDecoder && packet.stream_index == m_status->audioStreamIndex) { audioDecoder->decodePacket(&packet); AudioFramePtr ptr; while ((ptr = audioDecoder->getFrame()) != nullptr) { pushAudioData(std::move(ptr)); } } else if (subtitleDecoder && packet.stream_index == m_status->subtitleStreamIndex) { subtitleDecoder->decodePacket(&packet); SubtitleFramePtr ptr; while ((ptr = subtitleDecoder->getFrame()) != nullptr) { pushSubtitleData(std::move(ptr)); } } } if (isDecoding()) { // If we are still alive then read the last frames from the decoders videoDecoder->finishDecoding(); VideoFramePtr video_ptr; while ((video_ptr = videoDecoder->getFrame()) != nullptr) { pushFrameData(std::move(video_ptr)); } if (audioDecoder) { audioDecoder->finishDecoding(); AudioFramePtr audio_frame; while ((audio_frame = audioDecoder->getFrame()) != nullptr) { pushAudioData(std::move(audio_frame)); } } } // If the video is being looped then we start again from the beginning } while (m_properties.looping && isDecoding()); // Stop the decoder first to let the subtitle thread know that it shouldn't continue stopDecoder(); if (subtitle_thread) { subtitle_thread->join(); } } bool FFMPEGDecoder::hasAudio() const { return m_status->audioStreamIndex >= 0; } bool FFMPEGDecoder::hasSubtitles() const { return m_status->subtitleStream != nullptr; } bool FFMPEGDecoder::hasExternalSubtitle() const { return m_status->externalSubtitles; } void FFMPEGDecoder::close() { if (m_status) { m_status = nullptr; } if (m_input) { // This will delete the InputStream pointer and free all data m_input = nullptr; } if (m_subtitleInput) { m_subtitleInput = nullptr; } } void FFMPEGDecoder::runSubtitleDecoder(SubtitleDecoder* decoder) { auto ctx = m_subtitleInput->m_ctx->ctx(); AVPacket packet; while (isDecoding()) { auto read_err = av_read_frame(ctx, &packet); AVPacketScope scope(&packet); if (read_err < 0) { if (read_err == AVERROR_EOF) { // Finished reading -> break out of loop break; } else if (avio_feof(ctx->pb) != 0) { // Also EOF! break; } else { // Some kind of other error, try to continue reading char errorStr[512]; av_strerror(read_err, errorStr, sizeof(errorStr)); mprintf(("FFMPEG: Failed to read frame! Error: %s\n", errorStr)); // Skip packet continue; } } if (packet.stream_index == m_status->subtitleStreamIndex) { decoder->decodePacket(&packet); SubtitleFramePtr ptr; while ((ptr = decoder->getFrame()) != nullptr) { pushSubtitleData(std::move(ptr)); } } } if (isDecoding()) { // If we are still alive then read the last frames from the decoders decoder->finishDecoding(); SubtitleFramePtr ptr; while ((ptr = decoder->getFrame()) != nullptr) { pushSubtitleData(std::move(ptr)); } } } } }