diff --git a/content/media/fmp4/ffmpeg/FFmpegDataDecoder.cpp b/content/media/fmp4/ffmpeg/FFmpegDataDecoder.cpp index 0068e83b8e29..cfed3ac59d49 100644 --- a/content/media/fmp4/ffmpeg/FFmpegDataDecoder.cpp +++ b/content/media/fmp4/ffmpeg/FFmpegDataDecoder.cpp @@ -87,6 +87,9 @@ FFmpegDataDecoder::Init() // FFmpeg will call back to this to negotiate a video pixel format. mCodecContext.get_format = ChoosePixelFormat; + mCodecContext.extradata = mExtraData.begin(); + mCodecContext.extradata_size = mExtraData.length(); + AVDictionary* opts = nullptr; if (avcodec_open2(&mCodecContext, codec, &opts) < 0) { NS_WARNING("Couldn't initialise ffmpeg decoder"); diff --git a/content/media/fmp4/ffmpeg/FFmpegDataDecoder.h b/content/media/fmp4/ffmpeg/FFmpegDataDecoder.h index 7816ab00e3be..1bebdb140858 100644 --- a/content/media/fmp4/ffmpeg/FFmpegDataDecoder.h +++ b/content/media/fmp4/ffmpeg/FFmpegDataDecoder.h @@ -10,6 +10,7 @@ #include "FFmpegDecoderModule.h" #include "FFmpegRuntimeLinker.h" #include "FFmpegCompat.h" +#include "mozilla/Vector.h" namespace mozilla { @@ -31,6 +32,7 @@ public: protected: MediaTaskQueue* mTaskQueue; AVCodecContext mCodecContext; + Vector mExtraData; private: static bool sFFmpegInitDone; diff --git a/content/media/fmp4/ffmpeg/FFmpegH264Decoder.cpp b/content/media/fmp4/ffmpeg/FFmpegH264Decoder.cpp index 68c633e5267c..25bb9bfc0dcb 100644 --- a/content/media/fmp4/ffmpeg/FFmpegH264Decoder.cpp +++ b/content/media/fmp4/ffmpeg/FFmpegH264Decoder.cpp @@ -33,6 +33,7 @@ FFmpegH264Decoder::FFmpegH264Decoder( , mImageContainer(aImageContainer) { MOZ_COUNT_CTOR(FFmpegH264Decoder); + mExtraData.append(aConfig.extra_data.begin(), aConfig.extra_data.length()); } nsresult diff --git a/content/media/fmp4/wmf/WMFAudioOutputSource.cpp b/content/media/fmp4/wmf/WMFAudioOutputSource.cpp index 575747279a7e..829176b7c777 100644 --- a/content/media/fmp4/wmf/WMFAudioOutputSource.cpp +++ b/content/media/fmp4/wmf/WMFAudioOutputSource.cpp @@ -5,6 +5,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "WMFAudioOutputSource.h" +#include "mp4_demuxer/DecoderData.h" #include "VideoUtils.h" #include "WMFUtils.h" #include "nsTArray.h" @@ -127,6 +128,14 @@ WMFAudioOutputSource::Init() return decoder.forget(); } +HRESULT +WMFAudioOutputSource::Input(mp4_demuxer::MP4Sample* aSample) +{ + const uint8_t* data = reinterpret_cast(aSample->data); + uint32_t length = aSample->size; + return mDecoder->Input(data, length, aSample->composition_timestamp); +} + HRESULT WMFAudioOutputSource::Output(int64_t aStreamOffset, nsAutoPtr& aOutData) diff --git a/content/media/fmp4/wmf/WMFAudioOutputSource.h b/content/media/fmp4/wmf/WMFAudioOutputSource.h index 73d326613a5b..e3298057f697 100644 --- a/content/media/fmp4/wmf/WMFAudioOutputSource.h +++ b/content/media/fmp4/wmf/WMFAudioOutputSource.h @@ -22,6 +22,8 @@ public: virtual TemporaryRef Init() MOZ_OVERRIDE; + virtual HRESULT Input(mp4_demuxer::MP4Sample* aSample) MOZ_OVERRIDE; + // Note WMF's AAC decoder sometimes output negatively timestamped samples, // presumably they're the preroll samples, and we strip them. We may return // a null aOutput in this case. diff --git a/content/media/fmp4/wmf/WMFDecoderModule.cpp b/content/media/fmp4/wmf/WMFDecoderModule.cpp index 174c336d5d65..6542a1ba4762 100644 --- a/content/media/fmp4/wmf/WMFDecoderModule.cpp +++ b/content/media/fmp4/wmf/WMFDecoderModule.cpp @@ -70,7 +70,8 @@ WMFDecoderModule::CreateH264Decoder(const mp4_demuxer::VideoDecoderConfig& aConf MediaTaskQueue* aVideoTaskQueue, MediaDataDecoderCallback* aCallback) { - return new WMFMediaDataDecoder(new WMFVideoOutputSource(aLayersBackend, + return new WMFMediaDataDecoder(new WMFVideoOutputSource(aConfig, + aLayersBackend, aImageContainer, sDXVAEnabled), aVideoTaskQueue, diff --git a/content/media/fmp4/wmf/WMFMediaDataDecoder.cpp b/content/media/fmp4/wmf/WMFMediaDataDecoder.cpp index 7ee0ee13172c..b2c7df625ec7 100644 --- a/content/media/fmp4/wmf/WMFMediaDataDecoder.cpp +++ b/content/media/fmp4/wmf/WMFMediaDataDecoder.cpp @@ -67,11 +67,9 @@ WMFMediaDataDecoder::Input(mp4_demuxer::MP4Sample* aSample) void WMFMediaDataDecoder::ProcessDecode(mp4_demuxer::MP4Sample* aSample) { - const uint8_t* data = reinterpret_cast(aSample->data); - uint32_t length = aSample->size; - HRESULT hr = mDecoder->Input(data, length, aSample->composition_timestamp); + HRESULT hr = mSource->Input(aSample); if (FAILED(hr)) { - NS_WARNING("WMFAudioDecoder failed to input data"); + NS_WARNING("WMFOutputSource rejected sample"); mCallback->Error(); return; } diff --git a/content/media/fmp4/wmf/WMFMediaDataDecoder.h b/content/media/fmp4/wmf/WMFMediaDataDecoder.h index e5a5019b2b24..28312411e3e1 100644 --- a/content/media/fmp4/wmf/WMFMediaDataDecoder.h +++ b/content/media/fmp4/wmf/WMFMediaDataDecoder.h @@ -13,6 +13,8 @@ #include "MFTDecoder.h" #include "mozilla/RefPtr.h" +class mp4_demuxer::MP4Sample; + namespace mozilla { // Encapsulates the initialization of the MFTDecoder appropriate for decoding @@ -26,6 +28,11 @@ public: // Returns nullptr on failure. virtual TemporaryRef Init() = 0; + // Submit a compressed sample for decoding. + // This should forward to the MFTDecoder after performing + // any required sample formatting. + virtual HRESULT Input(mp4_demuxer::MP4Sample* aSample) = 0; + // Produces decoded output, if possible. Blocks until output can be produced, // or until no more is able to be produced. // Returns S_OK on success, or MF_E_TRANSFORM_NEED_MORE_INPUT if there's not diff --git a/content/media/fmp4/wmf/WMFVideoOutputSource.cpp b/content/media/fmp4/wmf/WMFVideoOutputSource.cpp index c1fbd8d98651..c4a31b0cd68a 100644 --- a/content/media/fmp4/wmf/WMFVideoOutputSource.cpp +++ b/content/media/fmp4/wmf/WMFVideoOutputSource.cpp @@ -13,6 +13,8 @@ #include "nsThreadUtils.h" #include "Layers.h" #include "mozilla/layers/LayersTypes.h" +#include "mp4_demuxer/AnnexB.h" +#include "mp4_demuxer/DecoderData.h" #include "prlog.h" #include "gfx2DGlue.h" @@ -30,12 +32,15 @@ using mozilla::layers::LayersBackend; namespace mozilla { -WMFVideoOutputSource::WMFVideoOutputSource(mozilla::layers::LayersBackend aLayersBackend, - mozilla::layers::ImageContainer* aImageContainer, - bool aDXVAEnabled) +WMFVideoOutputSource::WMFVideoOutputSource( + const mp4_demuxer::VideoDecoderConfig& aConfig, + mozilla::layers::LayersBackend aLayersBackend, + mozilla::layers::ImageContainer* aImageContainer, + bool aDXVAEnabled) : mVideoStride(0) , mVideoWidth(0) , mVideoHeight(0) + , mConfig(aConfig) , mImageContainer(aImageContainer) , mDXVAEnabled(aDXVAEnabled) , mLayersBackend(aLayersBackend) @@ -138,6 +143,17 @@ WMFVideoOutputSource::Init() return decoder.forget(); } +HRESULT +WMFVideoOutputSource::Input(mp4_demuxer::MP4Sample* aSample) +{ + // We must prepare samples in AVC Annex B. + mp4_demuxer::AnnexB::ConvertSample(aSample, mConfig.annex_b); + // Forward sample data to the decoder. + const uint8_t* data = reinterpret_cast(aSample->data); + uint32_t length = aSample->size; + return mDecoder->Input(data, length, aSample->composition_timestamp); +} + HRESULT WMFVideoOutputSource::ConfigureVideoFrameGeometry() { diff --git a/content/media/fmp4/wmf/WMFVideoOutputSource.h b/content/media/fmp4/wmf/WMFVideoOutputSource.h index 07fee720c6ac..c74af3f013d9 100644 --- a/content/media/fmp4/wmf/WMFVideoOutputSource.h +++ b/content/media/fmp4/wmf/WMFVideoOutputSource.h @@ -20,13 +20,16 @@ class DXVA2Manager; class WMFVideoOutputSource : public WMFOutputSource { public: - WMFVideoOutputSource(mozilla::layers::LayersBackend aLayersBackend, + WMFVideoOutputSource(const mp4_demuxer::VideoDecoderConfig& aConfig, + mozilla::layers::LayersBackend aLayersBackend, mozilla::layers::ImageContainer* aImageContainer, bool aDXVAEnabled); ~WMFVideoOutputSource(); virtual TemporaryRef Init() MOZ_OVERRIDE; + virtual HRESULT Input(mp4_demuxer::MP4Sample* aSample) MOZ_OVERRIDE; + virtual HRESULT Output(int64_t aStreamOffset, nsAutoPtr& aOutput) MOZ_OVERRIDE; @@ -51,6 +54,8 @@ private: uint32_t mVideoHeight; nsIntRect mPictureRegion; + const mp4_demuxer::VideoDecoderConfig& mConfig; + RefPtr mDecoder; RefPtr mImageContainer; nsAutoPtr mDXVA2Manager; diff --git a/media/libstagefright/binding/AnnexB.cpp b/media/libstagefright/binding/AnnexB.cpp index d676fa6b33e3..80cc5c696d5f 100644 --- a/media/libstagefright/binding/AnnexB.cpp +++ b/media/libstagefright/binding/AnnexB.cpp @@ -2,6 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "mozilla/ArrayUtils.h" #include "mp4_demuxer/AnnexB.h" #include "mp4_demuxer/ByteReader.h" #include "mp4_demuxer/DecoderData.h" @@ -13,6 +14,21 @@ namespace mp4_demuxer static const uint8_t kAnnexBDelimiter[] = { 0, 0, 0, 1 }; +void +AnnexB::ConvertSample(MP4Sample* aSample, + const mozilla::Vector& annexB) +{ + MOZ_ASSERT(aSample); + MOZ_ASSERT(aSample->data); + MOZ_ASSERT(aSample->size >= ArrayLength(kAnnexBDelimiter)); + // Overwrite the NAL length with the Annex B separator. + memcpy(aSample->data, kAnnexBDelimiter, ArrayLength(kAnnexBDelimiter)); + // Prepend the Annex B header with SPS and PPS tables to keyframes. + if (aSample->is_sync_point) { + aSample->Prepend(annexB.begin(), annexB.length()); + } +} + Vector AnnexB::ConvertExtraDataToAnnexB(mozilla::Vector& aExtraData) { @@ -36,9 +52,9 @@ AnnexB::ConvertExtraDataToAnnexB(mozilla::Vector& aExtraData) ByteReader reader(aExtraData); const uint8_t* ptr = reader.Read(5); if (ptr && ptr[0] == 1) { - // Append SPS then PSP - ConvertSpsOrPsp(reader, reader.ReadU8() & 31, &annexB); - ConvertSpsOrPsp(reader, reader.ReadU8(), &annexB); + // Append SPS then PPS + ConvertSPSOrPPS(reader, reader.ReadU8() & 31, &annexB); + ConvertSPSOrPPS(reader, reader.ReadU8(), &annexB); MOZ_ASSERT(!reader.Remaining()); } @@ -47,7 +63,7 @@ AnnexB::ConvertExtraDataToAnnexB(mozilla::Vector& aExtraData) } void -AnnexB::ConvertSpsOrPsp(ByteReader& aReader, uint8_t aCount, +AnnexB::ConvertSPSOrPPS(ByteReader& aReader, uint8_t aCount, Vector* aAnnexB) { for (int i = 0; i < aCount; i++) { @@ -62,4 +78,5 @@ AnnexB::ConvertSpsOrPsp(ByteReader& aReader, uint8_t aCount, aAnnexB->append(ptr, length); } } -} + +} // namespace mp4_demuxer diff --git a/media/libstagefright/binding/DecoderData.cpp b/media/libstagefright/binding/DecoderData.cpp index e0845a2adbf5..c86bf08668a8 100644 --- a/media/libstagefright/binding/DecoderData.cpp +++ b/media/libstagefright/binding/DecoderData.cpp @@ -83,7 +83,7 @@ VideoDecoderConfig::Update(sp& aMetaData, const char* aMimeType) uint32_t type; if (aMetaData->findData(kKeyAVCC, &type, &data, &size)) { - mozilla::Vector extra_data; + extra_data.clear(); extra_data.append(reinterpret_cast(data), size); annex_b = AnnexB::ConvertExtraDataToAnnexB(extra_data); } diff --git a/media/libstagefright/binding/include/mp4_demuxer/AnnexB.h b/media/libstagefright/binding/include/mp4_demuxer/AnnexB.h index ed631daa448e..f4be4df5f49b 100644 --- a/media/libstagefright/binding/include/mp4_demuxer/AnnexB.h +++ b/media/libstagefright/binding/include/mp4_demuxer/AnnexB.h @@ -2,24 +2,33 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#ifndef ANNEX_B_H_ -#define ANNEX_B_H_ +#ifndef MP4_DEMUXER_ANNEX_B_H_ +#define MP4_DEMUXER_ANNEX_B_H_ #include "mozilla/Vector.h" namespace mp4_demuxer { class ByteReader; +class MP4Sample; + class AnnexB { public: + // Convert a sample from NAL unit syntax to Annex B. + static void ConvertSample(MP4Sample* aSample, + const mozilla::Vector& annexB); + + // Parse an AVCC box and construct the Annex B sample header. static mozilla::Vector ConvertExtraDataToAnnexB( mozilla::Vector& aExtraData); private: - static void ConvertSpsOrPsp(ByteReader& aReader, uint8_t aCount, + // AVCC box parser helper. + static void ConvertSPSOrPPS(ByteReader& aReader, uint8_t aCount, mozilla::Vector* aAnnexB); }; -} -#endif +} // namespace mp4_demuxer + +#endif // MP4_DEMUXER_ANNEX_B_H_ diff --git a/media/libstagefright/binding/include/mp4_demuxer/DecoderData.h b/media/libstagefright/binding/include/mp4_demuxer/DecoderData.h index 1c39f81da52a..81d38c4fcee3 100644 --- a/media/libstagefright/binding/include/mp4_demuxer/DecoderData.h +++ b/media/libstagefright/binding/include/mp4_demuxer/DecoderData.h @@ -67,7 +67,8 @@ public: int32_t display_width; int32_t display_height; - mozilla::Vector annex_b; + mozilla::Vector extra_data; // Unparsed AVCDecoderConfig payload. + mozilla::Vector annex_b; // Parsed version for sample prepend. void Update(stagefright::sp& aMetaData, const char* aMimeType); bool IsValid(); diff --git a/media/libstagefright/binding/include/mp4_demuxer/mp4_demuxer.h b/media/libstagefright/binding/include/mp4_demuxer/mp4_demuxer.h index 8b9953de33b4..a60af43cbe16 100644 --- a/media/libstagefright/binding/include/mp4_demuxer/mp4_demuxer.h +++ b/media/libstagefright/binding/include/mp4_demuxer/mp4_demuxer.h @@ -44,8 +44,8 @@ public: void SeekAudio(Microseconds aTime); void SeekVideo(Microseconds aTime); - // DemuxAudioSample and DemuxVideoSample functions return nullptr on end of - // stream or error. + // DemuxAudioSample and DemuxVideoSample functions + // return nullptr on end of stream or error. MP4Sample* DemuxAudioSample(); MP4Sample* DemuxVideoSample(); diff --git a/media/libstagefright/binding/mp4_demuxer.cpp b/media/libstagefright/binding/mp4_demuxer.cpp index f55fc7fd3a60..51c2cef53dfd 100644 --- a/media/libstagefright/binding/mp4_demuxer.cpp +++ b/media/libstagefright/binding/mp4_demuxer.cpp @@ -65,7 +65,8 @@ private: nsAutoPtr mSource; }; -MP4Demuxer::MP4Demuxer(Stream* source) : mPrivate(new StageFrightPrivate()) +MP4Demuxer::MP4Demuxer(Stream* source) + : mPrivate(new StageFrightPrivate()) { mPrivate->mExtractor = new MPEG4Extractor(new DataSourceAdapter(source)); } @@ -180,11 +181,7 @@ MP4Demuxer::DemuxVideoSample() sample->Update(); - if (sample->is_sync_point) { - sample->Prepend(mVideoConfig.annex_b.begin(), - mVideoConfig.annex_b.length()); - } - return sample.forget(); } -} + +} // namespace mp4_demuxer diff --git a/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp b/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp index cc600f698d9c..b53584bb498f 100644 --- a/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp +++ b/media/libstagefright/frameworks/av/media/libstagefright/MPEG4Extractor.cpp @@ -3301,11 +3301,10 @@ status_t MPEG4Source::read( } CHECK(dstOffset + 4 <= mBuffer->size()); - - dstData[dstOffset++] = 0; - dstData[dstOffset++] = 0; - dstData[dstOffset++] = 0; - dstData[dstOffset++] = 1; + dstData[dstOffset++] = (uint8_t) (nalLength >> 24); + dstData[dstOffset++] = (uint8_t) (nalLength >> 16); + dstData[dstOffset++] = (uint8_t) (nalLength >> 8); + dstData[dstOffset++] = (uint8_t) nalLength; memcpy(&dstData[dstOffset], &mSrcBuffer[srcOffset], nalLength); srcOffset += nalLength; dstOffset += nalLength; @@ -3589,11 +3588,10 @@ status_t MPEG4Source::fragmentedRead( } CHECK(dstOffset + 4 <= mBuffer->size()); - - dstData[dstOffset++] = 0; - dstData[dstOffset++] = 0; - dstData[dstOffset++] = 0; - dstData[dstOffset++] = 1; + dstData[dstOffset++] = (uint8_t) (nalLength >> 24); + dstData[dstOffset++] = (uint8_t) (nalLength >> 16); + dstData[dstOffset++] = (uint8_t) (nalLength >> 8); + dstData[dstOffset++] = (uint8_t) nalLength; memcpy(&dstData[dstOffset], &mSrcBuffer[srcOffset], nalLength); srcOffset += nalLength; dstOffset += nalLength; diff --git a/media/libstagefright/moz.build b/media/libstagefright/moz.build index ba2295f4c239..f991d7f283b4 100644 --- a/media/libstagefright/moz.build +++ b/media/libstagefright/moz.build @@ -46,6 +46,7 @@ if CONFIG['OS_TARGET'] != 'Android': ] EXPORTS.mp4_demuxer += [ + 'binding/include/mp4_demuxer/AnnexB.h', 'binding/include/mp4_demuxer/DecoderData.h', 'binding/include/mp4_demuxer/mp4_demuxer.h', ]