Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-13 21:35:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1300296: P2. Don't rely on MP4 container to properly report if a frame is a keyframe. r=kentuckyfriedtakahe

Browse Source 
There are too many cases where the MP4 is improperly muxed and frames are incorrectly reported as keyframe.
Instead we now look inside the H264 stream and check for IDR frames.

We also ensure that the first frame returned after a seek is always a true keyframe.

For plain MP4, seeking in those broken files will lead to broken A/V sync. The only way to fix this would be to check for the frame type when reading the samples table. However, this would require to read the entire stream which isn't viable.

MozReview-Commit-ID: Cpv5y7HVD0N

--HG--
extra : rebase_source : 2f92032fe39ed6ad6c2b82438f405040b5e7d30c
This commit is contained in:
Jean-Yves Avenard 2016-09-04 21:33:23 +10:00
parent 4b2bec7900
commit 75e90695b5
1 changed files with 90 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
dom/media/fmp4/MP4Demuxer.cpp
View File

@ -14,6 +14,7 @@
#include "mp4_demuxer/ResourceStream.h"
#include "mp4_demuxer/BufferStream.h"
#include "mp4_demuxer/Index.h"
#include "nsPrintfCString.h"
// Used for telemetry
#include "mozilla/Telemetry.h"
@ -27,6 +28,8 @@ mozilla::LogModule* GetDemuxerLog() {
return gMediaDemuxerLog;
}
#define LOG(arg, ...) MOZ_LOG(gMediaDemuxerLog, mozilla::LogLevel::Debug, ("MP4Demuxer(%p)::%s: " arg, this, __func__, ##__VA_ARGS__))
namespace mozilla {
class MP4TrackDemuxer : public MediaTrackDemuxer
@ -55,7 +58,7 @@ public:
private:
friend class MP4Demuxer;
void NotifyDataArrived();
void UpdateSamples(nsTArray<RefPtr<MediaRawData>>& aSamples);
already_AddRefed<MediaRawData> GetNextSample();
void EnsureUpToDateIndex();
void SetNextKeyFrameTime();
RefPtr<MP4Demuxer> mParent;
@ -68,6 +71,7 @@ private:
RefPtr<MediaRawData> mQueuedSample;
bool mNeedReIndex;
bool mNeedSPSForTelemetry;
bool mIsH264 = false;
};
@ -241,6 +245,7 @@ MP4TrackDemuxer::MP4TrackDemuxer(MP4Demuxer* aParent,
if (videoInfo &&
(mInfo->mMimeType.EqualsLiteral("video/mp4") ||
mInfo->mMimeType.EqualsLiteral("video/avc"))) {
mIsH264 = true;
RefPtr<MediaByteBuffer> extraData = videoInfo->mExtraData;
mNeedSPSForTelemetry = AccumulateSPSTelemetry(extraData);
mp4_demuxer::SPSData spsdata;
@ -289,15 +294,76 @@ MP4TrackDemuxer::Seek(media::TimeUnit aTime)
mIterator->Seek(seekTime);
// Check what time we actually seeked to.
mQueuedSample = mIterator->GetNext();
if (mQueuedSample) {
seekTime = mQueuedSample->mTime;
}
RefPtr<MediaRawData> sample;
do {
sample = GetNextSample();
if (!sample) {
return SeekPromise::CreateAndReject(DemuxerFailureReason::END_OF_STREAM, __func__);
}
if (!sample->Size()) {
// This sample can't be decoded, continue searching.
continue;
}
if (sample->mKeyframe) {
mQueuedSample = sample;
seekTime = mQueuedSample->mTime;
}
} while (!mQueuedSample);
SetNextKeyFrameTime();
return SeekPromise::CreateAndResolve(media::TimeUnit::FromMicroseconds(seekTime), __func__);
}
already_AddRefed<MediaRawData>
MP4TrackDemuxer::GetNextSample()
{
RefPtr<MediaRawData> sample = mIterator->GetNext();
if (!sample) {
return nullptr;
}
if (mInfo->GetAsVideoInfo()) {
sample->mExtraData = mInfo->GetAsVideoInfo()->mExtraData;
if (mIsH264) {
mp4_demuxer::H264::FrameType type =
mp4_demuxer::H264::GetFrameType(sample);
switch (type) {
case mp4_demuxer::H264::FrameType::I_FRAME: MOZ_FALLTHROUGH;
case mp4_demuxer::H264::FrameType::OTHER:
{
bool keyframe = type == mp4_demuxer::H264::FrameType::I_FRAME;
if (sample->mKeyframe != keyframe) {
NS_WARNING(nsPrintfCString("Frame incorrectly marked as %skeyframe @ pts:%lld dur:%u dts:%lld",
keyframe ? "" : "non-",
sample->mTime,
sample->mDuration,
sample->mTimecode).get());
sample->mKeyframe = keyframe;
}
break;
}
case mp4_demuxer::H264::FrameType::INVALID:
NS_WARNING(nsPrintfCString("Invalid H264 frame @ pts:%lld dur:%u dts:%lld",
sample->mTime,
sample->mDuration,
sample->mTimecode).get());
// We could reject the sample now, however demuxer errors are fatal.
// So we keep the invalid frame, relying on the H264 decoder to
// handle the error later.
// TODO: make demuxer errors non-fatal.
break;
}
}
}
if (sample->mCrypto.mValid) {
nsAutoPtr<MediaRawDataWriter> writer(sample->CreateWriter());
writer->mCrypto.mMode = mInfo->mCrypto.mMode;
writer->mCrypto.mIVSize = mInfo->mCrypto.mIVSize;
writer->mCrypto.mKeyId.AppendElements(mInfo->mCrypto.mKeyId);
}
return sample.forget();
}
RefPtr<MP4TrackDemuxer::SamplesPromise>
MP4TrackDemuxer::GetSamples(int32_t aNumSamples)
{
@ -308,12 +374,14 @@ MP4TrackDemuxer::GetSamples(int32_t aNumSamples)
}
if (mQueuedSample) {
MOZ_ASSERT(mQueuedSample->mKeyframe,
"mQueuedSample must be a keyframe");
samples->mSamples.AppendElement(mQueuedSample);
mQueuedSample = nullptr;
aNumSamples--;
}
RefPtr<MediaRawData> sample;
while (aNumSamples && (sample = mIterator->GetNext())) {
while (aNumSamples && (sample = GetNextSample())) {
if (!sample->Size()) {
continue;
}
@ -324,7 +392,19 @@ MP4TrackDemuxer::GetSamples(int32_t aNumSamples)
if (samples->mSamples.IsEmpty()) {
return SamplesPromise::CreateAndReject(DemuxerFailureReason::END_OF_STREAM, __func__);
} else {
UpdateSamples(samples->mSamples);
for (const auto& sample : samples->mSamples) {
// Collect telemetry from h264 Annex B SPS.
if (mNeedSPSForTelemetry && mp4_demuxer::AnnexB::HasSPS(sample)) {
RefPtr<MediaByteBuffer> extradata =
mp4_demuxer::AnnexB::ExtractExtraData(sample);
mNeedSPSForTelemetry = AccumulateSPSTelemetry(extradata);
}
}
if (mNextKeyframeTime.isNothing() ||
samples->mSamples.LastElement()->mTime >= mNextKeyframeTime.value().ToMicroseconds()) {
SetNextKeyFrameTime();
}
return SamplesPromise::CreateAndResolve(samples, __func__);
}
}
@ -349,33 +429,6 @@ MP4TrackDemuxer::Reset()
SetNextKeyFrameTime();
}
void
MP4TrackDemuxer::UpdateSamples(nsTArray<RefPtr<MediaRawData>>& aSamples)
{
for (size_t i = 0; i < aSamples.Length(); i++) {
MediaRawData* sample = aSamples[i];
// Collect telemetry from h264 Annex B SPS.
if (mNeedSPSForTelemetry && mp4_demuxer::AnnexB::HasSPS(sample)) {
RefPtr<MediaByteBuffer> extradata =
mp4_demuxer::AnnexB::ExtractExtraData(sample);
mNeedSPSForTelemetry = AccumulateSPSTelemetry(extradata);
}
if (sample->mCrypto.mValid) {
nsAutoPtr<MediaRawDataWriter> writer(sample->CreateWriter());
writer->mCrypto.mMode = mInfo->mCrypto.mMode;
writer->mCrypto.mIVSize = mInfo->mCrypto.mIVSize;
writer->mCrypto.mKeyId.AppendElements(mInfo->mCrypto.mKeyId);
}
if (mInfo->GetAsVideoInfo()) {
sample->mExtraData = mInfo->GetAsVideoInfo()->mExtraData;
}
}
if (mNextKeyframeTime.isNothing() ||
aSamples.LastElement()->mTime >= mNextKeyframeTime.value().ToMicroseconds()) {
SetNextKeyFrameTime();
}
}
nsresult
MP4TrackDemuxer::GetNextRandomAccessPoint(media::TimeUnit* aTime)
{
@ -397,7 +450,7 @@ MP4TrackDemuxer::SkipToNextRandomAccessPoint(media::TimeUnit aTimeThreshold)
uint32_t parsed = 0;
bool found = false;
RefPtr<MediaRawData> sample;
while (!found && (sample = mIterator->GetNext())) {
while (!found && (sample = GetNextSample())) {
parsed++;
if (sample->mKeyframe && sample->mTime >= aTimeThreshold.ToMicroseconds()) {
found = true;
@ -441,3 +494,5 @@ MP4TrackDemuxer::BreakCycles()
}
} // namespace mozilla
#undef LOG