From 7a59d2552d940820bb83e44ef536db15bb9f69bd Mon Sep 17 00:00:00 2001 From: Alastor Wu Date: Thu, 27 Jun 2024 14:47:33 +0000 Subject: [PATCH] Bug 1878510 - When recreating WebM demuxer when encountering a new segment, propagate media time (esr115 rebase) r=media-playback-reviewers,alwu, a=dmeehan Differential Revision: https://phabricator.services.mozilla.com/D215116 --- dom/media/mediasource/TrackBuffersManager.cpp | 42 +++++++++++++++++-- dom/media/mediasource/TrackBuffersManager.h | 2 + dom/media/webm/WebMDemuxer.cpp | 21 +++++++++- dom/media/webm/WebMDemuxer.h | 6 ++- 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/dom/media/mediasource/TrackBuffersManager.cpp b/dom/media/mediasource/TrackBuffersManager.cpp index 779e1bd9d11a..5f4df9dc4867 100644 --- a/dom/media/mediasource/TrackBuffersManager.cpp +++ b/dom/media/mediasource/TrackBuffersManager.cpp @@ -917,7 +917,11 @@ void TrackBuffersManager::SegmentParserLoop() { if (mNewMediaSegmentStarted) { if (NS_SUCCEEDED(newData) && mLastParsedEndTime.isSome() && start < mLastParsedEndTime.ref()) { - MSE_DEBUG("Re-creating demuxer"); + MSE_DEBUG("Re-creating demuxer, new start (%" PRId64 + ") is smaller than last parsed end time (%" PRId64 ")", + start.ToMicroseconds(), + mLastParsedEndTime->ToMicroseconds()); + mFrameEndTimeBeforeRecreateDemuxer = Some(end); ResetDemuxingState(); return; } @@ -1033,8 +1037,15 @@ void TrackBuffersManager::CreateDemuxerforMIMEType() { if (mType.Type() == MEDIAMIMETYPE(VIDEO_WEBM) || mType.Type() == MEDIAMIMETYPE(AUDIO_WEBM)) { - mInputDemuxer = - new WebMDemuxer(mCurrentInputBuffer, true /* IsMediaSource*/); + if (mFrameEndTimeBeforeRecreateDemuxer) { + MSE_DEBUG( + "CreateDemuxerFromMimeType: " + "mFrameEndTimeBeforeRecreateDemuxer=%" PRId64, + mFrameEndTimeBeforeRecreateDemuxer->ToMicroseconds()); + } + mInputDemuxer = new WebMDemuxer(mCurrentInputBuffer, true, + mFrameEndTimeBeforeRecreateDemuxer); + mFrameEndTimeBeforeRecreateDemuxer.reset(); DDLINKCHILD("demuxer", mInputDemuxer.get()); return; } @@ -1043,6 +1054,7 @@ void TrackBuffersManager::CreateDemuxerforMIMEType() { if (mType.Type() == MEDIAMIMETYPE(VIDEO_MP4) || mType.Type() == MEDIAMIMETYPE(AUDIO_MP4)) { mInputDemuxer = new MP4Demuxer(mCurrentInputBuffer); + mFrameEndTimeBeforeRecreateDemuxer.reset(); DDLINKCHILD("demuxer", mInputDemuxer.get()); return; } @@ -1622,9 +1634,11 @@ void TrackBuffersManager::MaybeDispatchEncryptedEvent( void TrackBuffersManager::OnVideoDemuxCompleted( RefPtr aSamples) { mTaskQueueCapability->AssertOnCurrentThread(); - MSE_DEBUG("%zu video samples demuxed", aSamples->GetSamples().Length()); mVideoTracks.mDemuxRequest.Complete(); mVideoTracks.mQueuedSamples.AppendElements(aSamples->GetSamples()); + MSE_DEBUG("%zu video samples demuxed, queued-sz=%zu", + aSamples->GetSamples().Length(), + mVideoTracks.mQueuedSamples.Length()); MaybeDispatchEncryptedEvent(aSamples->GetSamples()); DoDemuxAudio(); @@ -1875,6 +1889,9 @@ void TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples, // coded frame. if (!sample->mKeyframe) { previouslyDroppedSample = nullptr; + SAMPLE_DEBUGV("skipping sample [%" PRId64 ",%" PRId64 "]", + sample->mTime.ToMicroseconds(), + sample->GetEndTime().ToMicroseconds()); continue; } // 2. Set the need random access point flag on track buffer to false. @@ -1951,6 +1968,7 @@ void TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples, // 4. Unset the highest end timestamp on all track buffers. // 5. Set the need random access point flag on all track buffers to // true. + MSE_DEBUG("Resetting append state"); track->ResetAppendState(); } // 6. Jump to the Loop Top step above to restart processing of the current @@ -2469,10 +2487,12 @@ void TrackBuffersManager::RecreateParser(bool aReuseInitData) { mParser = ContainerParser::CreateForMIMEType(mType); DDLINKCHILD("parser", mParser.get()); if (aReuseInitData && mInitData) { + MSE_DEBUG("Using existing init data to reset parser"); TimeUnit start, end; mParser->ParseStartAndEndTimestamps(MediaSpan(mInitData), start, end); mProcessedInput = mInitData->Length(); } else { + MSE_DEBUG("Resetting parser, not reusing init data"); mProcessedInput = 0; } } @@ -2757,12 +2777,20 @@ const MediaRawData* TrackBuffersManager::GetSample(TrackInfo::TrackType aTrack, const TrackBuffer& track = GetTrackBuffer(aTrack); if (aIndex >= track.Length()) { + MSE_DEBUGV( + "Can't get sample due to reaching to the end, index=%u, " + "length=%zu", + aIndex, track.Length()); // reached the end. return nullptr; } if (!(aExpectedDts + aFuzz).IsValid() || !(aExpectedPts + aFuzz).IsValid()) { // Time overflow, it seems like we also reached the end. + MSE_DEBUGV("Can't get sample due to time overflow, expectedPts=%" PRId64 + ", aExpectedDts=%" PRId64 ", fuzz=%" PRId64, + aExpectedPts.ToMicroseconds(), aExpectedPts.ToMicroseconds(), + aFuzz.ToMicroseconds()); return nullptr; } @@ -2773,6 +2801,12 @@ const MediaRawData* TrackBuffersManager::GetSample(TrackInfo::TrackType aTrack, return sample; } + MSE_DEBUGV("Can't get sample due to big gap, sample=%" PRId64 + ", expectedPts=%" PRId64 ", aExpectedDts=%" PRId64 + ", fuzz=%" PRId64, + sample->mTime.ToMicroseconds(), aExpectedPts.ToMicroseconds(), + aExpectedPts.ToMicroseconds(), aFuzz.ToMicroseconds()); + // Gap is too big. End of Stream or Waiting for Data. // TODO, check that we have continuous data based on the sanitized buffered // range instead. diff --git a/dom/media/mediasource/TrackBuffersManager.h b/dom/media/mediasource/TrackBuffersManager.h index 0b3b64b8fe1e..537b0720ff13 100644 --- a/dom/media/mediasource/TrackBuffersManager.h +++ b/dom/media/mediasource/TrackBuffersManager.h @@ -561,6 +561,8 @@ class TrackBuffersManager final // mTaskQueue. However, there's special locking around mTaskQueue, so we keep // both for now. Maybe> mTaskQueueCapability; + + Maybe mFrameEndTimeBeforeRecreateDemuxer; }; } // namespace mozilla diff --git a/dom/media/webm/WebMDemuxer.cpp b/dom/media/webm/WebMDemuxer.cpp index fce8f0e51290..d7d8040c03d4 100644 --- a/dom/media/webm/WebMDemuxer.cpp +++ b/dom/media/webm/WebMDemuxer.cpp @@ -16,6 +16,7 @@ #include "gfx2DGlue.h" #include "gfxUtils.h" #include "mozilla/EndianUtils.h" +#include "mozilla/Maybe.h" #include "mozilla/SharedThreadPool.h" #include "MediaDataDemuxer.h" #include "nsAutoRef.h" @@ -151,7 +152,9 @@ int WebMDemuxer::NestEggContext::Init() { WebMDemuxer::WebMDemuxer(MediaResource* aResource) : WebMDemuxer(aResource, false) {} -WebMDemuxer::WebMDemuxer(MediaResource* aResource, bool aIsMediaSource) +WebMDemuxer::WebMDemuxer( + MediaResource* aResource, bool aIsMediaSource, + Maybe aFrameEndTimeBeforeRecreateDemuxer) : mVideoContext(this, aResource), mAudioContext(this, aResource), mBufferedState(nullptr), @@ -170,6 +173,14 @@ WebMDemuxer::WebMDemuxer(MediaResource* aResource, bool aIsMediaSource) // Audio/video contexts hold a MediaResourceIndex. DDLINKCHILD("video context", mVideoContext.GetResource()); DDLINKCHILD("audio context", mAudioContext.GetResource()); + + MOZ_ASSERT_IF(!aIsMediaSource, + aFrameEndTimeBeforeRecreateDemuxer.isNothing()); + if (aIsMediaSource && aFrameEndTimeBeforeRecreateDemuxer) { + mVideoFrameEndTimeBeforeReset = aFrameEndTimeBeforeRecreateDemuxer; + WEBM_DEBUG("Set mVideoFrameEndTimeBeforeReset=%" PRId64, + mVideoFrameEndTimeBeforeReset->ToMicroseconds()); + } } WebMDemuxer::~WebMDemuxer() { @@ -586,6 +597,10 @@ nsresult WebMDemuxer::GetNextPacket(TrackInfo::TrackType aType, } int64_t tstamp = holder->Timestamp(); int64_t duration = holder->Duration(); + if (aType == TrackInfo::TrackType::kVideoTrack) { + WEBM_DEBUG("GetNextPacket(video): tstamp=%" PRId64 ", duration=%" PRId64, + tstamp, duration); + } // The end time of this frame is the start time of the next frame. Fetch // the timestamp of the next packet for this track. If we've reached the @@ -607,6 +622,10 @@ nsresult WebMDemuxer::GetNextPacket(TrackInfo::TrackType aType, next_tstamp = tstamp + duration; } else if (lastFrameTime.isSome()) { next_tstamp = tstamp + (tstamp - lastFrameTime.ref()); + } else if (mVideoFrameEndTimeBeforeReset) { + WEBM_DEBUG("Setting next timestamp to be %" PRId64 " us", + mVideoFrameEndTimeBeforeReset->ToMicroseconds()); + next_tstamp = mVideoFrameEndTimeBeforeReset->ToMicroseconds(); } else if (mIsMediaSource) { (this->*pushPacket)(holder); } else { diff --git a/dom/media/webm/WebMDemuxer.h b/dom/media/webm/WebMDemuxer.h index 3d8ff67f9ede..eb1d12d2a792 100644 --- a/dom/media/webm/WebMDemuxer.h +++ b/dom/media/webm/WebMDemuxer.h @@ -94,7 +94,9 @@ class WebMDemuxer : public MediaDataDemuxer, explicit WebMDemuxer(MediaResource* aResource); // Indicate if the WebMDemuxer is to be used with MediaSource. In which // case the demuxer will stop reads to the last known complete block. - WebMDemuxer(MediaResource* aResource, bool aIsMediaSource); + WebMDemuxer( + MediaResource* aResource, bool aIsMediaSource, + Maybe aFrameEndTimeBeforeRecreateDemuxer = Nothing()); RefPtr Init() override; @@ -223,6 +225,8 @@ class WebMDemuxer : public MediaDataDemuxer, Maybe mLastAudioFrameTime; Maybe mLastVideoFrameTime; + Maybe mVideoFrameEndTimeBeforeReset; + // Codec ID of audio track int mAudioCodec; // Codec ID of video track