diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index 04ecf560c5d5..01507679a683 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -17,6 +17,7 @@ #include "mediasink/DecodedAudioDataSink.h" #include "mediasink/AudioSinkWrapper.h" +#include "mediasink/VideoSink.h" #include "mediasink/DecodedStream.h" #include "mozilla/DebugOnly.h" #include "mozilla/Logging.h" @@ -305,7 +306,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mMetadataManager.Connect(mReader->TimedMetadataEvent(), OwnerThread()); - mMediaSink = CreateAudioSink(); + mMediaSink = CreateMediaSink(mAudioCaptured); #ifdef MOZ_EME mCDMProxyPromise.Begin(mDecoder->RequestCDMProxy()->Then( @@ -381,6 +382,23 @@ MediaDecoderStateMachine::CreateAudioSink() return new AudioSinkWrapper(mTaskQueue, audioSinkCreator); } +already_AddRefed +MediaDecoderStateMachine::CreateMediaSink(bool aAudioCaptured) +{ + // TODO: We can't really create a new DecodedStream until OutputStreamManager + // is extracted. It is tricky that the implementation of DecodedStream + // happens to allow reuse after shutdown without creating a new one. + RefPtr audioSink = aAudioCaptured ? + mStreamSink : CreateAudioSink(); + + RefPtr mediaSink = new VideoSink(mTaskQueue, + audioSink, + mVideoQueue, + mDecoder->GetVideoFrameContainer(), + mRealTime); + return mediaSink.forget(); +} + bool MediaDecoderStateMachine::HasFutureAudio() { MOZ_ASSERT(OnTaskQueue()); @@ -1489,7 +1507,8 @@ void MediaDecoderStateMachine::StopMediaSink() if (mMediaSink->IsStarted()) { DECODER_LOG("Stop MediaSink"); mMediaSink->Stop(); - mMediaSinkPromise.DisconnectIfExists(); + mMediaSinkAudioPromise.DisconnectIfExists(); + mMediaSinkVideoPromise.DisconnectIfExists(); } } @@ -1764,12 +1783,20 @@ MediaDecoderStateMachine::StartMediaSink() mAudioCompleted = false; mMediaSink->Start(GetMediaTime(), mInfo); - auto promise = mMediaSink->OnEnded(TrackInfo::kAudioTrack); - if (promise) { - mMediaSinkPromise.Begin(promise->Then( + auto videoPromise = mMediaSink->OnEnded(TrackInfo::kVideoTrack); + auto audioPromise = mMediaSink->OnEnded(TrackInfo::kAudioTrack); + + if (audioPromise) { + mMediaSinkAudioPromise.Begin(audioPromise->Then( OwnerThread(), __func__, this, - &MediaDecoderStateMachine::OnMediaSinkComplete, - &MediaDecoderStateMachine::OnMediaSinkError)); + &MediaDecoderStateMachine::OnMediaSinkAudioComplete, + &MediaDecoderStateMachine::OnMediaSinkAudioError)); + } + if (videoPromise) { + mMediaSinkVideoPromise.Begin(videoPromise->Then( + OwnerThread(), __func__, this, + &MediaDecoderStateMachine::OnMediaSinkVideoComplete, + &MediaDecoderStateMachine::OnMediaSinkVideoError)); } } } @@ -2907,22 +2934,43 @@ MediaDecoderStateMachine::AudioEndTime() const return -1; } -void MediaDecoderStateMachine::OnMediaSinkComplete() +void +MediaDecoderStateMachine::OnMediaSinkVideoComplete() { MOZ_ASSERT(OnTaskQueue()); - mMediaSinkPromise.Complete(); + mMediaSinkVideoPromise.Complete(); + ScheduleStateMachine(); +} + +void +MediaDecoderStateMachine::OnMediaSinkVideoError() +{ + MOZ_ASSERT(OnTaskQueue()); + + mMediaSinkVideoPromise.Complete(); + if (HasAudio()) { + return; + } + DecodeError(); +} + +void MediaDecoderStateMachine::OnMediaSinkAudioComplete() +{ + MOZ_ASSERT(OnTaskQueue()); + + mMediaSinkAudioPromise.Complete(); // Set true only when we have audio. mAudioCompleted = mInfo.HasAudio(); // To notify PlaybackEnded as soon as possible. ScheduleStateMachine(); } -void MediaDecoderStateMachine::OnMediaSinkError() +void MediaDecoderStateMachine::OnMediaSinkAudioError() { MOZ_ASSERT(OnTaskQueue()); - mMediaSinkPromise.Complete(); + mMediaSinkAudioPromise.Complete(); // Set true only when we have audio. mAudioCompleted = mInfo.HasAudio(); @@ -2974,10 +3022,7 @@ MediaDecoderStateMachine::SetAudioCaptured(bool aCaptured) mMediaSink->Shutdown(); // Create a new sink according to whether audio is captured. - // TODO: We can't really create a new DecodedStream until OutputStreamManager - // is extracted. It is tricky that the implementation of DecodedStream - // happens to allow reuse after shutdown without creating a new one. - mMediaSink = aCaptured ? mStreamSink : CreateAudioSink(); + mMediaSink = CreateMediaSink(aCaptured); // Restore playback parameters. mMediaSink->SetPlaybackParams(params); diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h index efcf098c09bc..02f2b6ce80c0 100644 --- a/dom/media/MediaDecoderStateMachine.h +++ b/dom/media/MediaDecoderStateMachine.h @@ -484,6 +484,9 @@ protected: media::MediaSink* CreateAudioSink(); + // Always create mediasink which contains an AudioSink or StreamSink inside. + already_AddRefed CreateMediaSink(bool aAudioCaptured); + // Stops the media sink and shut it down. // The decoder monitor must be held with exactly one lock count. // Called on the state machine thread. @@ -638,12 +641,14 @@ protected: bool IsVideoDecoding(); private: - // Resolved by the MediaSink to signal that all outstanding work is complete - // and the sink is shutting down. - void OnMediaSinkComplete(); + // Resolved by the MediaSink to signal that all audio/video outstanding + // work is complete and identify which part(a/v) of the sink is shutting down. + void OnMediaSinkAudioComplete(); + void OnMediaSinkVideoComplete(); - // Rejected by the MediaSink to signal errors. - void OnMediaSinkError(); + // Rejected by the MediaSink to signal errors for audio/video. + void OnMediaSinkAudioError(); + void OnMediaSinkVideoError(); // Return true if the video decoder's decode speed can not catch up the // play time. @@ -1192,7 +1197,9 @@ private: // Media data resource from the decoder. RefPtr mResource; - MozPromiseRequestHolder mMediaSinkPromise; + // Track the complete & error for audio/video separately + MozPromiseRequestHolder mMediaSinkAudioPromise; + MozPromiseRequestHolder mMediaSinkVideoPromise; MediaEventListener mAudioQueueListener; MediaEventListener mVideoQueueListener; diff --git a/dom/media/mediasink/VideoSink.cpp b/dom/media/mediasink/VideoSink.cpp new file mode 100644 index 000000000000..a1fc0cacabd3 --- /dev/null +++ b/dom/media/mediasink/VideoSink.cpp @@ -0,0 +1,182 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "VideoSink.h" + +namespace mozilla { + +extern PRLogModuleInfo* gMediaDecoderLog; +#define VSINK_LOG(msg, ...) \ + MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, \ + ("VideoSink=%p " msg, this, ##__VA_ARGS__)) +#define VSINK_LOG_V(msg, ...) \ + MOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, \ + ("VideoSink=%p " msg, this, ##__VA_ARGS__)) + +using namespace mozilla::layers; + +namespace media { + +VideoSink::~VideoSink() +{ +} + +const MediaSink::PlaybackParams& +VideoSink::GetPlaybackParams() const +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + return mAudioSink->GetPlaybackParams(); +} + +void +VideoSink::SetPlaybackParams(const PlaybackParams& aParams) +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + mAudioSink->SetPlaybackParams(aParams); +} + +RefPtr +VideoSink::OnEnded(TrackType aType) +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts."); + + if (aType == TrackInfo::kAudioTrack) { + return mAudioSink->OnEnded(aType); + } else if (aType == TrackInfo::kVideoTrack) { + return mEndPromise; + } + return nullptr; +} + +int64_t +VideoSink::GetEndTime(TrackType aType) const +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts."); + + if (aType == TrackInfo::kVideoTrack) { + return mVideoFrameEndTime; + } else if (aType == TrackInfo::kAudioTrack) { + return mAudioSink->GetEndTime(aType); + } + return -1; +} + +int64_t +VideoSink::GetPosition(TimeStamp* aTimeStamp) const +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + + return mAudioSink->GetPosition(aTimeStamp); +} + +bool +VideoSink::HasUnplayedFrames(TrackType aType) const +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + MOZ_ASSERT(aType == TrackInfo::kAudioTrack, "Not implemented for non audio tracks."); + + return mAudioSink->HasUnplayedFrames(aType); +} + +void +VideoSink::SetPlaybackRate(double aPlaybackRate) +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + + mAudioSink->SetPlaybackRate(aPlaybackRate); +} + +void +VideoSink::SetPlaying(bool aPlaying) +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + + VSINK_LOG_V(" playing (%d) -> (%d)", mAudioSink->IsPlaying(), aPlaying); + + mAudioSink->SetPlaying(aPlaying); +} + +void +VideoSink::Start(int64_t aStartTime, const MediaInfo& aInfo) +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + VSINK_LOG("[%s]", __func__); + mAudioSink->Start(aStartTime, aInfo); + + if (aInfo.HasVideo()) { + mEndPromise = mEndPromiseHolder.Ensure(__func__); + mVideoSinkEndRequest.Begin(mEndPromise->Then( + mOwnerThread.get(), __func__, this, + &VideoSink::OnVideoEnded, + &VideoSink::OnVideoEnded)); + } +} + +void +VideoSink::OnVideoEnded() +{ + AssertOwnerThread(); + + mVideoSinkEndRequest.Complete(); +} + +void +VideoSink::Stop() +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + MOZ_ASSERT(mAudioSink->IsStarted(), "playback not started."); + VSINK_LOG("[%s]", __func__); + + mAudioSink->Stop(); + + mVideoSinkEndRequest.DisconnectIfExists(); + mEndPromiseHolder.ResolveIfExists(true, __func__); + mEndPromise = nullptr; +} + +bool +VideoSink::IsStarted() const +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + + return mAudioSink->IsStarted(); +} + +bool +VideoSink::IsPlaying() const +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + + return mAudioSink->IsPlaying(); +} + +void +VideoSink::Shutdown() +{ + AssertOwnerThread(); + MOZ_ASSERT(mAudioSink, "AudioSink should exist."); + MOZ_ASSERT(!mAudioSink->IsStarted(), "must be called after playback stops."); + VSINK_LOG("[%s]", __func__); + + mAudioSink->Shutdown(); +} + +} // namespace media +} // namespace mozilla diff --git a/dom/media/mediasink/VideoSink.h b/dom/media/mediasink/VideoSink.h new file mode 100644 index 000000000000..901f7e87e317 --- /dev/null +++ b/dom/media/mediasink/VideoSink.h @@ -0,0 +1,101 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef VideoSink_h_ +#define VideoSink_h_ + +#include "ImageContainer.h" +#include "MediaSink.h" +#include "mozilla/AbstractThread.h" +#include "mozilla/MozPromise.h" +#include "mozilla/RefPtr.h" +#include "mozilla/TimeStamp.h" +#include "VideoFrameContainer.h" + +namespace mozilla { + +class VideoFrameContainer; +template class MediaQueue; + +namespace media { + +class VideoSink : public MediaSink +{ +public: + VideoSink(AbstractThread* aThread, + MediaSink* aAudioSink, + MediaQueue& aVideoQueue, + VideoFrameContainer* aContainer, + bool aRealTime) + : mOwnerThread(aThread) + , mAudioSink(aAudioSink) + , mVideoQueue(aVideoQueue) + , mContainer(aContainer) + , mRealTime(aRealTime) + , mVideoFrameEndTime(-1) + {} + + const PlaybackParams& GetPlaybackParams() const override; + + void SetPlaybackParams(const PlaybackParams& aParams) override; + + RefPtr OnEnded(TrackType aType) override; + + int64_t GetEndTime(TrackType aType) const override; + + int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const override; + + bool HasUnplayedFrames(TrackType aType) const override; + + void SetPlaybackRate(double aPlaybackRate) override; + + void SetPlaying(bool aPlaying) override; + + void Start(int64_t aStartTime, const MediaInfo& aInfo) override; + + void Stop() override; + + bool IsStarted() const override; + + bool IsPlaying() const override; + + void Shutdown() override; + +private: + virtual ~VideoSink(); + + void AssertOwnerThread() const + { + MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn()); + } + + MediaQueue& VideoQueue() const { + return mVideoQueue; + } + + void OnVideoEnded(); + + const RefPtr mOwnerThread; + RefPtr mAudioSink; + MediaQueue& mVideoQueue; + VideoFrameContainer* mContainer; + + // True if we are decoding a real-time stream. + const bool mRealTime; + + RefPtr mEndPromise; + MozPromiseHolder mEndPromiseHolder; + MozPromiseRequestHolder mVideoSinkEndRequest; + + // The presentation end time of the last video frame which has been displayed + // in microseconds. + int64_t mVideoFrameEndTime; +}; + +} // namespace media +} // namespace mozilla + +#endif diff --git a/dom/media/mediasink/moz.build b/dom/media/mediasink/moz.build index 931bf0fd59c6..228bd2e28100 100644 --- a/dom/media/mediasink/moz.build +++ b/dom/media/mediasink/moz.build @@ -8,6 +8,7 @@ UNIFIED_SOURCES += [ 'AudioSinkWrapper.cpp', 'DecodedAudioDataSink.cpp', 'DecodedStream.cpp', + 'VideoSink.cpp', ] FINAL_LIBRARY = 'xul'