From a04205e495f853f22929ba5513e31adf73049fef Mon Sep 17 00:00:00 2001 From: Andreas Pehrson Date: Wed, 26 Aug 2020 14:25:40 +0000 Subject: [PATCH] Bug 1592539 - Use a MediaStreamRenderer to render video tracks into a secondary container. r=jib,mconley This re-uses MediaStreamRenderer to render video into a secondary container, to get identical logic for the clone target as for the clone source. The MediaStreamRenderer can handle audio, currentTime and more as well, but those are not used here. Differential Revision: https://phabricator.services.mozilla.com/D87139 --- dom/html/HTMLMediaElement.cpp | 49 ++++++++++++++++++++++++ dom/html/HTMLMediaElement.h | 15 ++++++++ dom/html/HTMLVideoElement.cpp | 71 ++++++++++++----------------------- dom/html/HTMLVideoElement.h | 14 ++++--- 4 files changed, 97 insertions(+), 52 deletions(-) diff --git a/dom/html/HTMLMediaElement.cpp b/dom/html/HTMLMediaElement.cpp index ac4e3dc4350e..7470e5467f44 100644 --- a/dom/html/HTMLMediaElement.cpp +++ b/dom/html/HTMLMediaElement.cpp @@ -2004,6 +2004,10 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement, // guard for this. tmp->mMediaStreamRenderer = nullptr; } + if (tmp->mSecondaryMediaStreamRenderer) { + tmp->mSecondaryMediaStreamRenderer->Shutdown(); + tmp->mSecondaryMediaStreamRenderer = nullptr; + } if (tmp->mMediaStreamTrackListener) { tmp->mSrcStream->UnregisterTrackListener( tmp->mMediaStreamTrackListener.get()); @@ -2560,6 +2564,12 @@ void HTMLMediaElement::ResetState() { mMediaStreamRenderer->Shutdown(); mMediaStreamRenderer = nullptr; } + if (mSecondaryMediaStreamRenderer) { + // mSecondaryMediaStreamRenderer, has a strong reference to + // the secondary VideoFrameContainer. + mSecondaryMediaStreamRenderer->Shutdown(); + mSecondaryMediaStreamRenderer = nullptr; + } } void HTMLMediaElement::SelectResourceWrapper() { @@ -2696,6 +2706,9 @@ void HTMLMediaElement::NotifyMediaTrackEnabled(dom::MediaTrack* aTrack) { if (mMediaStreamRenderer) { mMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack); } + if (mSecondaryMediaStreamRenderer) { + mSecondaryMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack); + } nsContentUtils::CombineResourcePrincipals( &mSrcStreamVideoPrincipal, mSelectedVideoStreamTrack->GetPrincipal()); } @@ -2749,6 +2762,9 @@ void HTMLMediaElement::NotifyMediaTrackDisabled(dom::MediaTrack* aTrack) { if (mMediaStreamRenderer) { mMediaStreamRenderer->RemoveTrack(mSelectedVideoStreamTrack); } + if (mSecondaryMediaStreamRenderer) { + mSecondaryMediaStreamRenderer->RemoveTrack(mSelectedVideoStreamTrack); + } mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(this); mSelectedVideoStreamTrack = nullptr; } @@ -5224,6 +5240,9 @@ void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags) { if (mMediaStreamRenderer) { mMediaStreamRenderer->Start(); } + if (mSecondaryMediaStreamRenderer) { + mSecondaryMediaStreamRenderer->Start(); + } SetCapturedOutputStreamsEnabled(true); // Unmute // If the input is a media stream, we don't check its data and always regard @@ -5233,6 +5252,9 @@ void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags) { if (mMediaStreamRenderer) { mMediaStreamRenderer->Stop(); } + if (mSecondaryMediaStreamRenderer) { + mSecondaryMediaStreamRenderer->Stop(); + } SetCapturedOutputStreamsEnabled(false); // Mute } } @@ -5316,6 +5338,8 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback() { } mSelectedVideoStreamTrack = nullptr; + MOZ_ASSERT_IF(mSecondaryMediaStreamRenderer, + !mMediaStreamRenderer == !mSecondaryMediaStreamRenderer); if (mMediaStreamRenderer) { mWatchManager.Unwatch(mPaused, &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying); @@ -5331,6 +5355,10 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback() { mMediaStreamRenderer->Shutdown(); mMediaStreamRenderer = nullptr; } + if (mSecondaryMediaStreamRenderer) { + mSecondaryMediaStreamRenderer->Shutdown(); + mSecondaryMediaStreamRenderer = nullptr; + } mSrcStream->UnregisterTrackListener(mMediaStreamTrackListener.get()); mMediaStreamTrackListener = nullptr; @@ -7903,6 +7931,27 @@ void HTMLMediaElement::StartMediaControlKeyListenerIfNeeded() { mMediaControlKeyListener->Start(); } +void HTMLMediaElement::SetSecondaryMediaStreamRenderer( + VideoFrameContainer* aContainer, + FirstFrameVideoOutput* aFirstFrameOutput /* = nullptr */) { + MOZ_ASSERT(mSrcStream); + MOZ_ASSERT(mMediaStreamRenderer); + if (mSecondaryMediaStreamRenderer) { + mSecondaryMediaStreamRenderer->Shutdown(); + mSecondaryMediaStreamRenderer = nullptr; + } + if (aContainer) { + mSecondaryMediaStreamRenderer = MakeAndAddRef( + mAbstractMainThread, aContainer, aFirstFrameOutput, this); + if (mSrcStreamIsPlaying) { + mSecondaryMediaStreamRenderer->Start(); + } + if (mSelectedVideoStreamTrack) { + mSecondaryMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack); + } + } +} + void HTMLMediaElement::UpdateMediaControlAfterPictureInPictureModeChanged() { if (IsBeingUsedInPictureInPictureMode()) { // When media enters PIP mode, we should ensure that the listener has been diff --git a/dom/html/HTMLMediaElement.h b/dom/html/HTMLMediaElement.h index d85b6ecaaec0..331273fb91e5 100644 --- a/dom/html/HTMLMediaElement.h +++ b/dom/html/HTMLMediaElement.h @@ -50,6 +50,7 @@ class ChannelMediaDecoder; class DecoderDoctorDiagnostics; class DOMMediaStream; class ErrorResult; +class FirstFrameVideoOutput; class MediaResource; class MediaDecoder; class MediaInputPort; @@ -58,6 +59,7 @@ class MediaTrackGraph; class MediaStreamWindowCapturer; struct SharedDummyTrack; class VideoFrameContainer; +class VideoOutput; namespace dom { class MediaKeys; class TextTrack; @@ -1350,6 +1352,15 @@ class HTMLMediaElement : public nsGenericHTMLElement, // key events. void ClearStopMediaControlTimerIfNeeded(); + // Sets a secondary renderer for mSrcStream, so this media element can be + // rendered in Picture-in-Picture mode when playing a MediaStream. A null + // aContainer will unset the secondary renderer. aFirstFrameOutput allows + // for injecting a listener of the callers choice for rendering the first + // frame. + void SetSecondaryMediaStreamRenderer( + VideoFrameContainer* aContainer, + FirstFrameVideoOutput* aFirstFrameOutput = nullptr); + // This function is used to update the status of media control when the media // changes its status of being used in the Picture-in-Picture mode. void UpdateMediaControlAfterPictureInPictureModeChanged(); @@ -1384,6 +1395,10 @@ class HTMLMediaElement : public nsGenericHTMLElement, // enabled audio tracks, while mSrcStream is set. RefPtr mMediaStreamRenderer; + // The secondary MediaStreamRenderer handles rendering of our selected video + // track to a secondary VideoFrameContainer, while mSrcStream is set. + RefPtr mSecondaryMediaStreamRenderer; + // True once PlaybackEnded() is called and we're playing a MediaStream. // Reset to false if we start playing mSrcStream again. Watchable mSrcStreamPlaybackEnded = { diff --git a/dom/html/HTMLVideoElement.cpp b/dom/html/HTMLVideoElement.cpp index 54a40cfdc995..dcb4a529a81a 100644 --- a/dom/html/HTMLVideoElement.cpp +++ b/dom/html/HTMLVideoElement.cpp @@ -51,35 +51,6 @@ nsGenericHTMLElement* NS_NewHTMLVideoElement( namespace mozilla { namespace dom { -class HTMLVideoElement::SecondaryVideoOutput : public VideoOutput { - // Main thread only. - WeakPtr mElement; - - public: - SecondaryVideoOutput(HTMLMediaElement* aElement, - VideoFrameContainer* aContainer, - AbstractThread* aMainThread) - : VideoOutput(aContainer, aMainThread), mElement(aElement) {} - - void Forget() { - MOZ_ASSERT(NS_IsMainThread()); - mElement = nullptr; - } - - void NotifyDirectListenerInstalled(InstallationResult aResult) override { - if (aResult == InstallationResult::SUCCESS) { - mMainThread->Dispatch(NS_NewRunnableFunction( - "HTMLMediaElement::OnSecondaryVideoContainerInstalled", - [self = RefPtr(this), this] { - if (mElement) { - mElement->OnSecondaryVideoContainerInstalled( - mVideoFrameContainer); - } - })); - } - } -}; - nsresult HTMLVideoElement::Clone(mozilla::dom::NodeInfo* aNodeInfo, nsINode** aResult) const { *aResult = nullptr; @@ -103,14 +74,8 @@ NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLVideoElement) NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(HTMLVideoElement) NS_IMPL_CYCLE_COLLECTION_UNLINK(mVisualCloneTarget) NS_IMPL_CYCLE_COLLECTION_UNLINK(mVisualCloneTargetPromise) - if (tmp->mSecondaryVideoOutput) { - MOZ_DIAGNOSTIC_ASSERT(tmp->mSelectedVideoStreamTrack); - tmp->mSelectedVideoStreamTrack->RemoveVideoOutput( - tmp->mSecondaryVideoOutput); - tmp->mSecondaryVideoOutput->Forget(); - tmp->mSecondaryVideoOutput = nullptr; - } NS_IMPL_CYCLE_COLLECTION_UNLINK(mVisualCloneSource) + tmp->mSecondaryVideoOutput = nullptr; NS_IMPL_CYCLE_COLLECTION_UNLINK_END_INHERITED(HTMLMediaElement) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLVideoElement, @@ -121,11 +86,14 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLVideoElement, NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END HTMLVideoElement::HTMLVideoElement(already_AddRefed&& aNodeInfo) - : HTMLMediaElement(std::move(aNodeInfo)), mIsOrientationLocked(false) { + : HTMLMediaElement(std::move(aNodeInfo)), + mIsOrientationLocked(false), + mVideoWatchManager(this, mAbstractMainThread) { DecoderDoctorLogger::LogConstruction(this); } HTMLVideoElement::~HTMLVideoElement() { + mVideoWatchManager.Shutdown(); DecoderDoctorLogger::LogDestruction(this); } @@ -555,17 +523,19 @@ void HTMLVideoElement::MaybeBeginCloningVisually() { } if (mDecoder) { - VideoFrameContainer* container = - mVisualCloneTarget->GetVideoFrameContainer(); - mDecoder->SetSecondaryVideoContainer(container); + mDecoder->SetSecondaryVideoContainer( + mVisualCloneTarget->GetVideoFrameContainer()); UpdateMediaControlAfterPictureInPictureModeChanged(); } else if (mSrcStream) { VideoFrameContainer* container = mVisualCloneTarget->GetVideoFrameContainer(); - if (container && mSelectedVideoStreamTrack) { - mSecondaryVideoOutput = MakeRefPtr( - this, container, mAbstractMainThread); - mSelectedVideoStreamTrack->AddVideoOutput(mSecondaryVideoOutput); + if (container) { + mSecondaryVideoOutput = + MakeRefPtr(container, mAbstractMainThread); + mVideoWatchManager.Watch( + mSecondaryVideoOutput->mFirstFrameRendered, + &HTMLVideoElement::OnSecondaryVideoOutputFirstFrameRendered); + SetSecondaryMediaStreamRenderer(container, mSecondaryVideoOutput); } UpdateMediaControlAfterPictureInPictureModeChanged(); } @@ -577,11 +547,13 @@ void HTMLVideoElement::EndCloningVisually() { if (mDecoder) { mDecoder->SetSecondaryVideoContainer(nullptr); } else if (mSrcStream) { - if (mSecondaryVideoOutput && mSelectedVideoStreamTrack) { - mSelectedVideoStreamTrack->RemoveVideoOutput(mSecondaryVideoOutput); - mSecondaryVideoOutput->Forget(); + if (mSecondaryVideoOutput) { + mVideoWatchManager.Unwatch( + mSecondaryVideoOutput->mFirstFrameRendered, + &HTMLVideoElement::OnSecondaryVideoOutputFirstFrameRendered); mSecondaryVideoOutput = nullptr; } + SetSecondaryMediaStreamRenderer(nullptr); } Unused << mVisualCloneTarget->SetVisualCloneSource(nullptr); @@ -615,5 +587,10 @@ void HTMLVideoElement::OnSecondaryVideoContainerInstalled( mVisualCloneTargetPromise = nullptr; } +void HTMLVideoElement::OnSecondaryVideoOutputFirstFrameRendered() { + OnSecondaryVideoContainerInstalled( + mVisualCloneTarget->GetVideoFrameContainer()); +} + } // namespace dom } // namespace mozilla diff --git a/dom/html/HTMLVideoElement.h b/dom/html/HTMLVideoElement.h index 89d68d521bd6..04b07ce4fd4a 100644 --- a/dom/html/HTMLVideoElement.h +++ b/dom/html/HTMLVideoElement.h @@ -152,6 +152,8 @@ class HTMLVideoElement final : public HTMLMediaElement { void OnSecondaryVideoContainerInstalled( const RefPtr& aSecondaryContainer) override; + void OnSecondaryVideoOutputFirstFrameRendered(); + protected: virtual ~HTMLVideoElement(); @@ -175,6 +177,8 @@ class HTMLVideoElement final : public HTMLMediaElement { bool mIsOrientationLocked; + WatchManager mVideoWatchManager; + private: bool SetVisualCloneTarget( RefPtr aVisualCloneTarget, @@ -192,11 +196,11 @@ class HTMLVideoElement final : public HTMLMediaElement { // Set when mVisualCloneTarget is set, and resolved (and unset) when the // secondary container has been applied to the underlying resource. RefPtr mVisualCloneTargetPromise; - // Set when mVisualCloneTarget is set and we are playing a MediaStream with a - // video track. This is the output wrapping the VideoFrameContainer of - // mVisualCloneTarget, so the selected video track can render its frames to - // it. - RefPtr mSecondaryVideoOutput; + // Set when beginning to clone visually and we are playing a MediaStream. + // This is the output wrapping the VideoFrameContainer of mVisualCloneTarget, + // so we can render its first frame, and resolve mVisualCloneTargetPromise as + // we do. + RefPtr mSecondaryVideoOutput; // If this video is the clone target of another video element, // then mVisualCloneSource points to that originating video // element.