Bug 1592539 - Use a MediaStreamRenderer to render video tracks into a secondary container. r=jib,mconley

This re-uses MediaStreamRenderer to render video into a secondary container, to
get identical logic for the clone target as for the clone source. The
MediaStreamRenderer can handle audio, currentTime and more as well, but those
are not used here.

Differential Revision: https://phabricator.services.mozilla.com/D87139

dom/html/HTMLMediaElement.cpp

@@ -2004,6 +2004,10 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement,
       // guard for this.
       tmp->mMediaStreamRenderer = nullptr;
     }
+    if (tmp->mSecondaryMediaStreamRenderer) {
+      tmp->mSecondaryMediaStreamRenderer->Shutdown();
+      tmp->mSecondaryMediaStreamRenderer = nullptr;
+    }
     if (tmp->mMediaStreamTrackListener) {
       tmp->mSrcStream->UnregisterTrackListener(
           tmp->mMediaStreamTrackListener.get());
@@ -2560,6 +2564,12 @@ void HTMLMediaElement::ResetState() {
     mMediaStreamRenderer->Shutdown();
     mMediaStreamRenderer = nullptr;
   }
+  if (mSecondaryMediaStreamRenderer) {
+    // mSecondaryMediaStreamRenderer, has a strong reference to
+    // the secondary VideoFrameContainer.
+    mSecondaryMediaStreamRenderer->Shutdown();
+    mSecondaryMediaStreamRenderer = nullptr;
+  }
 }
 
 void HTMLMediaElement::SelectResourceWrapper() {
@@ -2696,6 +2706,9 @@ void HTMLMediaElement::NotifyMediaTrackEnabled(dom::MediaTrack* aTrack) {
       if (mMediaStreamRenderer) {
         mMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
       }
+      if (mSecondaryMediaStreamRenderer) {
+        mSecondaryMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
+      }
       nsContentUtils::CombineResourcePrincipals(
           &mSrcStreamVideoPrincipal, mSelectedVideoStreamTrack->GetPrincipal());
     }
@@ -2749,6 +2762,9 @@ void HTMLMediaElement::NotifyMediaTrackDisabled(dom::MediaTrack* aTrack) {
       if (mMediaStreamRenderer) {
         mMediaStreamRenderer->RemoveTrack(mSelectedVideoStreamTrack);
       }
+      if (mSecondaryMediaStreamRenderer) {
+        mSecondaryMediaStreamRenderer->RemoveTrack(mSelectedVideoStreamTrack);
+      }
       mSelectedVideoStreamTrack->RemovePrincipalChangeObserver(this);
       mSelectedVideoStreamTrack = nullptr;
     }
@@ -5224,6 +5240,9 @@ void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags) {
     if (mMediaStreamRenderer) {
       mMediaStreamRenderer->Start();
     }
+    if (mSecondaryMediaStreamRenderer) {
+      mSecondaryMediaStreamRenderer->Start();
+    }
 
     SetCapturedOutputStreamsEnabled(true);  // Unmute
     // If the input is a media stream, we don't check its data and always regard
@@ -5233,6 +5252,9 @@ void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags) {
     if (mMediaStreamRenderer) {
       mMediaStreamRenderer->Stop();
     }
+    if (mSecondaryMediaStreamRenderer) {
+      mSecondaryMediaStreamRenderer->Stop();
+    }
     SetCapturedOutputStreamsEnabled(false);  // Mute
   }
 }
@@ -5316,6 +5338,8 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback() {
   }
   mSelectedVideoStreamTrack = nullptr;
 
+  MOZ_ASSERT_IF(mSecondaryMediaStreamRenderer,
+                !mMediaStreamRenderer == !mSecondaryMediaStreamRenderer);
   if (mMediaStreamRenderer) {
     mWatchManager.Unwatch(mPaused,
                           &HTMLMediaElement::UpdateSrcStreamPotentiallyPlaying);
@@ -5331,6 +5355,10 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback() {
     mMediaStreamRenderer->Shutdown();
     mMediaStreamRenderer = nullptr;
   }
+  if (mSecondaryMediaStreamRenderer) {
+    mSecondaryMediaStreamRenderer->Shutdown();
+    mSecondaryMediaStreamRenderer = nullptr;
+  }
 
   mSrcStream->UnregisterTrackListener(mMediaStreamTrackListener.get());
   mMediaStreamTrackListener = nullptr;
@@ -7903,6 +7931,27 @@ void HTMLMediaElement::StartMediaControlKeyListenerIfNeeded() {
   mMediaControlKeyListener->Start();
 }
 
+void HTMLMediaElement::SetSecondaryMediaStreamRenderer(
+    VideoFrameContainer* aContainer,
+    FirstFrameVideoOutput* aFirstFrameOutput /* = nullptr */) {
+  MOZ_ASSERT(mSrcStream);
+  MOZ_ASSERT(mMediaStreamRenderer);
+  if (mSecondaryMediaStreamRenderer) {
+    mSecondaryMediaStreamRenderer->Shutdown();
+    mSecondaryMediaStreamRenderer = nullptr;
+  }
+  if (aContainer) {
+    mSecondaryMediaStreamRenderer = MakeAndAddRef<MediaStreamRenderer>(
+        mAbstractMainThread, aContainer, aFirstFrameOutput, this);
+    if (mSrcStreamIsPlaying) {
+      mSecondaryMediaStreamRenderer->Start();
+    }
+    if (mSelectedVideoStreamTrack) {
+      mSecondaryMediaStreamRenderer->AddTrack(mSelectedVideoStreamTrack);
+    }
+  }
+}
+
 void HTMLMediaElement::UpdateMediaControlAfterPictureInPictureModeChanged() {
   if (IsBeingUsedInPictureInPictureMode()) {
     // When media enters PIP mode, we should ensure that the listener has been

dom/html/HTMLMediaElement.h

@@ -50,6 +50,7 @@ class ChannelMediaDecoder;
 class DecoderDoctorDiagnostics;
 class DOMMediaStream;
 class ErrorResult;
+class FirstFrameVideoOutput;
 class MediaResource;
 class MediaDecoder;
 class MediaInputPort;
@@ -58,6 +59,7 @@ class MediaTrackGraph;
 class MediaStreamWindowCapturer;
 struct SharedDummyTrack;
 class VideoFrameContainer;
+class VideoOutput;
 namespace dom {
 class MediaKeys;
 class TextTrack;
@@ -1350,6 +1352,15 @@ class HTMLMediaElement : public nsGenericHTMLElement,
   // key events.
   void ClearStopMediaControlTimerIfNeeded();
 
+  // Sets a secondary renderer for mSrcStream, so this media element can be
+  // rendered in Picture-in-Picture mode when playing a MediaStream. A null
+  // aContainer will unset the secondary renderer. aFirstFrameOutput allows
+  // for injecting a listener of the callers choice for rendering the first
+  // frame.
+  void SetSecondaryMediaStreamRenderer(
+      VideoFrameContainer* aContainer,
+      FirstFrameVideoOutput* aFirstFrameOutput = nullptr);
+
   // This function is used to update the status of media control when the media
   // changes its status of being used in the Picture-in-Picture mode.
   void UpdateMediaControlAfterPictureInPictureModeChanged();
@@ -1384,6 +1395,10 @@ class HTMLMediaElement : public nsGenericHTMLElement,
   // enabled audio tracks, while mSrcStream is set.
   RefPtr<MediaStreamRenderer> mMediaStreamRenderer;
 
+  // The secondary MediaStreamRenderer handles rendering of our selected video
+  // track to a secondary VideoFrameContainer, while mSrcStream is set.
+  RefPtr<MediaStreamRenderer> mSecondaryMediaStreamRenderer;
+
   // True once PlaybackEnded() is called and we're playing a MediaStream.
   // Reset to false if we start playing mSrcStream again.
   Watchable<bool> mSrcStreamPlaybackEnded = {

dom/html/HTMLVideoElement.cpp

@@ -51,35 +51,6 @@ nsGenericHTMLElement* NS_NewHTMLVideoElement(
 namespace mozilla {
 namespace dom {
 
-class HTMLVideoElement::SecondaryVideoOutput : public VideoOutput {
-  // Main thread only.
-  WeakPtr<HTMLMediaElement> mElement;
-
- public:
-  SecondaryVideoOutput(HTMLMediaElement* aElement,
-                       VideoFrameContainer* aContainer,
-                       AbstractThread* aMainThread)
-      : VideoOutput(aContainer, aMainThread), mElement(aElement) {}
-
-  void Forget() {
-    MOZ_ASSERT(NS_IsMainThread());
-    mElement = nullptr;
-  }
-
-  void NotifyDirectListenerInstalled(InstallationResult aResult) override {
-    if (aResult == InstallationResult::SUCCESS) {
-      mMainThread->Dispatch(NS_NewRunnableFunction(
-          "HTMLMediaElement::OnSecondaryVideoContainerInstalled",
-          [self = RefPtr<SecondaryVideoOutput>(this), this] {
-            if (mElement) {
-              mElement->OnSecondaryVideoContainerInstalled(
-                  mVideoFrameContainer);
-            }
-          }));
-    }
-  }
-};
-
 nsresult HTMLVideoElement::Clone(mozilla::dom::NodeInfo* aNodeInfo,
                                  nsINode** aResult) const {
   *aResult = nullptr;
@@ -103,14 +74,8 @@ NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLVideoElement)
 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(HTMLVideoElement)
   NS_IMPL_CYCLE_COLLECTION_UNLINK(mVisualCloneTarget)
   NS_IMPL_CYCLE_COLLECTION_UNLINK(mVisualCloneTargetPromise)
-  if (tmp->mSecondaryVideoOutput) {
-    MOZ_DIAGNOSTIC_ASSERT(tmp->mSelectedVideoStreamTrack);
-    tmp->mSelectedVideoStreamTrack->RemoveVideoOutput(
-        tmp->mSecondaryVideoOutput);
-    tmp->mSecondaryVideoOutput->Forget();
-    tmp->mSecondaryVideoOutput = nullptr;
-  }
   NS_IMPL_CYCLE_COLLECTION_UNLINK(mVisualCloneSource)
+  tmp->mSecondaryVideoOutput = nullptr;
 NS_IMPL_CYCLE_COLLECTION_UNLINK_END_INHERITED(HTMLMediaElement)
 
 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLVideoElement,
@@ -121,11 +86,14 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLVideoElement,
 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
 
 HTMLVideoElement::HTMLVideoElement(already_AddRefed<NodeInfo>&& aNodeInfo)
-    : HTMLMediaElement(std::move(aNodeInfo)), mIsOrientationLocked(false) {
+    : HTMLMediaElement(std::move(aNodeInfo)),
+      mIsOrientationLocked(false),
+      mVideoWatchManager(this, mAbstractMainThread) {
   DecoderDoctorLogger::LogConstruction(this);
 }
 
 HTMLVideoElement::~HTMLVideoElement() {
+  mVideoWatchManager.Shutdown();
   DecoderDoctorLogger::LogDestruction(this);
 }
 
@@ -555,17 +523,19 @@ void HTMLVideoElement::MaybeBeginCloningVisually() {
   }
 
   if (mDecoder) {
-    VideoFrameContainer* container =
-        mVisualCloneTarget->GetVideoFrameContainer();
-    mDecoder->SetSecondaryVideoContainer(container);
+    mDecoder->SetSecondaryVideoContainer(
+        mVisualCloneTarget->GetVideoFrameContainer());
     UpdateMediaControlAfterPictureInPictureModeChanged();
   } else if (mSrcStream) {
     VideoFrameContainer* container =
         mVisualCloneTarget->GetVideoFrameContainer();
-    if (container && mSelectedVideoStreamTrack) {
-      mSecondaryVideoOutput = MakeRefPtr<SecondaryVideoOutput>(
-          this, container, mAbstractMainThread);
-      mSelectedVideoStreamTrack->AddVideoOutput(mSecondaryVideoOutput);
+    if (container) {
+      mSecondaryVideoOutput =
+          MakeRefPtr<FirstFrameVideoOutput>(container, mAbstractMainThread);
+      mVideoWatchManager.Watch(
+          mSecondaryVideoOutput->mFirstFrameRendered,
+          &HTMLVideoElement::OnSecondaryVideoOutputFirstFrameRendered);
+      SetSecondaryMediaStreamRenderer(container, mSecondaryVideoOutput);
     }
     UpdateMediaControlAfterPictureInPictureModeChanged();
   }
@@ -577,11 +547,13 @@ void HTMLVideoElement::EndCloningVisually() {
   if (mDecoder) {
     mDecoder->SetSecondaryVideoContainer(nullptr);
   } else if (mSrcStream) {
-    if (mSecondaryVideoOutput && mSelectedVideoStreamTrack) {
-      mSelectedVideoStreamTrack->RemoveVideoOutput(mSecondaryVideoOutput);
-      mSecondaryVideoOutput->Forget();
+    if (mSecondaryVideoOutput) {
+      mVideoWatchManager.Unwatch(
+          mSecondaryVideoOutput->mFirstFrameRendered,
+          &HTMLVideoElement::OnSecondaryVideoOutputFirstFrameRendered);
       mSecondaryVideoOutput = nullptr;
     }
+    SetSecondaryMediaStreamRenderer(nullptr);
   }
 
   Unused << mVisualCloneTarget->SetVisualCloneSource(nullptr);
@@ -615,5 +587,10 @@ void HTMLVideoElement::OnSecondaryVideoContainerInstalled(
   mVisualCloneTargetPromise = nullptr;
 }
 
+void HTMLVideoElement::OnSecondaryVideoOutputFirstFrameRendered() {
+  OnSecondaryVideoContainerInstalled(
+      mVisualCloneTarget->GetVideoFrameContainer());
+}
+
 }  // namespace dom
 }  // namespace mozilla

dom/html/HTMLVideoElement.h

@@ -152,6 +152,8 @@ class HTMLVideoElement final : public HTMLMediaElement {
   void OnSecondaryVideoContainerInstalled(
       const RefPtr<VideoFrameContainer>& aSecondaryContainer) override;
 
+  void OnSecondaryVideoOutputFirstFrameRendered();
+
  protected:
   virtual ~HTMLVideoElement();
 
@@ -175,6 +177,8 @@ class HTMLVideoElement final : public HTMLMediaElement {
 
   bool mIsOrientationLocked;
 
+  WatchManager<HTMLVideoElement> mVideoWatchManager;
+
  private:
   bool SetVisualCloneTarget(
       RefPtr<HTMLVideoElement> aVisualCloneTarget,
@@ -192,11 +196,11 @@ class HTMLVideoElement final : public HTMLMediaElement {
   // Set when mVisualCloneTarget is set, and resolved (and unset) when the
   // secondary container has been applied to the underlying resource.
   RefPtr<Promise> mVisualCloneTargetPromise;
-  // Set when mVisualCloneTarget is set and we are playing a MediaStream with a
-  // video track. This is the output wrapping the VideoFrameContainer of
-  // mVisualCloneTarget, so the selected video track can render its frames to
-  // it.
-  RefPtr<SecondaryVideoOutput> mSecondaryVideoOutput;
+  // Set when beginning to clone visually and we are playing a MediaStream.
+  // This is the output wrapping the VideoFrameContainer of mVisualCloneTarget,
+  // so we can render its first frame, and resolve mVisualCloneTargetPromise as
+  // we do.
+  RefPtr<FirstFrameVideoOutput> mSecondaryVideoOutput;
   // If this video is the clone target of another video element,
   // then mVisualCloneSource points to that originating video
   // element.