Bug 1509316 - p1: move composite listening out of VideoData/VideoSink. r=jya,mattwoodrow

On Android, decoded buffers need to be send back to MediaCodec in order to be
rendered and/or recycled. The current mechanism introduced in bug 1299068 only
works for playback(VideoData/VideoSink) but not WebRTC(VideoFrame/VideoOutput).
Move the callback to SurfaceTextureImage because VideoData and VideoFrame both
own that when using MediaCodec, and move the notification to VideoFrameContainer
for both VideoSink and VideoOutput pass frames there for compositing.

Differential Revision: https://phabricator.services.mozilla.com/D45771

--HG--
extra : moz-landing-system : lando

dom/media/MediaData.cpp

@@ -224,25 +224,6 @@ VideoData::VideoData(int64_t aOffset, const TimeUnit& aTime,
 
 VideoData::~VideoData() {}
 
-void VideoData::SetListener(UniquePtr<Listener> aListener) {
-  MOZ_ASSERT(!mSentToCompositor,
-             "Listener should be registered before sending data");
-
-  mListener = std::move(aListener);
-}
-
-void VideoData::MarkSentToCompositor() {
-  if (mSentToCompositor) {
-    return;
-  }
-
-  mSentToCompositor = true;
-  if (mListener != nullptr) {
-    mListener->OnSentToCompositor();
-    mListener = nullptr;
-  }
-}
-
 size_t VideoData::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
   size_t size = aMallocSizeOf(this);
 

dom/media/MediaData.h

@@ -451,12 +451,6 @@ class VideoData : public MediaData {
     ColorRange mColorRange = ColorRange::LIMITED;
   };
 
-  class Listener {
-   public:
-    virtual void OnSentToCompositor() = 0;
-    virtual ~Listener() {}
-  };
-
   // Constructs a VideoData object. If aImage is nullptr, creates a new Image
   // holding a copy of the YCbCr data passed in aBuffer. If aImage is not
   // nullptr, it's stored as the underlying video image and aBuffer is assumed
@@ -511,8 +505,7 @@ class VideoData : public MediaData {
             const media::TimeUnit& aTimecode, IntSize aDisplay,
             uint32_t aFrameID);
 
-  void SetListener(UniquePtr<Listener> aListener);
-  void MarkSentToCompositor();
+  void MarkSentToCompositor() { mSentToCompositor = true; }
   bool IsSentToCompositor() { return mSentToCompositor; }
 
   void UpdateDuration(const media::TimeUnit& aDuration);
@@ -528,7 +521,6 @@ class VideoData : public MediaData {
   ~VideoData();
 
   bool mSentToCompositor;
-  UniquePtr<Listener> mListener;
   media::TimeUnit mNextKeyFrameTime;
 };
 

dom/media/VideoFrameContainer.cpp

@@ -5,8 +5,12 @@
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "VideoFrameContainer.h"
-#include "mozilla/Telemetry.h"
+
+#ifdef MOZ_WIDGET_ANDROID
+#include "GLImages.h"  // for SurfaceTextureImage
+#endif
 #include "MediaDecoderOwner.h"
+#include "mozilla/Telemetry.h"
 
 using namespace mozilla::layers;
 
@@ -78,9 +82,27 @@ void VideoFrameContainer::UpdatePrincipalHandleForFrameIDLocked(
   mFrameIDForPendingPrincipalHandle = aFrameID;
 }
 
+#ifdef MOZ_WIDGET_ANDROID
+static void NotifySetCurrent(Image* aImage) {
+  if (aImage == nullptr) {
+    return;
+  }
+
+  SurfaceTextureImage* image = aImage->AsSurfaceTextureImage();
+  if (image == nullptr) {
+    return;
+  }
+
+  image->OnSetCurrent();
+}
+#endif
+
 void VideoFrameContainer::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize,
                                           Image* aImage,
                                           const TimeStamp& aTargetTime) {
+#ifdef MOZ_WIDGET_ANDROID
+    NotifySetCurrent(aImage);
+#endif
   if (aImage) {
     MutexAutoLock lock(mMutex);
     AutoTArray<ImageContainer::NonOwningImage, 1> imageList;
@@ -95,6 +117,15 @@ void VideoFrameContainer::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize,
 void VideoFrameContainer::SetCurrentFrames(
     const gfx::IntSize& aIntrinsicSize,
     const nsTArray<ImageContainer::NonOwningImage>& aImages) {
+#ifdef MOZ_WIDGET_ANDROID
+  // When there are multiple frames, only the last one is effective
+  // (see bug 1299068 comment 4). Here I just count on VideoSink and VideoOutput
+  // to send one frame at a time and warn if not.
+  Unused << NS_WARN_IF(aImages.Length() > 1);
+  for (auto& image : aImages) {
+    NotifySetCurrent(image.mImage);
+  }
+#endif
   MutexAutoLock lock(mMutex);
   SetCurrentFramesLocked(aIntrinsicSize, aImages);
 }

dom/media/platforms/android/RemoteDataDecoder.cpp

@@ -62,13 +62,14 @@ class RemoteVideoDecoder : public RemoteDataDecoder {
  public:
   // Render the output to the surface when the frame is sent
   // to compositor, or release it if not presented.
-  class CompositeListener : private RenderOrReleaseOutput,
-                            public VideoData::Listener {
+  class CompositeListener
+      : private RenderOrReleaseOutput,
+        public layers::SurfaceTextureImage::SetCurrentCallback {
    public:
     CompositeListener(CodecProxy::Param aCodec, Sample::Param aSample)
         : RenderOrReleaseOutput(aCodec, aSample) {}
 
-    void OnSentToCompositor() override { ReleaseOutput(true); }
+    void operator()(void) override { ReleaseOutput(true); }
   };
 
   class InputInfo {
@@ -255,7 +256,7 @@ class RemoteVideoDecoder : public RemoteDataDecoder {
       return;
     }
 
-    UniquePtr<VideoData::Listener> releaseSample(
+    UniquePtr<layers::SurfaceTextureImage::SetCurrentCallback> releaseSample(
         new CompositeListener(mJavaDecoder, aSample));
 
     BufferInfo::LocalRef info = aSample->Info();
@@ -291,6 +292,8 @@ class RemoteVideoDecoder : public RemoteDataDecoder {
       RefPtr<layers::Image> img = new layers::SurfaceTextureImage(
           mSurfaceHandle, inputInfo.mImageSize, false /* NOT continuous */,
           gl::OriginPos::BottomLeft, mConfig.HasAlpha());
+      img->AsSurfaceTextureImage()->RegisterSetCurrentCallback(
+          std::move(releaseSample));
 
       RefPtr<VideoData> v = VideoData::CreateFromImage(
           inputInfo.mDisplaySize, offset,
@@ -299,7 +302,6 @@ class RemoteVideoDecoder : public RemoteDataDecoder {
           !!(flags & MediaCodec::BUFFER_FLAG_SYNC_FRAME),
           TimeUnit::FromMicroseconds(presentationTimeUs));
 
-      v->SetListener(std::move(releaseSample));
       RemoteDataDecoder::UpdateOutputStatus(std::move(v));
     }
 

gfx/layers/GLImages.h

@@ -31,6 +31,12 @@ class GLImage : public Image {
 
 class SurfaceTextureImage : public GLImage {
  public:
+  class SetCurrentCallback {
+   public:
+    virtual void operator()(void) = 0;
+    virtual ~SetCurrentCallback() {}
+  };
+
   SurfaceTextureImage(AndroidSurfaceTextureHandle aHandle,
                       const gfx::IntSize& aSize, bool aContinuous,
                       gl::OriginPos aOriginPos, bool aHasAlpha = true);
@@ -50,12 +56,24 @@ class SurfaceTextureImage : public GLImage {
 
   SurfaceTextureImage* AsSurfaceTextureImage() override { return this; }
 
+  void RegisterSetCurrentCallback(UniquePtr<SetCurrentCallback> aCallback) {
+    mSetCurrentCallback = std::move(aCallback);
+  }
+
+  void OnSetCurrent() {
+    if (mSetCurrentCallback) {
+      (*mSetCurrentCallback)();
+      mSetCurrentCallback.reset();
+    }
+  }
+
  private:
   AndroidSurfaceTextureHandle mHandle;
   gfx::IntSize mSize;
   bool mContinuous;
   gl::OriginPos mOriginPos;
   const bool mHasAlpha;
+  UniquePtr<SetCurrentCallback> mSetCurrentCallback;
 };
 
 #endif  // MOZ_WIDGET_ANDROID