Bug 1879417 - part1 : putting the video decode promise on hold if we already have enough data and Dcomp surface is not ready yet. r=jolin

In [1], we explain why media format reader would keep sending input to
the media engine, and in [2], we explain why that would cause the media
engine dropping frames.

We don't want the media format reader (MFR) to keep sending input
if the dcomp is not ready, because we don't need that much of data.
Therefore, postponing the decoded promise and resolving it when the
output (dcomp) is available.

By doing that, we would store input samples in a reasonable range.

[1] https://bugzilla.mozilla.org/show_bug.cgi?id=1879417#c4
[2] https://bugzilla.mozilla.org/show_bug.cgi?id=1879417#c0

Differential Revision: https://phabricator.services.mozilla.com/D201173

dom/media/platforms/wmf/MFMediaEngineStream.h

@@ -84,7 +84,7 @@ class MFMediaEngineStream
   // Return the type of the track, the result should be either audio or video.
   virtual TrackInfo::TrackType TrackType() = 0;
 
-  RefPtr<MediaDataDecoder::FlushPromise> Flush();
+  virtual RefPtr<MediaDataDecoder::FlushPromise> Flush();
 
   MediaEventProducer<TrackInfo::TrackType>& EndedEvent() { return mEndedEvent; }
 
@@ -93,7 +93,7 @@ class MFMediaEngineStream
 
   virtual MFMediaEngineVideoStream* AsVideoStream() { return nullptr; }
 
-  RefPtr<MediaDataDecoder::DecodePromise> OutputData(
+  virtual RefPtr<MediaDataDecoder::DecodePromise> OutputData(
       RefPtr<MediaRawData> aSample);
 
   virtual RefPtr<MediaDataDecoder::DecodePromise> Drain();

dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp

@@ -49,7 +49,7 @@ void MFMediaEngineVideoStream::SetKnowsCompositor(
        this]() {
         mKnowsCompositor = knowCompositor;
         LOG("Set SetKnowsCompositor=%p", mKnowsCompositor.get());
-        ResolvePendingDrainPromiseIfNeeded();
+        ResolvePendingPromisesIfNeeded();
       }));
 }
 
@@ -74,7 +74,7 @@ void MFMediaEngineVideoStream::SetDCompSurfaceHandle(HANDLE aDCompSurfaceHandle,
           }
         }
         LOG("Set DCompSurfaceHandle, handle=%p", mDCompSurfaceHandle);
-        ResolvePendingDrainPromiseIfNeeded();
+        ResolvePendingPromisesIfNeeded();
       }));
 }
 
@@ -240,6 +240,32 @@ bool MFMediaEngineVideoStream::IsDCompImageReady() {
   return true;
 }
 
+RefPtr<MediaDataDecoder::DecodePromise> MFMediaEngineVideoStream::OutputData(
+    RefPtr<MediaRawData> aSample) {
+  if (IsShutdown()) {
+    return MediaDataDecoder::DecodePromise::CreateAndReject(
+        MediaResult(NS_ERROR_FAILURE,
+                    RESULT_DETAIL("MFMediaEngineStream is shutdown")),
+        __func__);
+  }
+  AssertOnTaskQueue();
+  NotifyNewData(aSample);
+  MediaDataDecoder::DecodedData outputs;
+  if (RefPtr<MediaData> outputData = OutputDataInternal()) {
+    outputs.AppendElement(outputData);
+    LOGV("Output data [%" PRId64 ",%" PRId64 "]",
+         outputData->mTime.ToMicroseconds(),
+         outputData->GetEndTime().ToMicroseconds());
+  }
+  if (ShouldDelayVideoDecodeBeforeDcompReady()) {
+    LOG("Dcomp isn't ready and we already have enough video data. We will send "
+        "them back together at one when Dcomp is ready");
+    return mVideoDecodeBeforeDcompPromise.Ensure(__func__);
+  }
+  return MediaDataDecoder::DecodePromise::CreateAndResolve(std::move(outputs),
+                                                           __func__);
+}
+
 already_AddRefed<MediaData> MFMediaEngineVideoStream::OutputDataInternal() {
   AssertOnTaskQueue();
   if (mRawDataQueueForGeneratingOutput.GetSize() == 0 || !IsDCompImageReady()) {
@@ -266,23 +292,47 @@ RefPtr<MediaDataDecoder::DecodePromise> MFMediaEngineVideoStream::Drain() {
   return MFMediaEngineStream::Drain();
 }
 
-void MFMediaEngineVideoStream::ResolvePendingDrainPromiseIfNeeded() {
+RefPtr<MediaDataDecoder::FlushPromise> MFMediaEngineVideoStream::Flush() {
+  AssertOnTaskQueue();
+  auto promise = MFMediaEngineStream::Flush();
+  mPendingDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+  mVideoDecodeBeforeDcompPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED,
+                                                __func__);
+  return promise;
+}
+
+void MFMediaEngineVideoStream::ResolvePendingPromisesIfNeeded() {
   AssertOnTaskQueue();
-  if (mPendingDrainPromise.IsEmpty()) {
-    return;
-  }
   if (!IsDCompImageReady()) {
     return;
   }
-  MediaDataDecoder::DecodedData outputs;
-  while (RefPtr<MediaData> outputData = OutputDataInternal()) {
-    outputs.AppendElement(outputData);
-    LOGV("Output data [%" PRId64 ",%" PRId64 "]",
-         outputData->mTime.ToMicroseconds(),
-         outputData->GetEndTime().ToMicroseconds());
+
+  // Resolve decoding promise first, then drain promise
+  if (!mVideoDecodeBeforeDcompPromise.IsEmpty()) {
+    MediaDataDecoder::DecodedData outputs;
+    while (RefPtr<MediaData> outputData = OutputDataInternal()) {
+      outputs.AppendElement(outputData);
+      LOGV("Output data [%" PRId64 ",%" PRId64 "]",
+           outputData->mTime.ToMicroseconds(),
+           outputData->GetEndTime().ToMicroseconds());
+    }
+    mVideoDecodeBeforeDcompPromise.Resolve(std::move(outputs), __func__);
+    LOG("Resolved video decode before Dcomp promise");
+  }
+
+  // This drain promise could return no data, if all data has been processed in
+  // the decoding promise.
+  if (!mPendingDrainPromise.IsEmpty()) {
+    MediaDataDecoder::DecodedData outputs;
+    while (RefPtr<MediaData> outputData = OutputDataInternal()) {
+      outputs.AppendElement(outputData);
+      LOGV("Output data [%" PRId64 ",%" PRId64 "]",
+           outputData->mTime.ToMicroseconds(),
+           outputData->GetEndTime().ToMicroseconds());
+    }
+    mPendingDrainPromise.Resolve(std::move(outputs), __func__);
+    LOG("Resolved pending drain promise");
   }
-  mPendingDrainPromise.Resolve(std::move(outputs), __func__);
-  LOG("Resolved pending drain promise");
 }
 
 MediaDataDecoder::ConversionRequired MFMediaEngineVideoStream::NeedsConversion()
@@ -336,6 +386,8 @@ void MFMediaEngineVideoStream::UpdateConfig(const VideoInfo& aInfo) {
 void MFMediaEngineVideoStream::ShutdownCleanUpOnTaskQueue() {
   AssertOnTaskQueue();
   mPendingDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+  mVideoDecodeBeforeDcompPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED,
+                                                __func__);
 }
 
 bool MFMediaEngineVideoStream::IsEnded() const {
@@ -352,6 +404,10 @@ bool MFMediaEngineVideoStream::IsEnded() const {
 
 bool MFMediaEngineVideoStream::IsEncrypted() const { return mIsEncrypted; }
 
+bool MFMediaEngineVideoStream::ShouldDelayVideoDecodeBeforeDcompReady() {
+  return HasEnoughRawData() && !IsDCompImageReady();
+}
+
 nsCString MFMediaEngineVideoStream::GetCodecName() const {
   switch (mStreamType) {
     case WMFStreamType::H264:

dom/media/platforms/wmf/MFMediaEngineVideoStream.h

@@ -19,6 +19,7 @@ class DcompSurfaceImage;
 }  // namespace layers
 
 class MFMediaSource;
+class MediaRawData;
 
 class MFMediaEngineVideoStream final : public MFMediaEngineStream {
  public:
@@ -50,8 +51,13 @@ class MFMediaEngineVideoStream final : public MFMediaEngineStream {
   // change happens during playback.
   void SetConfig(const TrackInfo& aConfig);
 
+  RefPtr<MediaDataDecoder::DecodePromise> OutputData(
+      RefPtr<MediaRawData> aSample) override;
+
   RefPtr<MediaDataDecoder::DecodePromise> Drain() override;
 
+  RefPtr<MediaDataDecoder::FlushPromise> Flush() override;
+
   bool IsEncrypted() const override;
 
  private:
@@ -66,12 +72,23 @@ class MFMediaEngineVideoStream final : public MFMediaEngineStream {
 
   bool IsDCompImageReady();
 
-  void ResolvePendingDrainPromiseIfNeeded();
+  // Those promises are used to handle decode/drain which happens before the
+  // Dcomp surface is ready.
+  void ResolvePendingPromisesIfNeeded();
 
   void ShutdownCleanUpOnTaskQueue() override;
 
   bool IsEnded() const override;
 
+  // Before Dcomp surface is ready, we can't return any video data due to
+  // lacking of the image, which should only happen on the beginning of the
+  // video playback. In that situation, once we have enough video raw data, we
+  // can stop delaying the decode promise by waiting the Dcomp surface and
+  // resolveing the promise when Dcomp surface is ready. Doing so helps to keep
+  // the decode promise pending, so that the MFR won't keep sending more input
+  // data, which we actually don't need that many.
+  bool ShouldDelayVideoDecodeBeforeDcompReady();
+
   // Task queue only members.
   HANDLE mDCompSurfaceHandle;
   bool mNeedRecreateImage;
@@ -98,6 +115,12 @@ class MFMediaEngineVideoStream final : public MFMediaEngineStream {
   // have dcomp image.
   MozPromiseHolder<MediaDataDecoder::DecodePromise> mPendingDrainPromise;
 
+  // The promise used to return all video output which are requested before the
+  // Dcomp surface is ready. This should only be used once in entire playback,
+  // typically happening around the beginning of the playback.
+  MozPromiseHolder<MediaDataDecoder::DecodePromise>
+      mVideoDecodeBeforeDcompPromise;
+
   // Set when `CreateMediaType()` is called.
   bool mIsEncrypted = false;
 };