Bug 1884060 - part1 : use precise duration for the media engine playback. r=media-playback-reviewers,padenot

The `MediaQueue::Duration()` only reports an approximate result, it
could be inaccurate for video raw data, if the data in the queue is not
sorted by order, which is actually very common. Because for video, it
would be stored in DTS order, not PTS order, which means the video data
has a later timestamp could be decoded first than a video data with an
earlier timestamp.

For the media engine playback, we check the queue duration to determine
if we need more data, so I want to know the precise duration, not an
inaccurate one.

Differential Revision: https://phabricator.services.mozilla.com/D203872

dom/media/MediaQueue.h

@@ -25,6 +25,7 @@ extern LazyLogModule gMediaDecoderLog;
 
 class AudioData;
 class VideoData;
+class EncodedFrame;
 
 template <typename T>
 struct TimestampAdjustmentTrait {
@@ -46,13 +47,24 @@ struct NonTimestampAdjustmentTrait {
   static const bool mValue = !TimestampAdjustmentTrait<T>::mValue;
 };
 
+template <typename T>
+struct DurationTypeTrait {
+  using type = media::TimeUnit;
+};
+
+template <>
+struct DurationTypeTrait<EncodedFrame> {
+  using type = uint64_t;
+};
+
 template <class T>
 class MediaQueue : private nsRefPtrDeque<T> {
  public:
-  MediaQueue()
+  explicit MediaQueue(bool aEnablePreciseDuration = false)
       : nsRefPtrDeque<T>(),
         mRecursiveMutex("mediaqueue"),
-        mEndOfStream(false) {}
+        mEndOfStream(false),
+        mEnablePreciseDuration(aEnablePreciseDuration) {}
 
   ~MediaQueue() { Reset(); }
 
@@ -97,6 +109,7 @@ class MediaQueue : private nsRefPtrDeque<T> {
       AdjustTimeStampIfNeeded(aItem);
     }
     nsRefPtrDeque<T>::PushFront(aItem);
+    AddDurationToPreciseDuration(aItem);
   }
 
   inline void Push(T* aItem) {
@@ -112,6 +125,7 @@ class MediaQueue : private nsRefPtrDeque<T> {
     MOZ_DIAGNOSTIC_ASSERT(item->GetEndTime() >= item->mTime);
     AdjustTimeStampIfNeeded(item);
     nsRefPtrDeque<T>::Push(dont_AddRef(item));
+    AddDurationToPreciseDuration(item);
     mPushEvent.Notify(RefPtr<T>(item));
 
     // Pushing new data after queue has ended means that the stream is active
@@ -126,6 +140,7 @@ class MediaQueue : private nsRefPtrDeque<T> {
     RefPtr<T> rv = nsRefPtrDeque<T>::PopFront();
     if (rv) {
       MOZ_DIAGNOSTIC_ASSERT(rv->GetEndTime() >= rv->mTime);
+      SubtractDurationFromPreciseDuration(rv);
       mPopFrontEvent.Notify(RefPtr<T>(rv));
     }
     return rv.forget();
@@ -133,7 +148,12 @@ class MediaQueue : private nsRefPtrDeque<T> {
 
   inline already_AddRefed<T> PopBack() {
     RecursiveMutexAutoLock lock(mRecursiveMutex);
-    return nsRefPtrDeque<T>::Pop();
+    RefPtr<T> rv = nsRefPtrDeque<T>::Pop();
+    if (rv) {
+      MOZ_DIAGNOSTIC_ASSERT(rv->GetEndTime() >= rv->mTime);
+      SubtractDurationFromPreciseDuration(rv);
+    }
+    return rv.forget();
   }
 
   inline RefPtr<T> PeekFront() const {
@@ -151,6 +171,7 @@ class MediaQueue : private nsRefPtrDeque<T> {
     nsRefPtrDeque<T>::Erase();
     SetOffset(media::TimeUnit::Zero());
     mEndOfStream = false;
+    ResetPreciseDuration();
   }
 
   bool AtEndOfStream() const {
@@ -186,6 +207,12 @@ class MediaQueue : private nsRefPtrDeque<T> {
     return (last->GetEndTime() - first->mTime).ToMicroseconds();
   }
 
+  // Return a precise duration if the feature is enabled. Otherwise, return -1.
+  int64_t PreciseDuration() const {
+    RecursiveMutexAutoLock lock(mRecursiveMutex);
+    return GetPreciseDuration();
+  }
+
   void LockedForEach(nsDequeFunctor<T>& aFunctor) const {
     RecursiveMutexAutoLock lock(mRecursiveMutex);
     nsRefPtrDeque<T>::ForEach(aFunctor);
@@ -268,6 +295,59 @@ class MediaQueue : private nsRefPtrDeque<T> {
   // the media queue starts receiving looped data, which timestamp needs to be
   // modified.
   media::TimeUnit mOffset;
+
+  inline void AddDurationToPreciseDuration(T* aItem) {
+    if (!mEnablePreciseDuration) {
+      return;
+    }
+    if constexpr (std::is_same_v<typename DurationTypeTrait<T>::type,
+                                 media::TimeUnit> ||
+                  std::is_same_v<typename DurationTypeTrait<T>::type,
+                                 uint64_t>) {
+      mPreciseDuration += aItem->mDuration;
+    }
+  }
+
+  inline void SubtractDurationFromPreciseDuration(T* aItem) {
+    if (!mEnablePreciseDuration) {
+      return;
+    }
+    if constexpr (std::is_same_v<typename DurationTypeTrait<T>::type,
+                                 media::TimeUnit> ||
+                  std::is_same_v<typename DurationTypeTrait<T>::type,
+                                 uint64_t>) {
+      mPreciseDuration -= aItem->mDuration;
+    }
+  }
+
+  inline void ResetPreciseDuration() {
+    if (!mEnablePreciseDuration) {
+      return;
+    }
+    if constexpr (std::is_same_v<typename DurationTypeTrait<T>::type,
+                                 media::TimeUnit>) {
+      mPreciseDuration = media::TimeUnit::Zero();
+    } else if constexpr (std::is_same_v<typename DurationTypeTrait<T>::type,
+                                        uint64_t>) {
+      mPreciseDuration = 0;
+    }
+  }
+
+  inline int64_t GetPreciseDuration() const {
+    if (mEnablePreciseDuration) {
+      if constexpr (std::is_same_v<typename DurationTypeTrait<T>::type,
+                                   media::TimeUnit>) {
+        return mPreciseDuration.ToMicroseconds();
+      } else if constexpr (std::is_same_v<typename DurationTypeTrait<T>::type,
+                                          uint64_t>) {
+        return mPreciseDuration;
+      }
+    }
+    return -1;
+  }
+
+  typename DurationTypeTrait<T>::type mPreciseDuration;
+  const bool mEnablePreciseDuration = false;
 };
 
 }  // namespace mozilla

dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp

@@ -93,7 +93,7 @@ HRESULT MFMediaEngineAudioStream::CreateMediaType(const TrackInfo& aInfo,
 bool MFMediaEngineAudioStream::HasEnoughRawData() const {
   // If more than this much raw audio is queued, we'll hold off request more
   // audio.
-  return mRawDataQueueForFeedingEngine.Duration() >=
+  return mRawDataQueueForFeedingEngine.PreciseDuration() >=
          StaticPrefs::media_wmf_media_engine_raw_data_threshold_audio();
 }
 

dom/media/platforms/wmf/MFMediaEngineStream.cpp

@@ -107,7 +107,11 @@ MFMediaEngineStreamWrapper::NeedsConversion() const {
 }
 
 MFMediaEngineStream::MFMediaEngineStream()
-    : mIsShutdown(false), mIsSelected(false), mReceivedEOS(false) {
+    : mIsShutdown(false),
+      mIsSelected(false),
+      mRawDataQueueForFeedingEngine(true /* aEnablePreciseDuration */),
+      mRawDataQueueForGeneratingOutput(true /* aEnablePreciseDuration */),
+      mReceivedEOS(false) {
   MOZ_COUNT_CTOR(MFMediaEngineStream);
 }
 
@@ -489,7 +493,7 @@ void MFMediaEngineStream::NotifyNewData(MediaRawData* aSample) {
         "], queue size=%zu, queue duration=%" PRId64,
         aSample->mTime.ToMicroseconds(), aSample->GetEndTime().ToMicroseconds(),
         mRawDataQueueForFeedingEngine.GetSize(),
-        mRawDataQueueForFeedingEngine.Duration());
+        mRawDataQueueForFeedingEngine.PreciseDuration());
   if (mReceivedEOS) {
     SLOG("Receive a new data, cancel old EOS flag");
     mReceivedEOS = false;
@@ -504,7 +508,7 @@ void MFMediaEngineStream::SendRequestSampleEvent(bool aIsEnough) {
   AssertOnTaskQueue();
   SLOGV("data is %s, queue duration=%" PRId64,
         aIsEnough ? "enough" : "not enough",
-        mRawDataQueueForFeedingEngine.Duration());
+        mRawDataQueueForFeedingEngine.PreciseDuration());
   mParentSource->mRequestSampleEvent.Notify(
       SampleRequest{TrackType(), aIsEnough});
 }

dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp

@@ -209,7 +209,7 @@ HRESULT MFMediaEngineVideoStream::CreateMediaType(const TrackInfo& aInfo,
 bool MFMediaEngineVideoStream::HasEnoughRawData() const {
   // If more than this much raw video is queued, we'll hold off request more
   // video.
-  return mRawDataQueueForFeedingEngine.Duration() >=
+  return mRawDataQueueForFeedingEngine.PreciseDuration() >=
          StaticPrefs::media_wmf_media_engine_raw_data_threshold_video();
 }