Bug 1194918 - Add VideoSink which contains either AudioSinkWrapper or DecodedStreamSink as a default operating MediaSink in MDSM. r=jwwang.

dom/media/MediaDecoderStateMachine.cpp

@@ -17,6 +17,7 @@
 
 #include "mediasink/DecodedAudioDataSink.h"
 #include "mediasink/AudioSinkWrapper.h"
+#include "mediasink/VideoSink.h"
 #include "mediasink/DecodedStream.h"
 #include "mozilla/DebugOnly.h"
 #include "mozilla/Logging.h"
@@ -305,7 +306,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
 
   mMetadataManager.Connect(mReader->TimedMetadataEvent(), OwnerThread());
 
-  mMediaSink = CreateAudioSink();
+  mMediaSink = CreateMediaSink(mAudioCaptured);
 
 #ifdef MOZ_EME
   mCDMProxyPromise.Begin(mDecoder->RequestCDMProxy()->Then(
@@ -381,6 +382,23 @@ MediaDecoderStateMachine::CreateAudioSink()
   return new AudioSinkWrapper(mTaskQueue, audioSinkCreator);
 }
 
+already_AddRefed<media::MediaSink>
+MediaDecoderStateMachine::CreateMediaSink(bool aAudioCaptured)
+{
+  // TODO: We can't really create a new DecodedStream until OutputStreamManager
+  //       is extracted. It is tricky that the implementation of DecodedStream
+  //       happens to allow reuse after shutdown without creating a new one.
+  RefPtr<media::MediaSink> audioSink = aAudioCaptured ?
+    mStreamSink : CreateAudioSink();
+
+  RefPtr<media::MediaSink> mediaSink = new VideoSink(mTaskQueue,
+                                                       audioSink,
+                                                       mVideoQueue,
+                                                       mDecoder->GetVideoFrameContainer(),
+                                                       mRealTime);
+  return mediaSink.forget();
+}
+
 bool MediaDecoderStateMachine::HasFutureAudio()
 {
   MOZ_ASSERT(OnTaskQueue());
@@ -1489,7 +1507,8 @@ void MediaDecoderStateMachine::StopMediaSink()
   if (mMediaSink->IsStarted()) {
     DECODER_LOG("Stop MediaSink");
     mMediaSink->Stop();
-    mMediaSinkPromise.DisconnectIfExists();
+    mMediaSinkAudioPromise.DisconnectIfExists();
+    mMediaSinkVideoPromise.DisconnectIfExists();
   }
 }
 
@@ -1764,12 +1783,20 @@ MediaDecoderStateMachine::StartMediaSink()
     mAudioCompleted = false;
     mMediaSink->Start(GetMediaTime(), mInfo);
 
-    auto promise = mMediaSink->OnEnded(TrackInfo::kAudioTrack);
-    if (promise) {
-      mMediaSinkPromise.Begin(promise->Then(
+    auto videoPromise = mMediaSink->OnEnded(TrackInfo::kVideoTrack);
+    auto audioPromise = mMediaSink->OnEnded(TrackInfo::kAudioTrack);
+
+    if (audioPromise) {
+      mMediaSinkAudioPromise.Begin(audioPromise->Then(
         OwnerThread(), __func__, this,
-        &MediaDecoderStateMachine::OnMediaSinkComplete,
-        &MediaDecoderStateMachine::OnMediaSinkError));
+        &MediaDecoderStateMachine::OnMediaSinkAudioComplete,
+        &MediaDecoderStateMachine::OnMediaSinkAudioError));
+    }
+    if (videoPromise) {
+      mMediaSinkVideoPromise.Begin(videoPromise->Then(
+        OwnerThread(), __func__, this,
+        &MediaDecoderStateMachine::OnMediaSinkVideoComplete,
+        &MediaDecoderStateMachine::OnMediaSinkVideoError));
     }
   }
 }
@@ -2907,22 +2934,43 @@ MediaDecoderStateMachine::AudioEndTime() const
   return -1;
 }
 
-void MediaDecoderStateMachine::OnMediaSinkComplete()
+void
+MediaDecoderStateMachine::OnMediaSinkVideoComplete()
 {
   MOZ_ASSERT(OnTaskQueue());
 
-  mMediaSinkPromise.Complete();
+  mMediaSinkVideoPromise.Complete();
+  ScheduleStateMachine();
+}
+
+void
+MediaDecoderStateMachine::OnMediaSinkVideoError()
+{
+  MOZ_ASSERT(OnTaskQueue());
+
+  mMediaSinkVideoPromise.Complete();
+  if (HasAudio()) {
+    return;
+  }
+  DecodeError();
+}
+
+void MediaDecoderStateMachine::OnMediaSinkAudioComplete()
+{
+  MOZ_ASSERT(OnTaskQueue());
+
+  mMediaSinkAudioPromise.Complete();
   // Set true only when we have audio.
   mAudioCompleted = mInfo.HasAudio();
   // To notify PlaybackEnded as soon as possible.
   ScheduleStateMachine();
 }
 
-void MediaDecoderStateMachine::OnMediaSinkError()
+void MediaDecoderStateMachine::OnMediaSinkAudioError()
 {
   MOZ_ASSERT(OnTaskQueue());
 
-  mMediaSinkPromise.Complete();
+  mMediaSinkAudioPromise.Complete();
   // Set true only when we have audio.
   mAudioCompleted = mInfo.HasAudio();
 
@@ -2974,10 +3022,7 @@ MediaDecoderStateMachine::SetAudioCaptured(bool aCaptured)
   mMediaSink->Shutdown();
 
   // Create a new sink according to whether audio is captured.
-  // TODO: We can't really create a new DecodedStream until OutputStreamManager
-  //       is extracted. It is tricky that the implementation of DecodedStream
-  //       happens to allow reuse after shutdown without creating a new one.
-  mMediaSink = aCaptured ? mStreamSink : CreateAudioSink();
+  mMediaSink = CreateMediaSink(aCaptured);
 
   // Restore playback parameters.
   mMediaSink->SetPlaybackParams(params);

dom/media/MediaDecoderStateMachine.h

@@ -484,6 +484,9 @@ protected:
 
   media::MediaSink* CreateAudioSink();
 
+  // Always create mediasink which contains an AudioSink or StreamSink inside.
+  already_AddRefed<media::MediaSink> CreateMediaSink(bool aAudioCaptured);
+
   // Stops the media sink and shut it down.
   // The decoder monitor must be held with exactly one lock count.
   // Called on the state machine thread.
@@ -638,12 +641,14 @@ protected:
   bool IsVideoDecoding();
 
 private:
-  // Resolved by the MediaSink to signal that all outstanding work is complete
-  // and the sink is shutting down.
-  void OnMediaSinkComplete();
+  // Resolved by the MediaSink to signal that all audio/video outstanding
+  // work is complete and identify which part(a/v) of the sink is shutting down.
+  void OnMediaSinkAudioComplete();
+  void OnMediaSinkVideoComplete();
 
-  // Rejected by the MediaSink to signal errors.
-  void OnMediaSinkError();
+  // Rejected by the MediaSink to signal errors for audio/video.
+  void OnMediaSinkAudioError();
+  void OnMediaSinkVideoError();
 
   // Return true if the video decoder's decode speed can not catch up the
   // play time.
@@ -1192,7 +1197,9 @@ private:
   // Media data resource from the decoder.
   RefPtr<MediaResource> mResource;
 
-  MozPromiseRequestHolder<GenericPromise> mMediaSinkPromise;
+  // Track the complete & error for audio/video separately
+  MozPromiseRequestHolder<GenericPromise> mMediaSinkAudioPromise;
+  MozPromiseRequestHolder<GenericPromise> mMediaSinkVideoPromise;
 
   MediaEventListener mAudioQueueListener;
   MediaEventListener mVideoQueueListener;

dom/media/mediasink/VideoSink.cpp

@@ -0,0 +1,182 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "VideoSink.h"
+
+namespace mozilla {
+
+extern PRLogModuleInfo* gMediaDecoderLog;
+#define VSINK_LOG(msg, ...) \
+  MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, \
+    ("VideoSink=%p " msg, this, ##__VA_ARGS__))
+#define VSINK_LOG_V(msg, ...) \
+  MOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, \
+  ("VideoSink=%p " msg, this, ##__VA_ARGS__))
+
+using namespace mozilla::layers;
+
+namespace media {
+
+VideoSink::~VideoSink()
+{
+}
+
+const MediaSink::PlaybackParams&
+VideoSink::GetPlaybackParams() const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  return mAudioSink->GetPlaybackParams();
+}
+
+void
+VideoSink::SetPlaybackParams(const PlaybackParams& aParams)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  mAudioSink->SetPlaybackParams(aParams);
+}
+
+RefPtr<GenericPromise>
+VideoSink::OnEnded(TrackType aType)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
+
+  if (aType == TrackInfo::kAudioTrack) {
+    return mAudioSink->OnEnded(aType);
+  } else if (aType == TrackInfo::kVideoTrack) {
+    return mEndPromise;
+  }
+  return nullptr;
+}
+
+int64_t
+VideoSink::GetEndTime(TrackType aType) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
+
+  if (aType == TrackInfo::kVideoTrack) {
+    return mVideoFrameEndTime;
+  } else if (aType == TrackInfo::kAudioTrack) {
+    return mAudioSink->GetEndTime(aType);
+  }
+  return -1;
+}
+
+int64_t
+VideoSink::GetPosition(TimeStamp* aTimeStamp) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+
+  return mAudioSink->GetPosition(aTimeStamp);
+}
+
+bool
+VideoSink::HasUnplayedFrames(TrackType aType) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  MOZ_ASSERT(aType == TrackInfo::kAudioTrack, "Not implemented for non audio tracks.");
+
+  return mAudioSink->HasUnplayedFrames(aType);
+}
+
+void
+VideoSink::SetPlaybackRate(double aPlaybackRate)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+
+  mAudioSink->SetPlaybackRate(aPlaybackRate);
+}
+
+void
+VideoSink::SetPlaying(bool aPlaying)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+
+  VSINK_LOG_V(" playing (%d) -> (%d)", mAudioSink->IsPlaying(), aPlaying);
+
+  mAudioSink->SetPlaying(aPlaying);
+}
+
+void
+VideoSink::Start(int64_t aStartTime, const MediaInfo& aInfo)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  VSINK_LOG("[%s]", __func__);
+  mAudioSink->Start(aStartTime, aInfo);
+
+  if (aInfo.HasVideo()) {
+    mEndPromise = mEndPromiseHolder.Ensure(__func__);
+    mVideoSinkEndRequest.Begin(mEndPromise->Then(
+      mOwnerThread.get(), __func__, this,
+      &VideoSink::OnVideoEnded,
+      &VideoSink::OnVideoEnded));
+  }
+}
+
+void
+VideoSink::OnVideoEnded()
+{
+  AssertOwnerThread();
+
+  mVideoSinkEndRequest.Complete();
+}
+
+void
+VideoSink::Stop()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  MOZ_ASSERT(mAudioSink->IsStarted(), "playback not started.");
+  VSINK_LOG("[%s]", __func__);
+
+  mAudioSink->Stop();
+
+  mVideoSinkEndRequest.DisconnectIfExists();
+  mEndPromiseHolder.ResolveIfExists(true, __func__);
+  mEndPromise = nullptr;
+}
+
+bool
+VideoSink::IsStarted() const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+
+  return mAudioSink->IsStarted();
+}
+
+bool
+VideoSink::IsPlaying() const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+
+  return mAudioSink->IsPlaying();
+}
+
+void
+VideoSink::Shutdown()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  MOZ_ASSERT(!mAudioSink->IsStarted(), "must be called after playback stops.");
+  VSINK_LOG("[%s]", __func__);
+
+  mAudioSink->Shutdown();
+}
+
+} // namespace media
+} // namespace mozilla

dom/media/mediasink/VideoSink.h

@@ -0,0 +1,101 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef VideoSink_h_
+#define VideoSink_h_
+
+#include "ImageContainer.h"
+#include "MediaSink.h"
+#include "mozilla/AbstractThread.h"
+#include "mozilla/MozPromise.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/TimeStamp.h"
+#include "VideoFrameContainer.h"
+
+namespace mozilla {
+
+class VideoFrameContainer;
+template <class T> class MediaQueue;
+
+namespace media {
+
+class VideoSink : public MediaSink
+{
+public:
+  VideoSink(AbstractThread* aThread,
+            MediaSink* aAudioSink,
+            MediaQueue<MediaData>& aVideoQueue,
+            VideoFrameContainer* aContainer,
+            bool aRealTime)
+    : mOwnerThread(aThread)
+    , mAudioSink(aAudioSink)
+    , mVideoQueue(aVideoQueue)
+    , mContainer(aContainer)
+    , mRealTime(aRealTime)
+    , mVideoFrameEndTime(-1)
+  {}
+
+  const PlaybackParams& GetPlaybackParams() const override;
+
+  void SetPlaybackParams(const PlaybackParams& aParams) override;
+
+  RefPtr<GenericPromise> OnEnded(TrackType aType) override;
+
+  int64_t GetEndTime(TrackType aType) const override;
+
+  int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const override;
+
+  bool HasUnplayedFrames(TrackType aType) const override;
+
+  void SetPlaybackRate(double aPlaybackRate) override;
+
+  void SetPlaying(bool aPlaying) override;
+
+  void Start(int64_t aStartTime, const MediaInfo& aInfo) override;
+
+  void Stop() override;
+
+  bool IsStarted() const override;
+
+  bool IsPlaying() const override;
+
+  void Shutdown() override;
+
+private:
+  virtual ~VideoSink();
+
+  void AssertOwnerThread() const
+  {
+    MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
+  }
+
+  MediaQueue<MediaData>& VideoQueue() const {
+    return mVideoQueue;
+  }
+
+  void OnVideoEnded();
+
+  const RefPtr<AbstractThread> mOwnerThread;
+  RefPtr<MediaSink> mAudioSink;
+  MediaQueue<MediaData>& mVideoQueue;
+  VideoFrameContainer* mContainer;
+
+  // True if we are decoding a real-time stream.
+  const bool mRealTime;
+
+  RefPtr<GenericPromise> mEndPromise;
+  MozPromiseHolder<GenericPromise> mEndPromiseHolder;
+  MozPromiseRequestHolder<GenericPromise> mVideoSinkEndRequest;
+
+  // The presentation end time of the last video frame which has been displayed
+  // in microseconds.
+  int64_t mVideoFrameEndTime;
+};
+
+} // namespace media
+} // namespace mozilla
+
+#endif

dom/media/mediasink/moz.build

@@ -8,6 +8,7 @@ UNIFIED_SOURCES += [
     'AudioSinkWrapper.cpp',
     'DecodedAudioDataSink.cpp',
     'DecodedStream.cpp',
+    'VideoSink.cpp',
 ]
 
 FINAL_LIBRARY = 'xul'