Bug 881512 - Start processing multiple decoders. r=cajbir

content/media/BufferDecoder.h

@@ -46,7 +46,7 @@ public:
 
   virtual int64_t GetEndMediaTime() const MOZ_FINAL MOZ_OVERRIDE;
 
-  virtual int64_t GetMediaDuration() MOZ_FINAL MOZ_OVERRIDE;
+  virtual int64_t GetMediaDuration() MOZ_OVERRIDE;
 
   virtual void SetMediaDuration(int64_t aDuration) MOZ_OVERRIDE;
 

content/media/MediaDecoderReader.h

@@ -160,6 +160,8 @@ public:
   // by discarding audio samples and adjusting start times of video frames.
   nsresult DecodeToTarget(int64_t aTarget);
 
+  MediaInfo GetMediaInfo() { return mInfo; }
+
 protected:
 
   // Reference to the owning decoder object.

content/media/mediasource/MediaSourceDecoder.cpp

@@ -15,6 +15,7 @@
 #include "mozilla/dom/TimeRanges.h"
 #include "mozilla/mozalloc.h"
 #include "nsISupports.h"
+#include "nsIThread.h"
 #include "prlog.h"
 #include "MediaSource.h"
 #include "SubBufferDecoder.h"
@@ -41,9 +42,13 @@ class MediaSourceReader : public MediaDecoderReader
 public:
   MediaSourceReader(MediaSourceDecoder* aDecoder)
     : MediaDecoderReader(aDecoder)
+    , mActiveVideoReader(-1)
+    , mActiveAudioReader(-1)
   {
   }
 
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSourceReader)
+
   nsresult Init(MediaDecoderReader* aCloneDonor) MOZ_OVERRIDE
   {
     // Although we technically don't implement anything here, we return NS_OK
@@ -54,17 +59,28 @@ public:
 
   bool DecodeAudioData() MOZ_OVERRIDE
   {
-    if (GetAudioReader()) {
+    if (mActiveAudioReader == -1) {
-      return GetAudioReader()->DecodeAudioData();
+      MSE_DEBUG("%p DecodeAudioFrame called with no audio reader", this);
-    }
+      MOZ_ASSERT(mPendingDecoders.IsEmpty());
       return false;
     }
+    return mAudioReaders[mActiveAudioReader]->DecodeAudioData();
+  }
 
   bool DecodeVideoFrame(bool& aKeyFrameSkip, int64_t aTimeThreshold) MOZ_OVERRIDE
   {
-    if (GetVideoReader()) {
+    if (mActiveVideoReader == -1) {
-      return GetVideoReader()->DecodeVideoFrame(aKeyFrameSkip, aTimeThreshold);
+      MSE_DEBUG("%p DecodeVideoFrame called with no video reader", this);
+      MOZ_ASSERT(mPendingDecoders.IsEmpty());
+      return false;
     }
+    bool rv = mVideoReaders[mActiveVideoReader]->DecodeVideoFrame(aKeyFrameSkip, aTimeThreshold);
+    if (rv) {
+      return true;
+    }
+    MSE_DEBUG("%p MSR::DecodeVF %d (%p) returned false (readers=%u)",
+              this, mActiveVideoReader, mVideoReaders[mActiveVideoReader], mVideoReaders.Length());
+
     return false;
   }
 
@@ -88,19 +104,29 @@ public:
 
   nsresult GetBuffered(dom::TimeRanges* aBuffered, int64_t aStartTime) MOZ_OVERRIDE
   {
-    // XXX: Merge result with audio reader.
+    for (uint32_t i = 0; i < mVideoReaders.Length(); ++i) {
-    MediaDecoderReader* reader = GetVideoReader() ? GetVideoReader() : GetAudioReader();
+      nsRefPtr<dom::TimeRanges> r = new dom::TimeRanges();
-    if (reader) {
+      mVideoReaders[i]->GetBuffered(r, aStartTime);
-      return reader->GetBuffered(aBuffered, aStartTime);
+      aBuffered->Add(r->GetStartTime(), r->GetEndTime());
     }
+    for (uint32_t i = 0; i < mAudioReaders.Length(); ++i) {
+      nsRefPtr<dom::TimeRanges> r = new dom::TimeRanges();
+      mAudioReaders[i]->GetBuffered(r, aStartTime);
+      aBuffered->Add(r->GetStartTime(), r->GetEndTime());
+    }
+    aBuffered->Normalize();
     return NS_OK;
   }
 
   MediaQueue<AudioData>& AudioQueue() MOZ_OVERRIDE
   {
     // TODO: Share AudioQueue with SubReaders.
-    if (GetAudioReader()) {
+    for (uint32_t i = 0; i < mAudioReaders.Length(); ++i) {
-      return GetAudioReader()->AudioQueue();
+      MediaQueue<AudioData>& audioQueue = mAudioReaders[i]->AudioQueue();
+      // Empty existing queues in order.
+      if (audioQueue.GetSize() > 0) {
+        return audioQueue;
+      }
     }
     return MediaDecoderReader::AudioQueue();
   }
@@ -108,30 +134,38 @@ public:
   MediaQueue<VideoData>& VideoQueue() MOZ_OVERRIDE
   {
     // TODO: Share VideoQueue with SubReaders.
-    if (GetVideoReader()) {
+    for (uint32_t i = 0; i < mVideoReaders.Length(); ++i) {
-      return GetVideoReader()->VideoQueue();
+      MediaQueue<VideoData>& videoQueue = mVideoReaders[i]->VideoQueue();
+      // Empty existing queues in order.
+      if (videoQueue.GetSize() > 0) {
+        return videoQueue;
+      }
     }
     return MediaDecoderReader::VideoQueue();
   }
 
-private:
+  already_AddRefed<SubBufferDecoder> CreateSubDecoder(const nsACString& aType,
-  MediaDecoderReader* GetVideoReader()
+                                                      MediaSourceDecoder* aParentDecoder);
-  {
-    MediaSourceDecoder* decoder = static_cast<MediaSourceDecoder*>(mDecoder);
-    return decoder->GetVideoReader();
-  }
 
-  MediaDecoderReader* GetAudioReader()
+private:
-  {
+  bool EnsureWorkQueueInitialized();
-    MediaSourceDecoder* decoder = static_cast<MediaSourceDecoder*>(mDecoder);
+  nsresult EnqueueDecoderInitialization();
-    return decoder->GetAudioReader();
+  void CallDecoderInitialization();
-  }
+  void WaitForPendingDecoders();
+
+  nsTArray<nsRefPtr<SubBufferDecoder>> mPendingDecoders;
+  nsTArray<nsRefPtr<SubBufferDecoder>> mDecoders;
+
+  nsTArray<MediaDecoderReader*> mVideoReaders;
+  nsTArray<MediaDecoderReader*> mAudioReaders;
+  int32_t mActiveVideoReader;
+  int32_t mActiveAudioReader;
+
+  nsCOMPtr<nsIThread> mWorkQueue;
 };
 
 MediaSourceDecoder::MediaSourceDecoder(dom::HTMLMediaElement* aElement)
   : mMediaSource(nullptr)
-  , mVideoReader(nullptr),
-    mAudioReader(nullptr)
 {
   Init(aElement);
 }
@@ -146,7 +180,9 @@ MediaSourceDecoder::Clone()
 MediaDecoderStateMachine*
 MediaSourceDecoder::CreateStateMachine()
 {
-  return new MediaDecoderStateMachine(this, new MediaSourceReader(this));
+  // XXX: Find a cleaner way to retain a reference to our reader.
+  mReader = new MediaSourceReader(this);
+  return new MediaDecoderStateMachine(this, mReader);
 }
 
 nsresult
@@ -171,7 +207,7 @@ MediaSourceDecoder::GetSeekable(dom::TimeRanges* aSeekable)
   } else if (duration > 0 && mozilla::IsInfinite(duration)) {
     nsRefPtr<dom::TimeRanges> bufferedRanges = new dom::TimeRanges();
     GetBuffered(bufferedRanges);
-    aSeekable->Add(0, bufferedRanges->GetFinalEndTime());
+    aSeekable->Add(bufferedRanges->GetStartTime(), bufferedRanges->GetEndTime());
   } else {
     aSeekable->Add(0, duration);
   }
@@ -198,22 +234,107 @@ MediaSourceDecoder::DetachMediaSource()
   mMediaSource = nullptr;
 }
 
-SubBufferDecoder*
+already_AddRefed<SubBufferDecoder>
 MediaSourceDecoder::CreateSubDecoder(const nsACString& aType)
 {
-  MediaResource* resource = new SourceBufferResource(nullptr, aType);
+  return mReader->CreateSubDecoder(aType, this);
-  nsRefPtr<SubBufferDecoder> decoder = new SubBufferDecoder(resource, this);
+}
-  nsAutoPtr<MediaDecoderReader> reader(DecoderTraits::CreateReader(aType, decoder));
-  reader->Init(nullptr);
 
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
+bool
+MediaSourceReader::EnsureWorkQueueInitialized()
+{
+  // TODO: Use a global threadpool rather than a thread per MediaSource.
+  MOZ_ASSERT(NS_IsMainThread());
+  if (!mWorkQueue &&
+      NS_FAILED(NS_NewNamedThread("MediaSource",
+                                  getter_AddRefs(mWorkQueue),
+                                  nullptr,
+                                  MEDIA_THREAD_STACK_SIZE))) {
+    return false;
+  }
+  return true;
+}
+
+nsresult
+MediaSourceReader::EnqueueDecoderInitialization()
+{
+  if (!EnsureWorkQueueInitialized()) {
+    return NS_ERROR_FAILURE;
+  }
+  return mWorkQueue->Dispatch(NS_NewRunnableMethod(this, &MediaSourceReader::CallDecoderInitialization), NS_DISPATCH_NORMAL);
+}
+
+void
+MediaSourceReader::CallDecoderInitialization()
+{
+  ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
+  for (uint32_t i = 0; i < mPendingDecoders.Length(); ++i) {
+    nsRefPtr<SubBufferDecoder> decoder = mPendingDecoders[i];
+    MediaDecoderReader* reader = decoder->GetReader();
+    MSE_DEBUG("%p: Initializating subdecoder %p reader %p", this, decoder.get(), reader);
+
+    reader->SetActive();
+    MediaInfo mi;
+    nsAutoPtr<MetadataTags> tags; // TODO: Handle metadata.
+    nsresult rv;
+    {
+      ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor());
+      rv = reader->ReadMetadata(&mi, getter_Transfers(tags));
+    }
+    reader->SetIdle();
+    if (NS_FAILED(rv)) {
+      // XXX: Need to signal error back to owning SourceBuffer.
+      MSE_DEBUG("%p: Reader %p failed to initialize, rv=%x", this, reader, rv);
+      continue;
+    }
+
+    bool active = false;
+    if (mi.HasVideo()) {
+      MSE_DEBUG("%p: Reader %p has video track", this, reader);
+      mVideoReaders.AppendElement(reader);
+      active = true;
+    }
+    if (mi.HasAudio()) {
+      MSE_DEBUG("%p: Reader %p has audio track", this, reader);
+      mAudioReaders.AppendElement(reader);
+      active = true;
+    }
+
+    if (active) {
       mDecoders.AppendElement(decoder);
-  mReaders.AppendElement(reader);
+    } else {
-  MSE_DEBUG("Registered subdecoder %p subreader %p", decoder.get(), reader.get());
+      MSE_DEBUG("%p: Reader %p not activated", this, reader);
+    }
+  }
+  mPendingDecoders.Clear();
   mon.NotifyAll();
+}
 
+void
+MediaSourceReader::WaitForPendingDecoders()
+{
+  ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
+  while (!mPendingDecoders.IsEmpty()) {
+    mon.Wait();
+  }
+}
+
+already_AddRefed<SubBufferDecoder>
+MediaSourceReader::CreateSubDecoder(const nsACString& aType, MediaSourceDecoder* aParentDecoder)
+{
+  nsRefPtr<SubBufferDecoder> decoder =
+    new SubBufferDecoder(new SourceBufferResource(nullptr, aType), aParentDecoder);
+  nsAutoPtr<MediaDecoderReader> reader(DecoderTraits::CreateReader(aType, decoder));
+  if (!reader) {
+    return nullptr;
+  }
+  reader->Init(nullptr);
+  ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
+  MSE_DEBUG("Registered subdecoder %p subreader %p", decoder.get(), reader.get());
   decoder->SetReader(reader.forget());
-  return decoder;
+  mPendingDecoders.AppendElement(decoder);
+  EnqueueDecoderInitialization();
+  return decoder.forget();
 }
 
 nsresult
@@ -222,36 +343,44 @@ MediaSourceReader::ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags)
   mDecoder->SetMediaSeekable(true);
   mDecoder->SetTransportSeekable(false);
 
-  MediaSourceDecoder* decoder = static_cast<MediaSourceDecoder*>(mDecoder);
+  WaitForPendingDecoders();
-  const nsTArray<MediaDecoderReader*>& readers = decoder->GetReaders();
+
-  for (uint32_t i = 0; i < readers.Length(); ++i) {
+  // XXX: Make subdecoder setup async, so that use cases like bug 989888 can
-    MediaDecoderReader* reader = readers[i];
+  // work.  This will require teaching the state machine about dynamic track
-    MediaInfo mi;
+  // changes (and multiple tracks).
-    nsresult rv = reader->ReadMetadata(&mi, aTags);
+  // Shorter term, make this block until we've got at least one video track
-    MSE_DEBUG("ReadMetadata on SB reader %p", reader);
+  // and lie about having an audio track, then resample/remix as necessary
-    if (NS_FAILED(rv)) {
+  // to match any audio track added later to fit the format we lied about
-      return rv;
+  // now.  For now we just configure what we've got and cross our fingers.
-    }
+  int64_t maxDuration = -1;
+  for (uint32_t i = 0; i < mDecoders.Length(); ++i) {
+    MediaDecoderReader* reader = mDecoders[i]->GetReader();
+
+    reader->SetActive(); // XXX check where this should be called
+
+    MediaInfo mi = reader->GetMediaInfo();
+
     if (mi.HasVideo() && !mInfo.HasVideo()) {
+      MOZ_ASSERT(mActiveVideoReader == -1);
+      mActiveVideoReader = i;
       mInfo.mVideo = mi.mVideo;
-      decoder->SetVideoReader(reader);
+      maxDuration = std::max(maxDuration, mDecoders[i]->GetMediaDuration());
     }
     if (mi.HasAudio() && !mInfo.HasAudio()) {
+      MOZ_ASSERT(mActiveAudioReader == -1);
+      mActiveAudioReader = i;
       mInfo.mAudio = mi.mAudio;
-      decoder->SetAudioReader(reader);
+      maxDuration = std::max(maxDuration, mDecoders[i]->GetMediaDuration());
     }
   }
   *aInfo = mInfo;
 
+  if (maxDuration != -1) {
+    ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
+    mDecoder->SetMediaDuration(maxDuration);
+  }
+
   return NS_OK;
 }
 
-double
-MediaSourceDecoder::GetMediaSourceDuration()
-{
-  return mMediaSource ?
-           mMediaSource->Duration() :
-           mDuration / static_cast<double>(USECS_PER_S);
-}
-
 } // namespace mozilla

content/media/mediasource/MediaSourceDecoder.h

@@ -48,51 +48,14 @@ public:
   void AttachMediaSource(dom::MediaSource* aMediaSource);
   void DetachMediaSource();
 
-  SubBufferDecoder* CreateSubDecoder(const nsACString& aType);
+  already_AddRefed<SubBufferDecoder> CreateSubDecoder(const nsACString& aType);
-
-  const nsTArray<MediaDecoderReader*>& GetReaders()
-  {
-    ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-    while (mReaders.Length() == 0) {
-      mon.Wait();
-    }
-    return mReaders;
-  }
-
-  void SetVideoReader(MediaDecoderReader* aReader)
-  {
-    MOZ_ASSERT(aReader && !mVideoReader);
-    mVideoReader = aReader;
-  }
-
-  void SetAudioReader(MediaDecoderReader* aReader)
-  {
-    MOZ_ASSERT(aReader && !mAudioReader);
-    mAudioReader = aReader;
-  }
-
-  MediaDecoderReader* GetVideoReader()
-  {
-    return mVideoReader;
-  }
-
-  MediaDecoderReader* GetAudioReader()
-  {
-    return mAudioReader;
-  }
-
-  // Returns the duration in seconds as provided by the attached MediaSource.
-  // If no MediaSource is attached, returns the duration tracked by the decoder.
-  double GetMediaSourceDuration();
 
 private:
+  // The owning MediaSource holds a strong reference to this decoder, and
+  // calls Attach/DetachMediaSource on this decoder to set and clear
+  // mMediaSource.
   dom::MediaSource* mMediaSource;
-
+  nsRefPtr<MediaSourceReader> mReader;
-  nsTArray<nsRefPtr<SubBufferDecoder> > mDecoders;
-  nsTArray<MediaDecoderReader*> mReaders; // Readers owned by Decoders.
-
-  MediaDecoderReader* mVideoReader;
-  MediaDecoderReader* mAudioReader;
 };
 
 } // namespace mozilla

content/media/mediasource/SourceBuffer.cpp

@@ -69,7 +69,7 @@ SubBufferDecoder::GetResource() const
 void
 SubBufferDecoder::SetMediaDuration(int64_t aDuration)
 {
-  mParentDecoder->SetMediaDuration(aDuration);
+  mMediaDuration = aDuration;
 }
 
 void
@@ -103,13 +103,12 @@ SubBufferDecoder::ConvertToByteOffset(double aTime)
   // purposes of eviction this should be adequate since we have the
   // byte threshold as well to ensure data actually gets evicted and
   // we ensure we don't evict before the current playable point.
-  double duration = mParentDecoder->GetMediaSourceDuration();
+  if (mMediaDuration == -1) {
-  if (duration <= 0.0 || IsNaN(duration)) {
     return -1;
   }
   int64_t length = GetResource()->GetLength();
   MOZ_ASSERT(length > 0);
-  int64_t offset = (aTime / duration) * length;
+  int64_t offset = (aTime / (double(mMediaDuration) / USECS_PER_S)) * length;
   return offset;
 }
 

content/media/mediasource/SubBufferDecoder.h

@@ -30,6 +30,11 @@ public:
     mReader = aReader;
   }
 
+  MediaDecoderReader* GetReader()
+  {
+    return mReader;
+  }
+
   virtual ReentrantMonitor& GetReentrantMonitor() MOZ_OVERRIDE;
   virtual bool OnStateMachineThread() const MOZ_OVERRIDE;
   virtual bool OnDecodeThread() const MOZ_OVERRIDE;
@@ -44,8 +49,11 @@ public:
   {
     mReader->NotifyDataArrived(aBuffer, aLength, aOffset);
 
-    // XXX: aOffset makes no sense here, need view of "data timeline".
+    // XXX: Params make no sense to parent decoder as it relates to a
-    mParentDecoder->NotifyDataArrived(aBuffer, aLength, aOffset);
+    // specific SubBufferDecoder's data stream.  Pass bogus values here to
+    // force parent decoder's state machine to recompute end time for
+    // infinite length media.
+    mParentDecoder->NotifyDataArrived(nullptr, 0, 0);
   }
 
   nsresult GetBuffered(dom::TimeRanges* aBuffered)
@@ -58,9 +66,15 @@ public:
   // cached data. Returns -1 if no such value is computable.
   int64_t ConvertToByteOffset(double aTime);
 
+  int64_t GetMediaDuration() MOZ_OVERRIDE
+  {
+    return mMediaDuration;
+  }
+
 private:
   MediaSourceDecoder* mParentDecoder;
   nsAutoPtr<MediaDecoderReader> mReader;
+  int64_t mMediaDuration;
 };
 
 } // namespace mozilla