From 7a07911d5f35c756552ba603f58ed3e76d55c20b Mon Sep 17 00:00:00 2001 From: ctai Date: Fri, 27 May 2016 14:33:50 +0800 Subject: [PATCH] Bug 1201363 - Let MediaStreamVideoSink bind with particular video track. r=jesup MozReview-Commit-ID: FcjnmDKuRQI --HG-- extra : amend_source : 5591de347c10931f4188fa550c96a253d0c0b5f3 --- dom/camera/CameraPreviewMediaStream.cpp | 20 ++++----- dom/camera/CameraPreviewMediaStream.h | 20 ++++----- dom/media/MediaStreamGraph.cpp | 57 +++++++++++++++++-------- dom/media/MediaStreamGraph.h | 17 +++++--- dom/media/TrackUnionStream.h | 3 ++ 5 files changed, 73 insertions(+), 44 deletions(-) diff --git a/dom/camera/CameraPreviewMediaStream.cpp b/dom/camera/CameraPreviewMediaStream.cpp index 095b2c60543e..2ba86c03f70a 100644 --- a/dom/camera/CameraPreviewMediaStream.cpp +++ b/dom/camera/CameraPreviewMediaStream.cpp @@ -60,18 +60,18 @@ CameraPreviewMediaStream::RemoveAudioOutput(void* aKey) } void -CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink) +CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) { MutexAutoLock lock(mMutex); RefPtr sink = aSink; - AddVideoOutputImpl(sink.forget()); + AddVideoOutputImpl(sink.forget(), aID); } void -CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink) +CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) { MutexAutoLock lock(mMutex); - RemoveVideoOutputImpl(aSink); + RemoveVideoOutputImpl(aSink, aID); } void @@ -126,8 +126,8 @@ CameraPreviewMediaStream::Invalidate() { MutexAutoLock lock(mMutex); --mInvalidatePending; - for (MediaStreamVideoSink* sink : mVideoOutputs) { - VideoFrameContainer* output = sink->AsVideoFrameContainer(); + for (const TrackBound& sink : mVideoOutputs) { + VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer(); if (!output) { continue; } @@ -168,8 +168,8 @@ CameraPreviewMediaStream::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Im mDiscardedFrames = 0; TimeStamp now = TimeStamp::Now(); - for (MediaStreamVideoSink* sink : mVideoOutputs) { - VideoFrameContainer* output = sink->AsVideoFrameContainer(); + for (const TrackBound& sink : mVideoOutputs) { + VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer(); if (!output) { continue; } @@ -187,8 +187,8 @@ CameraPreviewMediaStream::ClearCurrentFrame() { MutexAutoLock lock(mMutex); - for (MediaStreamVideoSink* sink : mVideoOutputs) { - VideoFrameContainer* output = sink->AsVideoFrameContainer(); + for (const TrackBound& sink : mVideoOutputs) { + VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer(); if (!output) { continue; } diff --git a/dom/camera/CameraPreviewMediaStream.h b/dom/camera/CameraPreviewMediaStream.h index 6fee30f3a26c..b765467ccd99 100644 --- a/dom/camera/CameraPreviewMediaStream.h +++ b/dom/camera/CameraPreviewMediaStream.h @@ -43,16 +43,16 @@ class CameraPreviewMediaStream : public ProcessedMediaStream public: CameraPreviewMediaStream(); - virtual void AddAudioOutput(void* aKey) override; - virtual void SetAudioOutputVolume(void* aKey, float aVolume) override; - virtual void RemoveAudioOutput(void* aKey) override; - virtual void AddVideoOutput(MediaStreamVideoSink* aSink) override; - virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink) override; - virtual void Suspend() override {} - virtual void Resume() override {} - virtual void AddListener(MediaStreamListener* aListener) override; - virtual void RemoveListener(MediaStreamListener* aListener) override; - virtual void Destroy() override; + void AddAudioOutput(void* aKey) override; + void SetAudioOutputVolume(void* aKey, float aVolume) override; + void RemoveAudioOutput(void* aKey) override; + void AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) override; + void RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) override; + void Suspend() override {} + void Resume() override {} + void AddListener(MediaStreamListener* aListener) override; + void RemoveListener(MediaStreamListener* aListener) override; + void Destroy() override; void OnPreviewStateChange(bool aActive); void Invalidate(); diff --git a/dom/media/MediaStreamGraph.cpp b/dom/media/MediaStreamGraph.cpp index 666e0370a793..b0d0de1e5bcc 100644 --- a/dom/media/MediaStreamGraph.cpp +++ b/dom/media/MediaStreamGraph.cpp @@ -1025,10 +1025,10 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream) // "Call MediaStreamVideoSink::setCurrentFrames in SourceMediaStream::AppendToTrack." // of this bug. // This is a temp workaround to pass the build and test. - if (!aStream->mVideoOutputs[0]->AsVideoFrameContainer()) { + if (!aStream->mVideoOutputs[0].mListener->AsVideoFrameContainer()) { return; } - blackImage = aStream->mVideoOutputs[0]->AsVideoFrameContainer()-> + blackImage = aStream->mVideoOutputs[0].mListener->AsVideoFrameContainer()-> GetImageContainer()->CreatePlanarYCbCrImage(); if (blackImage) { // Sets the image to a single black pixel, which will be scaled to @@ -1053,8 +1053,8 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream) AutoTArray images; bool haveMultipleImages = false; - for (MediaStreamVideoSink* sink : aStream->mVideoOutputs) { - VideoFrameContainer* output = sink->AsVideoFrameContainer(); + for (const TrackBound& sink : aStream->mVideoOutputs) { + VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer(); if (!output) { continue; } @@ -2271,55 +2271,76 @@ MediaStream::RemoveAudioOutput(void* aKey) } void -MediaStream::AddVideoOutputImpl(already_AddRefed aSink) +MediaStream::AddVideoOutputImpl(already_AddRefed aSink, + TrackID aID) { RefPtr sink = aSink; STREAM_LOG(LogLevel::Info, ("MediaStream %p Adding MediaStreamVideoSink %p as output", this, sink.get())); - *mVideoOutputs.AppendElement() = sink.forget(); + MOZ_ASSERT(aID != TRACK_NONE); + for (auto entry : mVideoOutputs) { + if (entry.mListener == sink && + (entry.mTrackID == TRACK_ANY || entry.mTrackID == aID)) { + return; + } + } + TrackBound* l = mVideoOutputs.AppendElement(); + l->mListener = sink; + l->mTrackID = aID; } void -MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink) +MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink, + TrackID aID) { STREAM_LOG(LogLevel::Info, ("MediaStream %p Removing MediaStreamVideoSink %p as output", this, aSink)); + MOZ_ASSERT(aID != TRACK_NONE); + // Ensure that any frames currently queued for playback by the compositor // are removed. aSink->ClearFrames(); - mVideoOutputs.RemoveElement(aSink); + for (size_t i = 0; i < mVideoOutputs.Length(); ++i) { + if (mVideoOutputs[i].mListener == aSink && + (mVideoOutputs[i].mTrackID == TRACK_ANY || + mVideoOutputs[i].mTrackID == aID)) { + mVideoOutputs.RemoveElementAt(i); + } + } } void -MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink) +MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) { class Message : public ControlMessage { public: - Message(MediaStream* aStream, MediaStreamVideoSink* aSink) : - ControlMessage(aStream), mSink(aSink) {} + Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) : + ControlMessage(aStream), mSink(aSink), mID(aID) {} void Run() override { - mStream->AddVideoOutputImpl(mSink.forget()); + mStream->AddVideoOutputImpl(mSink.forget(), mID); } RefPtr mSink; + TrackID mID; }; - GraphImpl()->AppendMessage(MakeUnique(this, aSink)); + GraphImpl()->AppendMessage(MakeUnique(this, aSink, aID)); } void -MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink) +MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) { class Message : public ControlMessage { public: - Message(MediaStream* aStream, MediaStreamVideoSink* aSink) : - ControlMessage(aStream), mSink(aSink) {} + Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) : + ControlMessage(aStream), mSink(aSink), mID(aID) {} void Run() override { - mStream->RemoveVideoOutputImpl(mSink); + mStream->RemoveVideoOutputImpl(mSink, mID); } RefPtr mSink; + TrackID mID; }; - GraphImpl()->AppendMessage(MakeUnique(this, aSink)); + GraphImpl()->AppendMessage(MakeUnique(this, aSink, aID)); } void diff --git a/dom/media/MediaStreamGraph.h b/dom/media/MediaStreamGraph.h index 36595df59ad3..f86a044b057b 100644 --- a/dom/media/MediaStreamGraph.h +++ b/dom/media/MediaStreamGraph.h @@ -17,7 +17,6 @@ #include "nsIRunnable.h" #include "VideoSegment.h" #include "StreamTracks.h" -#include "MediaStreamVideoSink.h" #include "MainThreadUtils.h" #include "StreamTracks.h" #include "nsAutoPtr.h" @@ -167,8 +166,10 @@ class MediaInputPort; class MediaStreamGraphImpl; class MediaStreamListener; class MediaStreamTrackListener; +class MediaStreamVideoSink; class ProcessedMediaStream; class SourceMediaStream; +class TrackUnionStream; enum MediaStreamGraphEvent : uint32_t; enum TrackEventCommand : uint32_t; @@ -294,8 +295,10 @@ public: // Since a stream can be played multiple ways, we need to be able to // play to multiple MediaStreamVideoSinks. // Only the first enabled video track is played. - virtual void AddVideoOutput(MediaStreamVideoSink* aSink); - virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink); + virtual void AddVideoOutput(MediaStreamVideoSink* aSink, + TrackID aID = TRACK_ANY); + virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink, + TrackID aID = TRACK_ANY); // Explicitly suspend. Useful for example if a media element is pausing // and we need to stop its stream emitting its buffered data. As soon as the // Suspend message reaches the graph, the stream stops processing. It @@ -404,6 +407,7 @@ public: virtual SourceMediaStream* AsSourceStream() { return nullptr; } virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; } virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; } + virtual TrackUnionStream* AsTrackUnionStream() { return nullptr; } // These Impl methods perform the core functionality of the control methods // above, on the media graph thread. @@ -424,8 +428,9 @@ public: return !mAudioOutputs.IsEmpty(); } void RemoveAudioOutputImpl(void* aKey); - void AddVideoOutputImpl(already_AddRefed aSink); - void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink); + void AddVideoOutputImpl(already_AddRefed aSink, + TrackID aID); + void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink, TrackID aID); void AddListenerImpl(already_AddRefed aListener); void RemoveListenerImpl(MediaStreamListener* aListener); void RemoveAllListenersImpl(); @@ -584,7 +589,7 @@ protected: float mVolume; }; nsTArray mAudioOutputs; - nsTArray> mVideoOutputs; + nsTArray> mVideoOutputs; // We record the last played video frame to avoid playing the frame again // with a different frame id. VideoFrame mLastPlayedVideoFrame; diff --git a/dom/media/TrackUnionStream.h b/dom/media/TrackUnionStream.h index 96c46190a49c..4c935e96ad19 100644 --- a/dom/media/TrackUnionStream.h +++ b/dom/media/TrackUnionStream.h @@ -19,6 +19,9 @@ class TrackUnionStream : public ProcessedMediaStream { public: explicit TrackUnionStream(); + virtual TrackUnionStream* AsTrackUnionStream() override { return this; } + friend class DOMMediaStream; + void RemoveInput(MediaInputPort* aPort) override; void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;