diff --git a/dom/media/MediaStreamListener.h b/dom/media/MediaStreamListener.h index 637df4d13cc5..2036fa9be3da 100644 --- a/dom/media/MediaStreamListener.h +++ b/dom/media/MediaStreamListener.h @@ -7,10 +7,14 @@ #ifndef MOZILLA_MEDIASTREAMLISTENER_h_ #define MOZILLA_MEDIASTREAMLISTENER_h_ +#include "StreamTracks.h" + namespace mozilla { +class AudioSegment; class MediaStream; class MediaStreamGraph; +class VideoSegment; enum MediaStreamGraphEvent : uint32_t { EVENT_FINISHED, diff --git a/dom/media/MediaStreamVideoSink.cpp b/dom/media/MediaStreamVideoSink.cpp new file mode 100644 index 000000000000..40f2f517e0de --- /dev/null +++ b/dom/media/MediaStreamVideoSink.cpp @@ -0,0 +1,21 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MediaStreamVideoSink.h" + +#include "VideoSegment.h" + +namespace mozilla { +void +MediaStreamVideoSink::NotifyRealtimeTrackData(MediaStreamGraph* aGraph, + StreamTime aTrackOffset, + const MediaSegment& aMedia) +{ + if (aMedia.GetType() == MediaSegment::VIDEO) { + SetCurrentFrames(static_cast(aMedia)); + } +} + +} // namespace mozilla diff --git a/dom/media/MediaStreamVideoSink.h b/dom/media/MediaStreamVideoSink.h new file mode 100644 index 000000000000..344f99de7b13 --- /dev/null +++ b/dom/media/MediaStreamVideoSink.h @@ -0,0 +1,42 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MEDIASTREAMVIDEOSINK_H_ +#define MEDIASTREAMVIDEOSINK_H_ + +#include "mozilla/Pair.h" + +#include "gfxPoint.h" +#include "MediaStreamListener.h" + +namespace mozilla { + +class VideoFrameContainer; + +/** + * Base class of MediaStreamVideoSink family. This is the output of MediaStream. + */ +class MediaStreamVideoSink : public DirectMediaStreamTrackListener { +public: + // Method of DirectMediaStreamTrackListener. + void NotifyRealtimeTrackData(MediaStreamGraph* aGraph, + StreamTime aTrackOffset, + const MediaSegment& aMedia) override; + + // Call on any thread + virtual void SetCurrentFrames(const VideoSegment& aSegment) = 0; + virtual void ClearFrames() = 0; + + virtual VideoFrameContainer* AsVideoFrameContainer() { return nullptr; } + virtual void Invalidate() {} + +protected: + virtual ~MediaStreamVideoSink() {}; +}; + +} // namespace mozilla + +#endif /* MEDIASTREAMVIDEOSINK_H_ */ diff --git a/dom/media/VideoFrameContainer.cpp b/dom/media/VideoFrameContainer.cpp index 145faec1eb68..a0183b0a1439 100644 --- a/dom/media/VideoFrameContainer.cpp +++ b/dom/media/VideoFrameContainer.cpp @@ -14,17 +14,23 @@ using namespace mozilla::layers; namespace mozilla { +PRLogModuleInfo* gVideoFrameContainerLog; +#define CONTAINER_LOG(type, msg) MOZ_LOG(gVideoFrameContainerLog, type, msg) VideoFrameContainer::VideoFrameContainer(dom::HTMLMediaElement* aElement, already_AddRefed aContainer) : mElement(aElement), mImageContainer(aContainer), mMutex("nsVideoFrameContainer"), + mBlackImage(nullptr), mFrameID(0), mIntrinsicSizeChanged(false), mImageSizeChanged(false), mPendingPrincipalHandle(PRINCIPAL_HANDLE_NONE), mFrameIDForPendingPrincipalHandle(0) { NS_ASSERTION(aElement, "aElement must not be null"); NS_ASSERTION(mImageContainer, "aContainer must not be null"); + if (!gVideoFrameContainerLog) { + gVideoFrameContainerLog = PR_NewLogModule("VideoFrameContainer"); + } } VideoFrameContainer::~VideoFrameContainer() @@ -47,6 +53,126 @@ void VideoFrameContainer::UpdatePrincipalHandleForFrameID(const PrincipalHandle& mFrameIDForPendingPrincipalHandle = aFrameID; } +static void +SetImageToBlackPixel(PlanarYCbCrImage* aImage) +{ + uint8_t blackPixel[] = { 0x10, 0x80, 0x80 }; + + PlanarYCbCrData data; + data.mYChannel = blackPixel; + data.mCbChannel = blackPixel + 1; + data.mCrChannel = blackPixel + 2; + data.mYStride = data.mCbCrStride = 1; + data.mPicSize = data.mYSize = data.mCbCrSize = gfx::IntSize(1, 1); + aImage->CopyData(data); +} + +class VideoFrameContainerInvalidateRunnable : public Runnable { +public: + explicit VideoFrameContainerInvalidateRunnable(VideoFrameContainer* aVideoFrameContainer) + : mVideoFrameContainer(aVideoFrameContainer) + {} + NS_IMETHOD Run() + { + MOZ_ASSERT(NS_IsMainThread()); + + mVideoFrameContainer->Invalidate(); + + return NS_OK; + } +private: + RefPtr mVideoFrameContainer; +}; + +void VideoFrameContainer::SetCurrentFrames(const VideoSegment& aSegment) +{ + if (aSegment.IsEmpty()) { + return; + } + + MutexAutoLock lock(mMutex); + + // Collect any new frames produced in this iteration. + AutoTArray newImages; + PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE; + + VideoSegment::ConstChunkIterator iter(aSegment); + while (!iter.IsEnded()) { + VideoChunk chunk = *iter; + + const VideoFrame* frame = &chunk.mFrame; + if (*frame == mLastPlayedVideoFrame) { + iter.Next(); + continue; + } + + Image* image = frame->GetImage(); + CONTAINER_LOG(LogLevel::Verbose, + ("VideoFrameContainer %p writing video frame %p (%d x %d)", + this, image, frame->GetIntrinsicSize().width, + frame->GetIntrinsicSize().height)); + + if (frame->GetForceBlack()) { + if (!mBlackImage) { + mBlackImage = GetImageContainer()->CreatePlanarYCbCrImage(); + if (mBlackImage) { + // Sets the image to a single black pixel, which will be scaled to + // fill the rendered size. + SetImageToBlackPixel(mBlackImage->AsPlanarYCbCrImage()); + } + } + if (mBlackImage) { + image = mBlackImage; + } + } + // Don't append null image to the newImages. + if (!image) { + iter.Next(); + continue; + } + newImages.AppendElement(ImageContainer::NonOwningImage(image, chunk.mTimeStamp)); + + lastPrincipalHandle = chunk.GetPrincipalHandle(); + + mLastPlayedVideoFrame = *frame; + iter.Next(); + } + + // Don't update if there are no changes. + if (newImages.IsEmpty()) { + return; + } + + AutoTArray images; + + bool principalHandleChanged = + lastPrincipalHandle != PRINCIPAL_HANDLE_NONE && + lastPrincipalHandle != GetLastPrincipalHandle(); + + // Add the frames from this iteration. + for (auto& image : newImages) { + image.mFrameID = NewFrameID(); + images.AppendElement(image); + } + + if (principalHandleChanged) { + UpdatePrincipalHandleForFrameID(lastPrincipalHandle, + newImages.LastElement().mFrameID); + } + + SetCurrentFramesLocked(mLastPlayedVideoFrame.GetIntrinsicSize(), images); + nsCOMPtr event = + new VideoFrameContainerInvalidateRunnable(this); + NS_DispatchToMainThread(event.forget()); + + images.ClearAndRetainStorage(); +} + +void VideoFrameContainer::ClearFrames() +{ + ClearFutureFrames(); +} + void VideoFrameContainer::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage, const TimeStamp& aTargetTime) diff --git a/dom/media/VideoFrameContainer.h b/dom/media/VideoFrameContainer.h index 4c5542b2715a..38c5736b4765 100644 --- a/dom/media/VideoFrameContainer.h +++ b/dom/media/VideoFrameContainer.h @@ -13,6 +13,8 @@ #include "nsCOMPtr.h" #include "ImageContainer.h" #include "MediaSegment.h" +#include "MediaStreamVideoSink.h" +#include "VideoSegment.h" namespace mozilla { @@ -29,19 +31,21 @@ class HTMLMediaElement; * element itself ... well, maybe we could, but it could be risky and/or * confusing. */ -class VideoFrameContainer { - ~VideoFrameContainer(); +class VideoFrameContainer : public MediaStreamVideoSink { + virtual ~VideoFrameContainer(); public: typedef layers::ImageContainer ImageContainer; typedef layers::Image Image; - NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoFrameContainer) - VideoFrameContainer(dom::HTMLMediaElement* aElement, already_AddRefed aContainer); // Call on any thread + virtual void SetCurrentFrames(const VideoSegment& aSegment) override; + virtual void ClearFrames() override; + void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage, + const TimeStamp& aTargetTime); // Returns the last principalHandle we notified mElement about. PrincipalHandle GetLastPrincipalHandle(); // We will notify mElement that aPrincipalHandle has been applied when all @@ -49,14 +53,13 @@ public: // aFrameID is ignored if aPrincipalHandle already is our pending principalHandle. void UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle, const ImageContainer::FrameID& aFrameID); - void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage, - const TimeStamp& aTargetTime); void SetCurrentFrames(const gfx::IntSize& aIntrinsicSize, const nsTArray& aImages); void ClearCurrentFrame(const gfx::IntSize& aIntrinsicSize) { SetCurrentFrames(aIntrinsicSize, nsTArray()); } + VideoFrameContainer* AsVideoFrameContainer() override { return this; } void ClearCurrentFrame(); // Make the current frame the only frame in the container, i.e. discard @@ -80,7 +83,7 @@ public: INVALIDATE_DEFAULT, INVALIDATE_FORCE }; - void Invalidate() { InvalidateWithFlags(INVALIDATE_DEFAULT); } + void Invalidate() override { InvalidateWithFlags(INVALIDATE_DEFAULT); } void InvalidateWithFlags(uint32_t aFlags); ImageContainer* GetImageContainer(); void ForgetElement() { mElement = nullptr; } @@ -98,6 +101,9 @@ protected: // mMutex protects all the fields below. Mutex mMutex; + // Once the frame is forced to black, we initialize mBlackImage for following + // frames. + RefPtr mBlackImage; // The intrinsic size is the ideal size which we should render the // ImageContainer's current Image at. // This can differ from the Image's actual size when the media resource @@ -107,6 +113,9 @@ protected: // We maintain our own mFrameID which is auto-incremented at every // SetCurrentFrame() or NewFrameID() call. ImageContainer::FrameID mFrameID; + // We record the last played video frame to avoid playing the frame again + // with a different frame id. + VideoFrame mLastPlayedVideoFrame; // True when the intrinsic size has been changed by SetCurrentFrame() since // the last call to Invalidate(). // The next call to Invalidate() will recalculate diff --git a/dom/media/VideoSegment.h b/dom/media/VideoSegment.h index 0eef313ee875..749646fbc298 100644 --- a/dom/media/VideoSegment.h +++ b/dom/media/VideoSegment.h @@ -137,6 +137,12 @@ public: { return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); } + + bool IsEmpty() const + { + return mChunks.IsEmpty(); + } + }; } // namespace mozilla diff --git a/dom/media/moz.build b/dom/media/moz.build index 55345511b3ad..c5542288496c 100644 --- a/dom/media/moz.build +++ b/dom/media/moz.build @@ -127,6 +127,7 @@ EXPORTS += [ 'MediaStatistics.h', 'MediaStreamGraph.h', 'MediaStreamListener.h', + 'MediaStreamVideoSink.h', 'MediaTimer.h', 'MediaTrack.h', 'MediaTrackList.h', @@ -239,6 +240,7 @@ UNIFIED_SOURCES += [ 'MediaStreamGraph.cpp', 'MediaStreamListener.cpp', 'MediaStreamTrack.cpp', + 'MediaStreamVideoSink.cpp', 'MediaTimer.cpp', 'MediaTrack.cpp', 'MediaTrackList.cpp',