Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-10 11:55:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1266646 - Change HTMLMediaElement::StreamSizeListerner to inherit MediaStreamTrackDirectListener. r=jesup, r=pehrsons

Browse Source 
MozReview-Commit-ID: HnNv9BnlbDy

--HG--
extra : rebase_source : 7d04886895910b00964c2ce36a337873195b12fa
This commit is contained in:
ctai 2016-05-10 17:02:15 +08:00
parent 916ef92d75
commit 4ab1e6cc57
9 changed files with 192 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
dom/html/HTMLMediaElement.cpp
View File

@ -273,6 +273,56 @@ public:
}
};
/**
* This listener observes the first video frame to arrive with a non-empty size,
* and calls HTMLMediaElement::ReceivedMediaStreamInitialSize() with that size.
*/
class HTMLMediaElement::StreamSizeListener : public MediaStreamTrackDirectListener {
public:
explicit StreamSizeListener(HTMLMediaElement* aElement) :
mElement(aElement),
mInitialSizeFound(false)
{}
void Forget() { mElement = nullptr; }
void ReceivedSize(gfx::IntSize aSize)
{
if (!mElement) {
return;
}
RefPtr<HTMLMediaElement> deathGrip = mElement;
mElement->UpdateInitialMediaSize(aSize);
}
void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
StreamTime aTrackOffset,
const MediaSegment& aMedia) override
{
if (mInitialSizeFound || aMedia.GetType() != MediaSegment::VIDEO) {
return;
}
const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0,0)) {
mInitialSizeFound = true;
nsCOMPtr<nsIRunnable> event =
NewRunnableMethod<gfx::IntSize>(
this, &StreamSizeListener::ReceivedSize,
c->mFrame.GetIntrinsicSize());
aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
return;
}
}
}
private:
// These fields may only be accessed on the main thread
HTMLMediaElement* mElement;
// These fields may only be accessed on the MSG thread
bool mInitialSizeFound;
};
/**
* There is a reference cycle involving this class: MediaLoadListener
* holds a reference to the HTMLMediaElement, which holds a reference
@ -484,6 +534,7 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTM
#ifdef MOZ_EME
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaKeys)
#endif
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSelectedVideoStreamTrack)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTMLElement)
@ -510,6 +561,7 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTMLE
#ifdef MOZ_EME
NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaKeys)
#endif
NS_IMPL_CYCLE_COLLECTION_UNLINK(mSelectedVideoStreamTrack)
NS_IMPL_CYCLE_COLLECTION_UNLINK_END
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(HTMLMediaElement)
@ -724,6 +776,14 @@ void HTMLMediaElement::AbortExistingLoads()
bool fireTimeUpdate = false;
// We need to remove StreamSizeListener before VideoTracks get emptied.
if (mMediaStreamSizeListener) {
mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
mSelectedVideoStreamTrack = nullptr;
mMediaStreamSizeListener->Forget();
mMediaStreamSizeListener = nullptr;
}
// When aborting the existing loads, empty the objects in audio track list and
// video track list, no events (in particular, no removetrack events) are
// fired as part of this. Ending MediaStream sends track ended notifications,
@ -3216,59 +3276,6 @@ private:
bool mPendingNotifyOutput;
};
/**
* This listener observes the first video frame to arrive with a non-empty size,
* and calls HTMLMediaElement::ReceivedMediaStreamInitialSize() with that size.
*/
class HTMLMediaElement::StreamSizeListener : public MediaStreamListener {
public:
explicit StreamSizeListener(HTMLMediaElement* aElement) :
mElement(aElement),
mInitialSizeFound(false)
{}
void Forget() { mElement = nullptr; }
void ReceivedSize(gfx::IntSize aSize)
{
if (!mElement) {
return;
}
RefPtr<HTMLMediaElement> deathGrip = mElement;
mElement->UpdateInitialMediaSize(aSize);
}
void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
StreamTime aTrackOffset,
uint32_t aTrackEvents,
const MediaSegment& aQueuedMedia,
MediaStream* aInputStream,
TrackID aInputTrackID) override
{
if (mInitialSizeFound || aQueuedMedia.GetType() != MediaSegment::VIDEO) {
return;
}
const VideoSegment& video = static_cast<const VideoSegment&>(aQueuedMedia);
for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0,0)) {
mInitialSizeFound = true;
nsCOMPtr<nsIRunnable> event =
NewRunnableMethod<gfx::IntSize>(
this, &StreamSizeListener::ReceivedSize,
c->mFrame.GetIntrinsicSize());
aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
return;
}
}
}
private:
// These fields may only be accessed on the main thread
HTMLMediaElement* mElement;
// These fields may only be accessed on the MSG thread
bool mInitialSizeFound;
};
class HTMLMediaElement::MediaStreamTracksAvailableCallback:
public OnTracksAvailableCallback
{
@ -3387,9 +3394,6 @@ void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream)
RefPtr<MediaStream> stream = GetSrcMediaStream();
if (stream) {
stream->SetAudioChannelType(mAudioChannel);
mMediaStreamSizeListener = new StreamSizeListener(this);
stream->AddListener(mMediaStreamSizeListener);
}
UpdateSrcMediaStreamPlaying();
@ -3420,10 +3424,8 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback()
UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM);
if (mMediaStreamSizeListener) {
RefPtr<MediaStream> stream = GetSrcMediaStream();
if (stream) {
stream->RemoveListener(mMediaStreamSizeListener);
}
mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
mSelectedVideoStreamTrack = nullptr;
mMediaStreamSizeListener->Forget();
mMediaStreamSizeListener = nullptr;
}
@ -3459,7 +3461,8 @@ CreateVideoTrack(VideoStreamTrack* aStreamTrack)
aStreamTrack->GetLabel(label);
return MediaTrackList::CreateVideoTrack(id, NS_LITERAL_STRING("main"),
label, EmptyString());
label, EmptyString(),
aStreamTrack);
}
void HTMLMediaElement::ConstructMediaTracks()
@ -3491,6 +3494,11 @@ void HTMLMediaElement::ConstructMediaTracks()
// must be selected.
int index = firstEnabledVideo >= 0 ? firstEnabledVideo : 0;
(*VideoTracks())[index]->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
VideoTrack* track = (*VideoTracks())[index];
VideoStreamTrack* streamTrack = track->GetVideoStreamTrack();
mMediaStreamSizeListener = new StreamSizeListener(this);
streamTrack->AddDirectListener(mMediaStreamSizeListener);
mSelectedVideoStreamTrack = streamTrack;
}
}
@ -3511,8 +3519,20 @@ HTMLMediaElement::NotifyMediaStreamTrackAdded(const RefPtr<MediaStreamTrack>& aT
RefPtr<AudioTrack> audioTrack = CreateAudioTrack(t);
AudioTracks()->AddTrack(audioTrack);
} else if (VideoStreamTrack* t = aTrack->AsVideoStreamTrack()) {
// TODO: Fix this per the spec on bug 1273443.
int32_t selectedIndex = VideoTracks()->SelectedIndex();
RefPtr<VideoTrack> videoTrack = CreateVideoTrack(t);
VideoTracks()->AddTrack(videoTrack);
// New MediaStreamTrack added, set the new added video track as selected
// video track when there is no selected track.
if (selectedIndex == -1) {
MOZ_ASSERT(!mSelectedVideoStreamTrack);
videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
mMediaStreamSizeListener = new StreamSizeListener(this);
t->AddDirectListener(mMediaStreamSizeListener);
mSelectedVideoStreamTrack = t;
}
}
}
@ -3531,6 +3551,47 @@ HTMLMediaElement::NotifyMediaStreamTrackRemoved(const RefPtr<MediaStreamTrack>&
AudioTracks()->RemoveTrack(t);
} else if (MediaTrack* t = VideoTracks()->GetTrackById(id)) {
VideoTracks()->RemoveTrack(t);
// TODO: Fix this per the spec on bug 1273443.
// If the removed media stream track is selected video track and there are
// still video tracks, change the selected video track to the first
// remaining track.
if (aTrack == mSelectedVideoStreamTrack) {
// The mMediaStreamSizeListener might already reset to nullptr.
if (mMediaStreamSizeListener) {
mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
}
mSelectedVideoStreamTrack = nullptr;
MOZ_ASSERT(mSrcStream);
nsTArray<RefPtr<VideoStreamTrack>> tracks;
mSrcStream->GetVideoTracks(tracks);
for (const RefPtr<VideoStreamTrack>& track : tracks) {
if (track->Ended() || !track->Enabled()) {
continue;
}
if (track->Enabled()) {
nsAutoString trackId;
track->GetId(trackId);
MediaTrack* videoTrack = VideoTracks()->GetTrackById(trackId);
MOZ_ASSERT(videoTrack);
videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
if (mMediaStreamSizeListener) {
track->AddDirectListener(mMediaStreamSizeListener);
}
mSelectedVideoStreamTrack = track;
return;
}
}
// There is no enabled video track existing, clean the
// mMediaStreamSizeListener.
if (mMediaStreamSizeListener) {
mMediaStreamSizeListener->Forget();
mMediaStreamSizeListener = nullptr;
}
}
} else {
// XXX (bug 1208328) Uncomment this when DOMMediaStream doesn't call
// NotifyTrackRemoved multiple times for the same track, i.e., when it
@ -4445,10 +4506,7 @@ void HTMLMediaElement::UpdateInitialMediaSize(const nsIntSize& aSize)
if (!mMediaStreamSizeListener) {
return;
}
RefPtr<MediaStream> stream = GetSrcMediaStream();
if (stream) {
stream->RemoveListener(mMediaStreamSizeListener);
}
mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
mMediaStreamSizeListener->Forget();
mMediaStreamSizeListener = nullptr;
}

2
dom/html/HTMLMediaElement.h
View File

@ -1239,6 +1239,8 @@ protected:
// Holds a reference to the size-getting MediaStreamListener attached to
// mSrcStream.
RefPtr<StreamSizeListener> mMediaStreamSizeListener;
// The selected video stream track which contained mMediaStreamSizeListener.
RefPtr<VideoStreamTrack> mSelectedVideoStreamTrack;
// Holds a reference to the MediaSource, if any, referenced by the src
// attribute on the media element.

17
dom/media/MediaStreamGraph.cpp
View File

@ -2927,15 +2927,20 @@ SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefed<MediaStreamTrackD
TrackData* data;
bool found;
bool isAudio;
bool isVideo;
RefPtr<MediaStreamTrackDirectListener> listener = aListener;
STREAM_LOG(LogLevel::Debug, ("Adding direct track listener %p bound to track %d to source stream %p",
listener.get(), aTrackID, this));
{
MutexAutoLock lock(mMutex);
data = FindDataForTrack(aTrackID);
found = !!data;
isAudio = found && data->mData->GetType() == MediaSegment::AUDIO;
if (found && isAudio) {
if (found) {
isAudio = data->mData->GetType() == MediaSegment::AUDIO;
isVideo = data->mData->GetType() == MediaSegment::VIDEO;
}
if (found && (isAudio || isVideo)) {
TrackBound<MediaStreamTrackDirectListener>* sourceListener =
mDirectTrackListeners.AppendElement();
sourceListener->mListener = listener;
@ -2949,11 +2954,11 @@ SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefed<MediaStreamTrackD
MediaStreamTrackDirectListener::InstallationResult::TRACK_NOT_FOUND_AT_SOURCE);
return;
}
if (!isAudio) {
STREAM_LOG(LogLevel::Warning, ("Source track for direct track listener %p is not audio",
if (!isAudio && !isVideo) {
STREAM_LOG(LogLevel::Warning, ("Source track for direct track listener %p is unknown",
listener.get()));
listener->NotifyDirectListenerInstalled(
MediaStreamTrackDirectListener::InstallationResult::TRACK_TYPE_NOT_SUPPORTED);
// It is not a video or audio track.
MOZ_ASSERT(true);
return;
}
STREAM_LOG(LogLevel::Debug, ("Added direct track listener %p", listener.get()));

16
dom/media/MediaStreamGraph.h
View File

@ -344,7 +344,8 @@ public:
* didn't exist. This should only happen if you try to install the listener
* directly to a SourceMediaStream that doesn't contain the given TrackID.
* TRACK_TYPE_NOT_SUPPORTED
* This is the failure when you install the listener to a non-audio track.
* This is the failure when you install the listener to a
* non-(audio or video) track.
* STREAM_NOT_SUPPORTED
* While looking for the data source of this track, we found a MediaStream
* that is not a SourceMediaStream or a TrackUnionStream.
@ -354,7 +355,6 @@ public:
*/
enum class InstallationResult {
TRACK_NOT_FOUND_AT_SOURCE,
TRACK_TYPE_NOT_SUPPORTED,
STREAM_NOT_SUPPORTED,
SUCCESS
};
@ -370,6 +370,15 @@ protected:
aTo.AppendNullData(aFrom.GetDuration());
}
void MirrorAndDisableSegment(VideoSegment& aFrom, VideoSegment& aTo)
{
aTo.Clear();
for (VideoSegment::ChunkIterator it(aFrom); !it.IsEnded(); it.Next()) {
aTo.AppendFrame(do_AddRef(it->mFrame.GetImage()), it->GetDuration(),
it->mFrame.GetIntrinsicSize(), it->GetPrincipalHandle(), true);
}
}
void NotifyRealtimeTrackDataAndApplyTrackDisabling(MediaStreamGraph* aGraph,
StreamTime aTrackOffset,
MediaSegment& aMedia)
@ -385,6 +394,9 @@ protected:
if (aMedia.GetType() == MediaSegment::AUDIO) {
MirrorAndDisableSegment(static_cast<AudioSegment&>(aMedia),
static_cast<AudioSegment&>(*mMedia));
} else if (aMedia.GetType() == MediaSegment::VIDEO) {
MirrorAndDisableSegment(static_cast<VideoSegment&>(aMedia),
static_cast<VideoSegment&>(*mMedia));
} else {
MOZ_CRASH("Unsupported media type");
}

6
dom/media/MediaTrackList.cpp
View File

@ -9,6 +9,7 @@
#include "mozilla/AsyncEventDispatcher.h"
#include "mozilla/dom/HTMLMediaElement.h"
#include "mozilla/dom/AudioTrack.h"
#include "mozilla/dom/VideoStreamTrack.h"
#include "mozilla/dom/VideoTrack.h"
#include "mozilla/dom/TrackEvent.h"
#include "nsThreadUtils.h"
@ -106,9 +107,10 @@ already_AddRefed<VideoTrack>
MediaTrackList::CreateVideoTrack(const nsAString& aId,
const nsAString& aKind,
const nsAString& aLabel,
const nsAString& aLanguage)
const nsAString& aLanguage,
VideoStreamTrack* aVideoTrack)
{
RefPtr<VideoTrack> track = new VideoTrack(aId, aKind, aLabel, aLanguage);
RefPtr<VideoTrack> track = new VideoTrack(aId, aKind, aLabel, aLanguage, aVideoTrack);
return track.forget();
}

6
dom/media/MediaTrackList.h
View File

@ -20,6 +20,7 @@ class AudioTrackList;
class VideoTrackList;
class AudioTrack;
class VideoTrack;
class VideoStreamTrack;
/**
* Base class of AudioTrackList and VideoTrackList. The AudioTrackList and
@ -58,11 +59,14 @@ public:
const nsAString& aLanguage,
bool aEnabled);
// For the case of src of HTMLMediaElement is non-MediaStream, leave the
// aVideoTrack as default(nullptr).
static already_AddRefed<VideoTrack>
CreateVideoTrack(const nsAString& aId,
const nsAString& aKind,
const nsAString& aLabel,
const nsAString& aLanguage);
const nsAString& aLanguage,
VideoStreamTrack* aVideoTrack = nullptr);
virtual void EmptyTracks();

16
dom/media/VideoTrack.cpp
View File

@ -5,6 +5,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "mozilla/dom/HTMLMediaElement.h"
#include "mozilla/dom/VideoStreamTrack.h"
#include "mozilla/dom/VideoTrack.h"
#include "mozilla/dom/VideoTrackBinding.h"
#include "mozilla/dom/VideoTrackList.h"
@ -15,12 +16,25 @@ namespace dom {
VideoTrack::VideoTrack(const nsAString& aId,
const nsAString& aKind,
const nsAString& aLabel,
const nsAString& aLanguage)
const nsAString& aLanguage,
VideoStreamTrack* aStreamTarck)
: MediaTrack(aId, aKind, aLabel, aLanguage)
, mSelected(false)
, mVideoStreamTrack(aStreamTarck)
{
}
VideoTrack::~VideoTrack()
{
}
NS_IMPL_CYCLE_COLLECTION_INHERITED(VideoTrack, MediaTrack, mVideoStreamTrack)
NS_IMPL_ADDREF_INHERITED(VideoTrack, MediaTrack)
NS_IMPL_RELEASE_INHERITED(VideoTrack, MediaTrack)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(VideoTrack)
NS_INTERFACE_MAP_END_INHERITING(MediaTrack)
JSObject*
VideoTrack::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto)
{

15
dom/media/VideoTrack.h
View File

@ -13,6 +13,7 @@ namespace mozilla {
namespace dom {
class VideoTrackList;
class VideoStreamTrack;
class VideoTrack : public MediaTrack
{
@ -20,7 +21,11 @@ public:
VideoTrack(const nsAString& aId,
const nsAString& aKind,
const nsAString& aLabel,
const nsAString& aLanguage);
const nsAString& aLanguage,
VideoStreamTrack* aStreamTarck = nullptr);
NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(VideoTrack, MediaTrack)
JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) override;
@ -36,6 +41,11 @@ public:
// aFlags contains FIRE_NO_EVENTS because no events are fired in such cases.
void SetEnabledInternal(bool aEnabled, int aFlags) override;
// Get associated video stream track when the video track comes from
// MediaStream. This might be nullptr when the src of owning HTMLMediaElement
// is not MediaStream.
VideoStreamTrack* GetVideoStreamTrack() { return mVideoStreamTrack; }
// WebIDL
bool Selected() const
{
@ -48,7 +58,10 @@ public:
void SetSelected(bool aSelected);
private:
virtual ~VideoTrack();
bool mSelected;
RefPtr<VideoStreamTrack> mVideoStreamTrack;
};
} // namespace dom

8
dom/media/VideoTrackList.cpp
View File

@ -31,18 +31,18 @@ VideoTrackList::RemoveTrack(const RefPtr<MediaTrack>& aTrack)
// need to be done after RemoveTrack. Also the call of
// |MediaTrackList::RemoveTrack| is necessary even when mSelectedIndex = -1.
bool found;
VideoTrack* videoTrack = IndexedGetter(mSelectedIndex, found);
VideoTrack* selectedVideoTrack = IndexedGetter(mSelectedIndex, found);
MediaTrackList::RemoveTrack(aTrack);
if (mSelectedIndex == -1) {
// There was no selected track and we don't select another track on removal.
return;
}
MOZ_ASSERT(found, "When mSelectedIndex is set it should point to a track");
MOZ_ASSERT(videoTrack, "The mSelectedIndex should be set to video track only");
MOZ_ASSERT(selectedVideoTrack, "The mSelectedIndex should be set to video track only");
// Let the caller of RemoveTrack deal with choosing the new selected track if
// it removes the currently-selected track.
if (aTrack == videoTrack) {
if (aTrack == selectedVideoTrack) {
mSelectedIndex = -1;
return;
}
@ -51,7 +51,7 @@ VideoTrackList::RemoveTrack(const RefPtr<MediaTrack>& aTrack)
// currently-selected video track. We need to find the new location of the
// selected track.
for (size_t ix = 0; ix < mTracks.Length(); ix++) {
if (mTracks[ix] == videoTrack) {
if (mTracks[ix] == selectedVideoTrack) {
mSelectedIndex = ix;
return;
}