diff --git a/dom/html/HTMLMediaElement.cpp b/dom/html/HTMLMediaElement.cpp index 38f7be612959..a5d536cc9532 100644 --- a/dom/html/HTMLMediaElement.cpp +++ b/dom/html/HTMLMediaElement.cpp @@ -938,7 +938,6 @@ void HTMLMediaElement::AbortExistingLoads() // We need to remove StreamSizeListener before VideoTracks get emptied. if (mMediaStreamSizeListener) { mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener); - mSelectedVideoStreamTrack = nullptr; mMediaStreamSizeListener->Forget(); mMediaStreamSizeListener = nullptr; } @@ -1248,8 +1247,9 @@ void HTMLMediaElement::NotifyMediaTrackEnabled(MediaTrack* aTrack) nsString id; aTrack->GetId(id); - LOG(LogLevel::Debug, ("MediaElement %p MediaTrack with id %s enabled", - this, NS_ConvertUTF16toUTF8(id).get())); + LOG(LogLevel::Debug, ("MediaElement %p %sTrack with id %s enabled", + this, aTrack->AsAudioTrack() ? "Audio" : "Video", + NS_ConvertUTF16toUTF8(id).get())); #endif MOZ_ASSERT((aTrack->AsAudioTrack() && aTrack->AsAudioTrack()->Enabled()) || @@ -1258,7 +1258,46 @@ void HTMLMediaElement::NotifyMediaTrackEnabled(MediaTrack* aTrack) if (aTrack->AsAudioTrack()) { SetMutedInternal(mMuted & ~MUTED_BY_AUDIO_TRACK); } else if (aTrack->AsVideoTrack()) { + if (!IsVideo()) { + MOZ_ASSERT(false); + return; + } mDisableVideo = false; + } else { + MOZ_ASSERT(false, "Unknown track type"); + } + + if (mSrcStream) { + if (aTrack->AsVideoTrack()) { + MOZ_ASSERT(!mSelectedVideoStreamTrack); + MOZ_ASSERT(!mMediaStreamSizeListener); + + mSelectedVideoStreamTrack = aTrack->AsVideoTrack()->GetVideoStreamTrack(); + VideoFrameContainer* container = GetVideoFrameContainer(); + if (mSrcStreamIsPlaying && container) { + mSelectedVideoStreamTrack->AddVideoOutput(container); + } + HTMLVideoElement* self = static_cast(this); + if (self->VideoWidth() <= 1 && self->VideoHeight() <= 1) { + // MediaInfo uses dummy values of 1 for width and height to + // mark video as valid. We need a new stream size listener + // if size is 0x0 or 1x1. + mMediaStreamSizeListener = new StreamSizeListener(this); + mSelectedVideoStreamTrack->AddDirectListener(mMediaStreamSizeListener); + } + } + + if (mReadyState == HAVE_NOTHING) { + // No MediaStreamTracks are captured until we have metadata. + return; + } + for (OutputMediaStream& ms : mOutputStreams) { + if (aTrack->AsVideoTrack() && ms.mCapturingAudioOnly) { + // If the output stream is for audio only we ignore video tracks. + continue; + } + AddCaptureMediaTrackToOutputStream(aTrack, ms); + } } } @@ -1272,14 +1311,14 @@ void HTMLMediaElement::NotifyMediaTrackDisabled(MediaTrack* aTrack) nsString id; aTrack->GetId(id); - LOG(LogLevel::Debug, ("MediaElement %p MediaTrack with id %s disabled", - this, NS_ConvertUTF16toUTF8(id).get())); + LOG(LogLevel::Debug, ("MediaElement %p %sTrack with id %s disabled", + this, aTrack->AsAudioTrack() ? "Audio" : "Video", + NS_ConvertUTF16toUTF8(id).get())); #endif MOZ_ASSERT((!aTrack->AsAudioTrack() || !aTrack->AsAudioTrack()->Enabled()) && (!aTrack->AsVideoTrack() || !aTrack->AsVideoTrack()->Selected())); - if (aTrack->AsAudioTrack()) { bool shouldMute = true; for (uint32_t i = 0; i < AudioTracks()->Length(); ++i) { @@ -1292,10 +1331,54 @@ void HTMLMediaElement::NotifyMediaTrackDisabled(MediaTrack* aTrack) SetMutedInternal(mMuted | MUTED_BY_AUDIO_TRACK); } } else if (aTrack->AsVideoTrack()) { - if (VideoTracks()->SelectedIndex() == -1) { - mDisableVideo = false; + if (mSrcStream) { + MOZ_ASSERT(mSelectedVideoStreamTrack); + if (mSelectedVideoStreamTrack && mMediaStreamSizeListener) { + mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener); + mMediaStreamSizeListener->Forget(); + mMediaStreamSizeListener = nullptr; + } + VideoFrameContainer* container = GetVideoFrameContainer(); + if (mSrcStreamIsPlaying && container) { + mSelectedVideoStreamTrack->RemoveVideoOutput(container); + } + mSelectedVideoStreamTrack = nullptr; } } + + for (OutputMediaStream& ms : mOutputStreams) { + if (ms.mCapturingDecoder) { + MOZ_ASSERT(!ms.mCapturingMediaStream); + continue; + } + MOZ_ASSERT(ms.mCapturingMediaStream); + for (int32_t i = ms.mTrackPorts.Length() - 1; i >= 0; --i) { + if (ms.mTrackPorts[i].first() == aTrack->GetId()) { + // The source of this track just ended. Force-notify that it ended. + // If we bounce it to the MediaStreamGraph it might not be picked up, + // for instance if the MediaInputPort was destroyed in the same + // iteration as it was added. + MediaStreamTrack* outputTrack = ms.mStream->FindOwnedDOMTrack( + ms.mTrackPorts[i].second()->GetDestination(), + ms.mTrackPorts[i].second()->GetDestinationTrackId()); + MOZ_ASSERT(outputTrack); + if (outputTrack) { + NS_DispatchToMainThread( + NewRunnableMethod(outputTrack, &MediaStreamTrack::NotifyEnded)); + } + + ms.mTrackPorts[i].second()->Destroy(); + ms.mTrackPorts.RemoveElementAt(i); + break; + } + } +#ifdef DEBUG + for (auto pair : ms.mTrackPorts) { + MOZ_ASSERT(pair.first() != aTrack->GetId(), + "The same MediaTrack was forwarded to the output stream more than once. This shouldn't happen."); + } +#endif + } } void HTMLMediaElement::NotifyMediaStreamTracksAvailable(DOMMediaStream* aStream) @@ -2128,6 +2211,81 @@ NS_IMETHODIMP HTMLMediaElement::SetMuted(bool aMuted) return NS_OK; } +class HTMLMediaElement::StreamCaptureTrackSource : + public MediaStreamTrackSource, + public MediaStreamTrackSource::Sink +{ +public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(StreamCaptureTrackSource, + MediaStreamTrackSource) + + explicit StreamCaptureTrackSource(MediaStreamTrackSource* aCapturedTrackSource) + : MediaStreamTrackSource(aCapturedTrackSource->GetPrincipal(), + true, + nsString()) + , mCapturedTrackSource(aCapturedTrackSource) + { + mCapturedTrackSource->RegisterSink(this); + } + + void Destroy() override + { + MOZ_ASSERT(mCapturedTrackSource); + if (mCapturedTrackSource) { + mCapturedTrackSource->UnregisterSink(this); + } + } + + MediaSourceEnum GetMediaSource() const override + { + return MediaSourceEnum::Other; + } + + CORSMode GetCORSMode() const override + { + return mCapturedTrackSource->GetCORSMode(); + } + + already_AddRefed + ApplyConstraints(nsPIDOMWindowInner* aWindow, + const dom::MediaTrackConstraints& aConstraints) override + { + RefPtr p = new PledgeVoid(); + p->Reject(new dom::MediaStreamError(aWindow, + NS_LITERAL_STRING("OverconstrainedError"), + NS_LITERAL_STRING(""))); + return p.forget(); + } + + void Stop() override + { + NS_ERROR("We're reporting remote=true to not be stoppable. " + "Stop() should not be called."); + } + + void PrincipalChanged() override + { + mPrincipal = mCapturedTrackSource->GetPrincipal(); + MediaStreamTrackSource::PrincipalChanged(); + } + +private: + virtual ~StreamCaptureTrackSource() {} + + RefPtr mCapturedTrackSource; +}; + +NS_IMPL_ADDREF_INHERITED(HTMLMediaElement::StreamCaptureTrackSource, + MediaStreamTrackSource) +NS_IMPL_RELEASE_INHERITED(HTMLMediaElement::StreamCaptureTrackSource, + MediaStreamTrackSource) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(HTMLMediaElement::StreamCaptureTrackSource) +NS_INTERFACE_MAP_END_INHERITING(MediaStreamTrackSource) +NS_IMPL_CYCLE_COLLECTION_INHERITED(HTMLMediaElement::StreamCaptureTrackSource, + MediaStreamTrackSource, + mCapturedTrackSource) + class HTMLMediaElement::DecoderCaptureTrackSource : public MediaStreamTrackSource, public DecoderPrincipalChangeObserver @@ -2222,6 +2380,11 @@ public: already_AddRefed GetMediaStreamTrackSource(TrackID aInputTrackID) override { + if (mElement && mElement->mSrcStream) { + NS_ERROR("Captured media element playing a stream adds tracks explicitly on main thread."); + return nullptr; + } + // We can return a new source each time here, even for different streams, // since the sources don't keep any internal state and all of them call // through to the same HTMLMediaElement. @@ -2247,8 +2410,115 @@ NS_IMPL_CYCLE_COLLECTION_INHERITED(HTMLMediaElement::CaptureStreamTrackSourceGet MediaStreamTrackSourceGetter, mElement) +void +HTMLMediaElement::SetCapturedOutputStreamsEnabled(bool aEnabled) { + for (OutputMediaStream& ms : mOutputStreams) { + if (ms.mCapturingDecoder) { + MOZ_ASSERT(!ms.mCapturingMediaStream); + continue; + } + for (auto pair : ms.mTrackPorts) { + MediaStream* outputSource = ms.mStream->GetInputStream(); + if (!outputSource) { + NS_ERROR("No output source stream"); + return; + } + + TrackID id = pair.second()->GetDestinationTrackId(); + outputSource->SetTrackEnabled(id, aEnabled); + + LOG(LogLevel::Debug, + ("%s track %d for captured MediaStream %p", + aEnabled ? "Enabled" : "Disabled", id, ms.mStream.get())); + } + } +} + +void +HTMLMediaElement::AddCaptureMediaTrackToOutputStream(MediaTrack* aTrack, + OutputMediaStream& aOutputStream, + bool aAsyncAddtrack) +{ + if (aOutputStream.mCapturingDecoder) { + MOZ_ASSERT(!aOutputStream.mCapturingMediaStream); + return; + } + aOutputStream.mCapturingMediaStream = true; + + MediaStream* outputSource = aOutputStream.mStream->GetInputStream(); + if (!outputSource) { + NS_ERROR("No output source stream"); + return; + } + + ProcessedMediaStream* processedOutputSource = + outputSource->AsProcessedStream(); + if (!processedOutputSource) { + NS_ERROR("Input stream not a ProcessedMediaStream"); + return; + } + + if (!aTrack) { + MOZ_ASSERT(false, "Bad MediaTrack"); + return; + } + + MediaStreamTrack* inputTrack = mSrcStream->GetTrackById(aTrack->GetId()); + MOZ_ASSERT(inputTrack); + if (!inputTrack) { + NS_ERROR("Input track not found in source stream"); + return; + } + +#if DEBUG + for (auto pair : aOutputStream.mTrackPorts) { + MOZ_ASSERT(pair.first() != aTrack->GetId(), + "Captured track already captured to output stream"); + } +#endif + + TrackID destinationTrackID = aOutputStream.mNextAvailableTrackID++; + RefPtr source = + new StreamCaptureTrackSource(&inputTrack->GetSource()); + + MediaSegment::Type type = inputTrack->AsAudioStreamTrack() + ? MediaSegment::AUDIO + : MediaSegment::VIDEO; + + RefPtr track = + aOutputStream.mStream->CreateDOMTrack(destinationTrackID, type, source); + + if (aAsyncAddtrack) { + NS_DispatchToMainThread( + NewRunnableMethod>( + aOutputStream.mStream, &DOMMediaStream::AddTrackInternal, track)); + } else { + aOutputStream.mStream->AddTrackInternal(track); + } + + // Track is muted initially, so we don't leak data if it's added while paused + // and an MSG iteration passes before the mute comes into effect. + processedOutputSource->SetTrackEnabled(destinationTrackID, false); + RefPtr port = + inputTrack->ForwardTrackContentsTo(processedOutputSource, + destinationTrackID); + + Pair> p(aTrack->GetId(), port); + aOutputStream.mTrackPorts.AppendElement(Move(p)); + + if (mSrcStreamIsPlaying) { + processedOutputSource->SetTrackEnabled(destinationTrackID, true); + } + + LOG(LogLevel::Debug, + ("Created %s track %p with id %d from track %p through MediaInputPort %p", + inputTrack->AsAudioStreamTrack() ? "audio" : "video", + track.get(), destinationTrackID, inputTrack, port.get())); +} + already_AddRefed HTMLMediaElement::CaptureStreamInternal(bool aFinishWhenEnded, + bool aCaptureAudio, MediaStreamGraph* aGraph) { nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow(); @@ -2277,30 +2547,91 @@ HTMLMediaElement::CaptureStreamInternal(bool aFinishWhenEnded, MediaStreamTrackSourceGetter* getter = new CaptureStreamTrackSourceGetter(this); out->mStream = DOMMediaStream::CreateTrackUnionStreamAsInput(window, aGraph, getter); out->mFinishWhenEnded = aFinishWhenEnded; + out->mCapturingAudioOnly = aCaptureAudio; + + if (aCaptureAudio) { + if (mSrcStream) { + // We don't support applying volume and mute to the captured stream, when + // capturing a MediaStream. + nsContentUtils::ReportToConsole(nsIScriptError::errorFlag, + NS_LITERAL_CSTRING("Media"), + OwnerDoc(), + nsContentUtils::eDOM_PROPERTIES, + "MediaElementAudioCaptureOfMediaStreamError"); + return nullptr; + } + + // mAudioCaptured tells the user that the audio played by this media element + // is being routed to the captureStreams *instead* of being played to + // speakers. + mAudioCaptured = true; + } + + if (mReadyState == HAVE_NOTHING) { + // Do not expose the tracks directly before we have metadata. + RefPtr result = out->mStream; + return result.forget(); + } - mAudioCaptured = true; if (mDecoder) { + out->mCapturingDecoder = true; mDecoder->AddOutputStream(out->mStream->GetInputStream()->AsProcessedStream(), aFinishWhenEnded); - if (mReadyState >= HAVE_METADATA) { - // Expose the tracks to JS directly. - if (HasAudio()) { - TrackID audioTrackId = mMediaInfo.mAudio.mTrackId; - RefPtr trackSource = - getter->GetMediaStreamTrackSource(audioTrackId); - RefPtr track = - out->mStream->CreateDOMTrack(audioTrackId, MediaSegment::AUDIO, - trackSource); - out->mStream->AddTrackInternal(track); + if (HasAudio()) { + TrackID audioTrackId = mMediaInfo.mAudio.mTrackId; + RefPtr trackSource = + getter->GetMediaStreamTrackSource(audioTrackId); + RefPtr track = + out->mStream->CreateDOMTrack(audioTrackId, MediaSegment::AUDIO, + trackSource); + out->mStream->AddTrackInternal(track); + LOG(LogLevel::Debug, + ("Created audio track %d for captured decoder", audioTrackId)); + } + if (IsVideo() && HasVideo() && !out->mCapturingAudioOnly) { + TrackID videoTrackId = mMediaInfo.mVideo.mTrackId; + RefPtr trackSource = + getter->GetMediaStreamTrackSource(videoTrackId); + RefPtr track = + out->mStream->CreateDOMTrack(videoTrackId, MediaSegment::VIDEO, + trackSource); + out->mStream->AddTrackInternal(track); + LOG(LogLevel::Debug, + ("Created video track %d for captured decoder", videoTrackId)); + } + } + + if (mSrcStream) { + out->mCapturingMediaStream = true; + MediaStream* inputStream = out->mStream->GetInputStream(); + if (!inputStream) { + NS_ERROR("No input stream"); + RefPtr result = out->mStream; + return result.forget(); + } + + ProcessedMediaStream* processedInputStream = + inputStream->AsProcessedStream(); + if (!processedInputStream) { + NS_ERROR("Input stream not a ProcessedMediaStream"); + RefPtr result = out->mStream; + return result.forget(); + } + + for (size_t i = 0; i < AudioTracks()->Length(); ++i) { + AudioTrack* t = (*AudioTracks())[i]; + if (t->Enabled()) { + AddCaptureMediaTrackToOutputStream(t, *out, false); } - if (HasVideo()) { - TrackID videoTrackId = mMediaInfo.mVideo.mTrackId; - RefPtr trackSource = - getter->GetMediaStreamTrackSource(videoTrackId); - RefPtr track = - out->mStream->CreateDOMTrack(videoTrackId, MediaSegment::VIDEO, - trackSource); - out->mStream->AddTrackInternal(track); + } + if (IsVideo() && !out->mCapturingAudioOnly) { + // Only add video tracks if we're a video element and the output stream + // wants video. + for (size_t i = 0; i < VideoTracks()->Length(); ++i) { + VideoTrack* t = (*VideoTracks())[i]; + if (t->Selected()) { + AddCaptureMediaTrackToOutputStream(t, *out, false); + } } } } @@ -2308,6 +2639,19 @@ HTMLMediaElement::CaptureStreamInternal(bool aFinishWhenEnded, return result.forget(); } +already_AddRefed +HTMLMediaElement::CaptureAudio(ErrorResult& aRv, + MediaStreamGraph* aGraph) +{ + RefPtr stream = CaptureStreamInternal(false, aGraph); + if (!stream) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + return stream.forget(); +} + already_AddRefed HTMLMediaElement::MozCaptureStream(ErrorResult& aRv, MediaStreamGraph* aGraph) @@ -2845,6 +3189,20 @@ HTMLMediaElement::WakeLockRelease() } } +HTMLMediaElement::OutputMediaStream::OutputMediaStream() + : mFinishWhenEnded(false) + , mCapturingAudioOnly(false) + , mCapturingDecoder(false) + , mCapturingMediaStream(false) + , mNextAvailableTrackID(1) {} + +HTMLMediaElement::OutputMediaStream::~OutputMediaStream() +{ + for (auto pair : mTrackPorts) { + pair.second()->Destroy(); + } +} + bool HTMLMediaElement::ParseAttribute(int32_t aNamespaceID, nsIAtom* aAttribute, const nsAString& aValue, @@ -3462,10 +3820,15 @@ nsresult HTMLMediaElement::FinishDecoderSetup(MediaDecoder* aDecoder, return rv; } - for (uint32_t i = 0; i < mOutputStreams.Length(); ++i) { - OutputMediaStream* ms = &mOutputStreams[i]; - aDecoder->AddOutputStream(ms->mStream->GetInputStream()->AsProcessedStream(), - ms->mFinishWhenEnded); + for (OutputMediaStream& ms : mOutputStreams) { + if (ms.mCapturingMediaStream) { + MOZ_ASSERT(!ms.mCapturingDecoder); + continue; + } + + ms.mCapturingDecoder = true; + aDecoder->AddOutputStream(ms.mStream->GetInputStream()->AsProcessedStream(), + ms.mFinishWhenEnded); } #ifdef MOZ_EME @@ -3719,13 +4082,8 @@ void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags) if (mSelectedVideoStreamTrack && container) { mSelectedVideoStreamTrack->AddVideoOutput(container); } - VideoTrack* videoTrack = VideoTracks()->GetSelectedTrack(); - if (videoTrack) { - VideoStreamTrack* videoStreamTrack = videoTrack->GetVideoStreamTrack(); - if (videoStreamTrack && container) { - videoStreamTrack->AddVideoOutput(container); - } - } + + SetCapturedOutputStreamsEnabled(true); // Unmute } else { if (stream) { mSrcStreamPausedCurrentTime = CurrentTime(); @@ -3737,13 +4095,8 @@ void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags) if (mSelectedVideoStreamTrack && container) { mSelectedVideoStreamTrack->RemoveVideoOutput(container); } - VideoTrack* videoTrack = VideoTracks()->GetSelectedTrack(); - if (videoTrack) { - VideoStreamTrack* videoStreamTrack = videoTrack->GetVideoStreamTrack(); - if (videoStreamTrack && container) { - videoStreamTrack->RemoveVideoOutput(container); - } - } + + SetCapturedOutputStreamsEnabled(false); // Mute } // If stream is null, then DOMMediaStream::Destroy must have been // called and that will remove all listeners/outputs. @@ -3782,7 +4135,11 @@ void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream) // If we pause this media element, track changes in the underlying stream // will continue to fire events at this element and alter its track list. // That's simpler than delaying the events, but probably confusing... - ConstructMediaTracks(); + nsTArray> tracks; + mSrcStream->GetTracks(tracks); + for (const RefPtr& track : tracks) { + NotifyMediaStreamTrackAdded(track); + } mSrcStream->OnTracksAvailable(new MediaStreamTracksAvailableCallback(this)); mMediaStreamTrackListener = new MediaStreamTrackListener(this); @@ -3805,11 +4162,14 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback() UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM); if (mMediaStreamSizeListener) { - mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener); - mSelectedVideoStreamTrack = nullptr; + MOZ_ASSERT(mSelectedVideoStreamTrack); + if (mSelectedVideoStreamTrack) { + mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener); + } mMediaStreamSizeListener->Forget(); - mMediaStreamSizeListener = nullptr; } + mSelectedVideoStreamTrack = nullptr; + mMediaStreamSizeListener = nullptr; mSrcStream->UnregisterTrackListener(mMediaStreamTrackListener); mMediaStreamTrackListener = nullptr; @@ -3817,6 +4177,13 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback() mSrcStream->RemovePrincipalChangeObserver(this); mSrcStreamVideoPrincipal = nullptr; + for (OutputMediaStream& ms : mOutputStreams) { + for (auto pair : ms.mTrackPorts) { + pair.second()->Destroy(); + } + ms.mTrackPorts.Clear(); + } + mSrcStream = nullptr; } @@ -3829,8 +4196,7 @@ CreateAudioTrack(AudioStreamTrack* aStreamTrack) aStreamTrack->GetLabel(label); return MediaTrackList::CreateAudioTrack(id, NS_LITERAL_STRING("main"), - label, EmptyString(), - aStreamTrack->Enabled()); + label, EmptyString(), true); } static already_AddRefed @@ -3846,57 +4212,22 @@ CreateVideoTrack(VideoStreamTrack* aStreamTrack) aStreamTrack); } -void HTMLMediaElement::ConstructMediaTracks() -{ - nsTArray> tracks; - mSrcStream->GetTracks(tracks); - - int firstEnabledVideo = -1; - for (const RefPtr& track : tracks) { - if (track->Ended()) { - continue; - } - - if (AudioStreamTrack* t = track->AsAudioStreamTrack()) { - RefPtr audioTrack = CreateAudioTrack(t); - AudioTracks()->AddTrack(audioTrack); - } else if (VideoStreamTrack* t = track->AsVideoStreamTrack()) { - RefPtr videoTrack = CreateVideoTrack(t); - VideoTracks()->AddTrack(videoTrack); - firstEnabledVideo = (t->Enabled() && firstEnabledVideo < 0) - ? (VideoTracks()->Length() - 1) - : firstEnabledVideo; - } - } - - if (VideoTracks()->Length() > 0) { - // If media resource does not indicate a particular set of video tracks to - // enable, the one that is listed first in the element's videoTracks object - // must be selected. - int index = firstEnabledVideo >= 0 ? firstEnabledVideo : 0; - (*VideoTracks())[index]->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS); - VideoTrack* track = (*VideoTracks())[index]; - VideoStreamTrack* streamTrack = track->GetVideoStreamTrack(); - mMediaStreamSizeListener = new StreamSizeListener(this); - streamTrack->AddDirectListener(mMediaStreamSizeListener); - mSelectedVideoStreamTrack = streamTrack; - if (GetVideoFrameContainer()) { - mSelectedVideoStreamTrack->AddVideoOutput(GetVideoFrameContainer()); - } - } -} - void HTMLMediaElement::NotifyMediaStreamTrackAdded(const RefPtr& aTrack) { MOZ_ASSERT(aTrack); + if (aTrack->Ended()) { + return; + } + #ifdef DEBUG nsString id; aTrack->GetId(id); - LOG(LogLevel::Debug, ("%p, Adding MediaTrack with id %s", - this, NS_ConvertUTF16toUTF8(id).get())); + LOG(LogLevel::Debug, ("%p, Adding %sTrack with id %s", + this, aTrack->AsAudioStreamTrack() ? "Audio" : "Video", + NS_ConvertUTF16toUTF8(id).get())); #endif if (AudioStreamTrack* t = aTrack->AsAudioStreamTrack()) { @@ -3904,23 +4235,17 @@ HTMLMediaElement::NotifyMediaStreamTrackAdded(const RefPtr& aT AudioTracks()->AddTrack(audioTrack); } else if (VideoStreamTrack* t = aTrack->AsVideoStreamTrack()) { // TODO: Fix this per the spec on bug 1273443. - int32_t selectedIndex = VideoTracks()->SelectedIndex(); + if (!IsVideo()) { + return; + } RefPtr videoTrack = CreateVideoTrack(t); VideoTracks()->AddTrack(videoTrack); // New MediaStreamTrack added, set the new added video track as selected // video track when there is no selected track. - if (selectedIndex == -1) { + if (VideoTracks()->SelectedIndex() == -1) { MOZ_ASSERT(!mSelectedVideoStreamTrack); videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS); - mMediaStreamSizeListener = new StreamSizeListener(this); - t->AddDirectListener(mMediaStreamSizeListener); - mSelectedVideoStreamTrack = t; - VideoFrameContainer* container = GetVideoFrameContainer(); - if (mSrcStreamIsPlaying && container) { - mSelectedVideoStreamTrack->AddVideoOutput(container); - } } - } } @@ -3932,62 +4257,14 @@ HTMLMediaElement::NotifyMediaStreamTrackRemoved(const RefPtr& nsAutoString id; aTrack->GetId(id); - LOG(LogLevel::Debug, ("%p, Removing MediaTrack with id %s", - this, NS_ConvertUTF16toUTF8(id).get())); + LOG(LogLevel::Debug, ("%p, Removing %sTrack with id %s", + this, aTrack->AsAudioStreamTrack() ? "Audio" : "Video", + NS_ConvertUTF16toUTF8(id).get())); if (MediaTrack* t = AudioTracks()->GetTrackById(id)) { AudioTracks()->RemoveTrack(t); } else if (MediaTrack* t = VideoTracks()->GetTrackById(id)) { VideoTracks()->RemoveTrack(t); - // TODO: Fix this per the spec on bug 1273443. - // If the removed media stream track is selected video track and there are - // still video tracks, change the selected video track to the first - // remaining track. - if (aTrack == mSelectedVideoStreamTrack) { - // The mMediaStreamSizeListener might already reset to nullptr. - if (mMediaStreamSizeListener) { - mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener); - } - VideoFrameContainer* container = GetVideoFrameContainer(); - if (mSrcStreamIsPlaying && container) { - mSelectedVideoStreamTrack->RemoveVideoOutput(container); - } - mSelectedVideoStreamTrack = nullptr; - MOZ_ASSERT(mSrcStream); - nsTArray> tracks; - mSrcStream->GetVideoTracks(tracks); - - for (const RefPtr& track : tracks) { - if (track->Ended()) { - continue; - } - if (!track->Enabled()) { - continue; - } - - nsAutoString trackId; - track->GetId(trackId); - MediaTrack* videoTrack = VideoTracks()->GetTrackById(trackId); - MOZ_ASSERT(videoTrack); - - videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS); - if (mMediaStreamSizeListener) { - track->AddDirectListener(mMediaStreamSizeListener); - } - mSelectedVideoStreamTrack = track; - if (container) { - mSelectedVideoStreamTrack->AddVideoOutput(container); - } - return; - } - - // There is no enabled video track existing, clean the - // mMediaStreamSizeListener. - if (mMediaStreamSizeListener) { - mMediaStreamSizeListener->Forget(); - mMediaStreamSizeListener = nullptr; - } - } } else { // XXX (bug 1208328) Uncomment this when DOMMediaStream doesn't call // NotifyTrackRemoved multiple times for the same track, i.e., when it @@ -4071,6 +4348,28 @@ void HTMLMediaElement::MetadataLoaded(const MediaInfo* aInfo, SetCurrentTime(mDefaultPlaybackStartPosition); mDefaultPlaybackStartPosition = 0.0; } + + if (!mSrcStream) { + return; + } + for (OutputMediaStream& ms : mOutputStreams) { + for (size_t i = 0; i < AudioTracks()->Length(); ++i) { + AudioTrack* t = (*AudioTracks())[i]; + if (t->Enabled()) { + AddCaptureMediaTrackToOutputStream(t, ms); + } + } + if (IsVideo() && !ms.mCapturingAudioOnly) { + // Only add video tracks if we're a video element and the output stream + // wants video. + for (size_t i = 0; i < VideoTracks()->Length(); ++i) { + VideoTrack* t = (*VideoTracks())[i]; + if (t->Selected()) { + AddCaptureMediaTrackToOutputStream(t, ms); + } + } + } + } } void HTMLMediaElement::FirstFrameLoaded() @@ -4175,6 +4474,8 @@ void HTMLMediaElement::PlaybackEnded() // Discard all output streams that have finished now. for (int32_t i = mOutputStreams.Length() - 1; i >= 0; --i) { if (mOutputStreams[i].mFinishWhenEnded) { + LOG(LogLevel::Debug, ("Playback ended. Removing output stream %p", + mOutputStreams[i].mStream.get())); mOutputStreams.RemoveElementAt(i); } } @@ -4913,6 +5214,12 @@ void HTMLMediaElement::UpdateInitialMediaSize(const nsIntSize& aSize) if (!mMediaStreamSizeListener) { return; } + + if (!mSelectedVideoStreamTrack) { + MOZ_ASSERT(false); + return; + } + mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener); mMediaStreamSizeListener->Forget(); mMediaStreamSizeListener = nullptr; diff --git a/dom/html/HTMLMediaElement.h b/dom/html/HTMLMediaElement.h index 6a1d0ef6d51a..b433dc7ccc74 100644 --- a/dom/html/HTMLMediaElement.h +++ b/dom/html/HTMLMediaElement.h @@ -316,6 +316,7 @@ public: */ bool RemoveDecoderPrincipalChangeObserver(DecoderPrincipalChangeObserver* aObserver); + class StreamCaptureTrackSource; class DecoderCaptureTrackSource; class CaptureStreamTrackSourceGetter; @@ -658,6 +659,9 @@ public: return mAutoplayEnabled; } + already_AddRefed CaptureAudio(ErrorResult& aRv, + MediaStreamGraph* aGraph = nullptr); + already_AddRefed MozCaptureStream(ErrorResult& aRv, MediaStreamGraph* aGraph = nullptr); @@ -784,6 +788,23 @@ protected: nsCOMPtr mTimer; }; + // Holds references to the DOM wrappers for the MediaStreams that we're + // writing to. + struct OutputMediaStream { + OutputMediaStream(); + ~OutputMediaStream(); + + RefPtr mStream; + bool mFinishWhenEnded; + bool mCapturingAudioOnly; + bool mCapturingDecoder; + bool mCapturingMediaStream; + + // The following members are keeping state for a captured MediaStream. + TrackID mNextAvailableTrackID; + nsTArray>> mTrackPorts; + }; + nsresult PlayInternal(bool aCallerIsChrome); /** Use this method to change the mReadyState member, so required @@ -836,13 +857,6 @@ protected: enum { REMOVING_SRC_STREAM = 0x1 }; void UpdateSrcMediaStreamPlaying(uint32_t aFlags = 0); - /** - * If loading and playing a MediaStream, for each MediaStreamTrack in the - * MediaStream, create a corresponding AudioTrack or VideoTrack during the - * phase of resource fetching. - */ - void ConstructMediaTracks(); - /** * Called by our DOMMediaStream::TrackListener when a new MediaStreamTrack has * been added to the playback stream of |mSrcStream|. @@ -856,13 +870,36 @@ protected: void NotifyMediaStreamTrackRemoved(const RefPtr& aTrack); /** - * Returns an nsDOMMediaStream containing the played contents of this + * Enables or disables all tracks forwarded from mSrcStream to all + * OutputMediaStreams. We do this for muting the tracks when pausing, + * and unmuting when playing the media element again. + * + * If mSrcStream is unset, this does nothing. + */ + void SetCapturedOutputStreamsEnabled(bool aEnabled); + + /** + * Create a new MediaStreamTrack for aTrack and add it to the DOMMediaStream + * in aOutputStream. This automatically sets the output track to enabled or + * disabled depending on our current playing state. + */ + void AddCaptureMediaTrackToOutputStream(MediaTrack* aTrack, + OutputMediaStream& aOutputStream, + bool aAsyncAddtrack = true); + + /** + * Returns an DOMMediaStream containing the played contents of this * element. When aFinishWhenEnded is true, when this element ends playback * we will finish the stream and not play any more into it. * When aFinishWhenEnded is false, ending playback does not finish the stream. * The stream will never finish. + * + * When aCaptureAudio is true, we stop playout of audio and instead route it + * to the DOMMediaStream. Volume and mute state will be applied to the audio + * reaching the stream. No video tracks will be captured in this case. */ already_AddRefed CaptureStreamInternal(bool aFinishWhenEnded, + bool aCaptureAudio, MediaStreamGraph* aGraph = nullptr); /** @@ -1259,10 +1296,6 @@ protected: // Holds references to the DOM wrappers for the MediaStreams that we're // writing to. - struct OutputMediaStream { - RefPtr mStream; - bool mFinishWhenEnded; - }; nsTArray mOutputStreams; // Holds a reference to the MediaStreamListener attached to mSrcStream's diff --git a/dom/media/DOMMediaStream.cpp b/dom/media/DOMMediaStream.cpp index 4d54ba90833b..ffcb4c2f2610 100644 --- a/dom/media/DOMMediaStream.cpp +++ b/dom/media/DOMMediaStream.cpp @@ -144,28 +144,35 @@ public: MediaStreamTrack* track = mStream->FindOwnedDOMTrack(aInputStream, aInputTrackID, aTrackID); - if (!track) { - // Track had not been created on main thread before, create it now. - NS_WARN_IF_FALSE(!mStream->mTracks.IsEmpty(), - "A new track was detected on the input stream; creating " - "a corresponding MediaStreamTrack. Initial tracks " - "should be added manually to immediately and " - "synchronously be available to JS."); - RefPtr source; - if (mStream->mTrackSourceGetter) { - source = mStream->mTrackSourceGetter->GetMediaStreamTrackSource(aTrackID); - } - if (!source) { - NS_ASSERTION(false, "Dynamic track created without an explicit TrackSource"); - nsPIDOMWindowInner* window = mStream->GetParentObject(); - nsIDocument* doc = window ? window->GetExtantDoc() : nullptr; - nsIPrincipal* principal = doc ? doc->NodePrincipal() : nullptr; - source = new BasicUnstoppableTrackSource(principal); - } - RefPtr newTrack = - mStream->CreateDOMTrack(aTrackID, aType, source); - mStream->AddTrackInternal(newTrack); + + if (track) { + LOG(LogLevel::Debug, ("DOMMediaStream %p Track %d from owned stream %p " + "bound to MediaStreamTrack %p.", + mStream, aTrackID, aInputStream, track)); + return; } + + // Track had not been created on main thread before, create it now. + NS_WARN_IF_FALSE(!mStream->mTracks.IsEmpty(), + "A new track was detected on the input stream; creating " + "a corresponding MediaStreamTrack. Initial tracks " + "should be added manually to immediately and " + "synchronously be available to JS."); + RefPtr source; + if (mStream->mTrackSourceGetter) { + source = mStream->mTrackSourceGetter->GetMediaStreamTrackSource(aTrackID); + } + if (!source) { + NS_ASSERTION(false, "Dynamic track created without an explicit TrackSource"); + nsPIDOMWindowInner* window = mStream->GetParentObject(); + nsIDocument* doc = window ? window->GetExtantDoc() : nullptr; + nsIPrincipal* principal = doc ? doc->NodePrincipal() : nullptr; + source = new BasicUnstoppableTrackSource(principal); + } + + RefPtr newTrack = + mStream->CreateDOMTrack(aTrackID, aType, source); + mStream->AddTrackInternal(newTrack); } void DoNotifyTrackEnded(MediaStream* aInputStream, TrackID aInputTrackID, @@ -1321,6 +1328,7 @@ DOMLocalMediaStream::WrapObject(JSContext* aCx, JS::Handle aGivenProt void DOMLocalMediaStream::Stop() { + LOG(LogLevel::Debug, ("DOMMediaStream %p Stop()", this)); nsCOMPtr pWindow = GetParentObject(); nsIDocument* document = pWindow ? pWindow->GetExtantDoc() : nullptr; nsContentUtils::ReportToConsole(nsIScriptError::warningFlag, diff --git a/dom/media/webaudio/AudioContext.cpp b/dom/media/webaudio/AudioContext.cpp index 20693ebead09..579cb9579a79 100644 --- a/dom/media/webaudio/AudioContext.cpp +++ b/dom/media/webaudio/AudioContext.cpp @@ -366,8 +366,8 @@ AudioContext::CreateMediaElementSource(HTMLMediaElement& aMediaElement, return nullptr; } - RefPtr stream = aMediaElement.MozCaptureStream(aRv, - mDestination->Stream()->Graph()); + RefPtr stream = + aMediaElement.CaptureAudio(aRv, mDestination->Stream()->Graph()); if (aRv.Failed()) { return nullptr; } diff --git a/dom/media/webaudio/MediaStreamAudioSourceNode.cpp b/dom/media/webaudio/MediaStreamAudioSourceNode.cpp index 5b1f4186281f..beedd5300476 100644 --- a/dom/media/webaudio/MediaStreamAudioSourceNode.cpp +++ b/dom/media/webaudio/MediaStreamAudioSourceNode.cpp @@ -60,7 +60,11 @@ MediaStreamAudioSourceNode::Create(AudioContext* aContext, void MediaStreamAudioSourceNode::Init(DOMMediaStream* aMediaStream, ErrorResult& aRv) { - MOZ_ASSERT(aMediaStream); + if (!aMediaStream) { + aRv.Throw(NS_ERROR_FAILURE); + return; + } + MediaStream* inputStream = aMediaStream->GetPlaybackStream(); MediaStreamGraph* graph = Context()->Graph(); if (NS_WARN_IF(graph != inputStream->Graph())) {