From 6ba30843e83724039db130275459d7b211fd69be Mon Sep 17 00:00:00 2001 From: Csoregi Natalia Date: Thu, 14 Nov 2019 00:32:51 +0200 Subject: [PATCH] Backed out 15 changesets (bug 1500049, bug 1172394, bug 1546756, bug 1302379) for failures on browser_disabledForMediaStreamVideos.js. CLOSED TREE Backed out changeset 355f090421a6 (bug 1500049) Backed out changeset 306341d0b586 (bug 1302379) Backed out changeset 3ff0d72d23a2 (bug 1546756) Backed out changeset a4f256e68cef (bug 1172394) Backed out changeset d0aa43657e8c (bug 1172394) Backed out changeset edff95b6f724 (bug 1172394) Backed out changeset 94bd21d9b396 (bug 1172394) Backed out changeset 7e7baa73e1ef (bug 1172394) Backed out changeset c3bd415507e8 (bug 1172394) Backed out changeset 1c45b135318d (bug 1172394) Backed out changeset c57c41e8c39e (bug 1172394) Backed out changeset a796541fe5ef (bug 1172394) Backed out changeset 89ad0b553b0f (bug 1172394) Backed out changeset 744fb77a5833 (bug 1172394) Backed out changeset afb4b226ff04 (bug 1172394) --- dom/html/HTMLMediaElement.cpp | 737 ++++++++---------- dom/html/HTMLMediaElement.h | 122 ++- dom/media/ChannelMediaDecoder.cpp | 20 +- dom/media/ChannelMediaDecoder.h | 5 - dom/media/ChannelMediaResource.cpp | 6 +- dom/media/ChannelMediaResource.h | 2 +- dom/media/CloneableWithRangeMediaResource.cpp | 4 +- dom/media/CloneableWithRangeMediaResource.h | 2 +- dom/media/DOMMediaStream.cpp | 19 +- dom/media/DOMMediaStream.h | 11 +- dom/media/FileMediaResource.cpp | 4 +- dom/media/FileMediaResource.h | 2 +- dom/media/MediaCache.cpp | 21 +- dom/media/MediaCache.h | 6 +- dom/media/MediaDecoder.cpp | 70 +- dom/media/MediaDecoder.h | 62 +- dom/media/MediaDecoderOwner.h | 8 + dom/media/MediaDecoderStateMachine.cpp | 236 ++++-- dom/media/MediaDecoderStateMachine.h | 68 +- dom/media/MediaResource.h | 7 +- dom/media/MediaStreamTrack.h | 2 +- dom/media/mediasink/AudioSink.cpp | 7 +- dom/media/mediasink/AudioSink.h | 21 +- dom/media/mediasink/AudioSinkWrapper.cpp | 20 +- dom/media/mediasink/AudioSinkWrapper.h | 11 +- dom/media/mediasink/DecodedStream.cpp | 312 ++++---- dom/media/mediasink/DecodedStream.h | 30 +- dom/media/mediasink/MediaSink.h | 26 +- dom/media/mediasink/OutputStreamManager.cpp | 357 +++++++++ dom/media/mediasink/OutputStreamManager.h | 161 ++++ dom/media/mediasink/VideoSink.cpp | 35 +- dom/media/mediasink/VideoSink.h | 6 +- dom/media/mediasink/moz.build | 1 + .../mediasource/SourceBufferResource.cpp | 4 +- dom/media/mediasource/SourceBufferResource.h | 2 +- ...st_mediatrack_consuming_mediaresource.html | 57 +- .../test/test_mediatrack_replay_from_end.html | 62 +- .../test_streams_element_capture_reset.html | 189 +++-- .../webaudio/MediaElementAudioSourceNode.cpp | 1 - .../webaudio/MediaStreamAudioSourceNode.cpp | 8 +- .../webaudio/MediaStreamAudioSourceNode.h | 2 +- ...tAudioSourceToScriptProcessorTest.html.ini | 8 + 42 files changed, 1586 insertions(+), 1148 deletions(-) create mode 100644 dom/media/mediasink/OutputStreamManager.cpp create mode 100644 dom/media/mediasink/OutputStreamManager.h create mode 100644 testing/web-platform/meta/webaudio/the-audio-api/the-mediaelementaudiosourcenode-interface/mediaElementAudioSourceToScriptProcessorTest.html.ini diff --git a/dom/html/HTMLMediaElement.cpp b/dom/html/HTMLMediaElement.cpp index 14a3ba02c36b..b6df508764aa 100644 --- a/dom/html/HTMLMediaElement.cpp +++ b/dom/html/HTMLMediaElement.cpp @@ -658,30 +658,23 @@ class HTMLMediaElement::MediaStreamRenderer WeakPtr mVideoTrack; }; -class HTMLMediaElement::MediaElementTrackSource +class HTMLMediaElement::StreamCaptureTrackSource : public MediaStreamTrackSource, public MediaStreamTrackSource::Sink { public: NS_DECL_ISUPPORTS_INHERITED - NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaElementTrackSource, + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(StreamCaptureTrackSource, MediaStreamTrackSource) - /* MediaDecoder track source */ - MediaElementTrackSource(ProcessedMediaTrack* aTrack, nsIPrincipal* aPrincipal) - : MediaStreamTrackSource(aPrincipal, nsString()), mTrack(aTrack) { - MOZ_ASSERT(mTrack); - } - - /* MediaStream track source */ - MediaElementTrackSource(MediaStreamTrackSource* aCapturedTrackSource, - ProcessedMediaTrack* aTrack, MediaInputPort* aPort) + StreamCaptureTrackSource(MediaStreamTrackSource* aCapturedTrackSource, + ProcessedMediaTrack* aStream, MediaInputPort* aPort) : MediaStreamTrackSource(aCapturedTrackSource->GetPrincipal(), nsString()), mCapturedTrackSource(aCapturedTrackSource), - mTrack(aTrack), + mTrack(aStream), mPort(aPort) { - MOZ_ASSERT(mTrack); MOZ_ASSERT(mCapturedTrackSource); + MOZ_ASSERT(mTrack); MOZ_ASSERT(mPort); mCapturedTrackSource->RegisterSink(this); @@ -695,18 +688,14 @@ class HTMLMediaElement::MediaElementTrackSource : DisabledTrackMode::SILENCE_FREEZE); } - void SetPrincipal(RefPtr aPrincipal) { - mPrincipal = std::move(aPrincipal); - MediaStreamTrackSource::PrincipalChanged(); - } - void Destroy() override { if (mCapturedTrackSource) { mCapturedTrackSource->UnregisterSink(this); mCapturedTrackSource = nullptr; } - if (mTrack && !mTrack->IsDestroyed()) { + if (mTrack) { mTrack->Destroy(); + mTrack = nullptr; } if (mPort) { mPort->Destroy(); @@ -718,11 +707,7 @@ class HTMLMediaElement::MediaElementTrackSource return MediaSourceEnum::Other; } - void Stop() override { - // Do nothing. There may appear new output streams - // that need tracks sourced from this source, so we - // cannot destroy things yet. - } + void Stop() override { Destroy(); } /** * Do not keep the track source alive. The source lifetime is controlled by @@ -745,69 +730,50 @@ class HTMLMediaElement::MediaElementTrackSource return; } - SetPrincipal(mCapturedTrackSource->GetPrincipal()); + mPrincipal = mCapturedTrackSource->GetPrincipal(); + MediaStreamTrackSource::PrincipalChanged(); } void MutedChanged(bool aNewState) override { + if (!mCapturedTrackSource) { + // This could happen during shutdown. + return; + } + MediaStreamTrackSource::MutedChanged(aNewState); } void OverrideEnded() override { + if (!mCapturedTrackSource) { + // This could happen during shutdown. + return; + } + Destroy(); MediaStreamTrackSource::OverrideEnded(); } - ProcessedMediaTrack* Track() const { return mTrack; } - private: - virtual ~MediaElementTrackSource() { Destroy(); }; + virtual ~StreamCaptureTrackSource() { + MOZ_ASSERT(!mCapturedTrackSource); + MOZ_ASSERT(!mTrack); + MOZ_ASSERT(!mPort); + }; RefPtr mCapturedTrackSource; - const RefPtr mTrack; + RefPtr mTrack; RefPtr mPort; }; -HTMLMediaElement::OutputMediaStream::OutputMediaStream( - RefPtr aStream, bool aCapturingAudioOnly, - bool aFinishWhenEnded) - : mStream(std::move(aStream)), - mCapturingAudioOnly(aCapturingAudioOnly), - mFinishWhenEnded(aFinishWhenEnded) {} -HTMLMediaElement::OutputMediaStream::~OutputMediaStream() = default; - -void ImplCycleCollectionTraverse(nsCycleCollectionTraversalCallback& aCallback, - HTMLMediaElement::OutputMediaStream& aField, - const char* aName, uint32_t aFlags) { - ImplCycleCollectionTraverse(aCallback, aField.mStream, "mStream", aFlags); - ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedLoadingSrc, - "mFinishWhenEndedLoadingSrc", aFlags); - ImplCycleCollectionTraverse(aCallback, aField.mFinishWhenEndedAttrStream, - "mFinishWhenEndedAttrStream", aFlags); -} - -void ImplCycleCollectionUnlink(HTMLMediaElement::OutputMediaStream& aField) { - ImplCycleCollectionUnlink(aField.mStream); - ImplCycleCollectionUnlink(aField.mFinishWhenEndedLoadingSrc); - ImplCycleCollectionUnlink(aField.mFinishWhenEndedAttrStream); -} - -NS_IMPL_ADDREF_INHERITED(HTMLMediaElement::MediaElementTrackSource, +NS_IMPL_ADDREF_INHERITED(HTMLMediaElement::StreamCaptureTrackSource, MediaStreamTrackSource) -NS_IMPL_RELEASE_INHERITED(HTMLMediaElement::MediaElementTrackSource, +NS_IMPL_RELEASE_INHERITED(HTMLMediaElement::StreamCaptureTrackSource, MediaStreamTrackSource) NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION( - HTMLMediaElement::MediaElementTrackSource) + HTMLMediaElement::StreamCaptureTrackSource) NS_INTERFACE_MAP_END_INHERITING(MediaStreamTrackSource) -NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLMediaElement::MediaElementTrackSource) -NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED( - HTMLMediaElement::MediaElementTrackSource, MediaStreamTrackSource) - tmp->Destroy(); - NS_IMPL_CYCLE_COLLECTION_UNLINK(mCapturedTrackSource) -NS_IMPL_CYCLE_COLLECTION_UNLINK_END -NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED( - HTMLMediaElement::MediaElementTrackSource, MediaStreamTrackSource) - NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCapturedTrackSource) -NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END +NS_IMPL_CYCLE_COLLECTION_INHERITED(HTMLMediaElement::StreamCaptureTrackSource, + MediaStreamTrackSource, mCapturedTrackSource) /** * There is a reference cycle involving this class: MediaLoadListener @@ -1676,8 +1642,9 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement, NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSourceLoadCandidate) NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioChannelWrapper) NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mErrorSink->mError) - NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputStreams) - NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputTrackSources); + for (uint32_t i = 0; i < tmp->mOutputStreams.Length(); ++i) { + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutputStreams[i].mStream) + } NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPlayed); NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTextTrackManager) NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioTrackList) @@ -1709,8 +1676,10 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement, } NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioChannelWrapper) NS_IMPL_CYCLE_COLLECTION_UNLINK(mErrorSink->mError) + for (OutputMediaStream& s : tmp->mOutputStreams) { + s.mStream->SetFinishedOnInactive(true); + } NS_IMPL_CYCLE_COLLECTION_UNLINK(mOutputStreams) - NS_IMPL_CYCLE_COLLECTION_UNLINK(mOutputTrackSources) NS_IMPL_CYCLE_COLLECTION_UNLINK(mPlayed) NS_IMPL_CYCLE_COLLECTION_UNLINK(mTextTrackManager) NS_IMPL_CYCLE_COLLECTION_UNLINK(mAudioTrackList) @@ -1865,6 +1834,7 @@ void HTMLMediaElement::ShutdownDecoder() { if (mMediaSource) { mMediaSource->CompletePendingTransactions(); } + DiscardFinishWhenEndedOutputStreams(); mDecoder->Shutdown(); DDUNLINKCHILD(mDecoder.get()); mDecoder = nullptr; @@ -1947,6 +1917,17 @@ void HTMLMediaElement::AbortExistingLoads() { mFirstFrameListener = nullptr; } + // When aborting the existing loads, empty the objects in audio track list and + // video track list, no events (in particular, no removetrack events) are + // fired as part of this. Ending MediaTrack sends track ended notifications, + // so we empty the track lists prior. + if (AudioTracks()) { + AudioTracks()->EmptyTracks(); + } + if (VideoTracks()) { + VideoTracks()->EmptyTracks(); + } + if (mDecoder) { fireTimeUpdate = mDecoder->GetCurrentTime() != 0.0; ShutdownDecoder(); @@ -1955,6 +1936,8 @@ void HTMLMediaElement::AbortExistingLoads() { EndSrcMediaStreamPlayback(); } + DiscardFinishWhenEndedOutputStreams(); + RemoveMediaElementFromURITable(); mLoadingSrc = nullptr; mLoadingSrcTriggeringPrincipal = nullptr; @@ -1996,7 +1979,6 @@ void HTMLMediaElement::AbortExistingLoads() { RejectPromises(TakePendingPlayPromises(), NS_ERROR_DOM_MEDIA_ABORT_ERR); } ChangeNetworkState(NETWORK_EMPTY); - RemoveMediaTracks(); ChangeReadyState(HAVE_NOTHING); // TODO: Apply the rules for text track cue rendering Bug 865407 @@ -2044,7 +2026,6 @@ void HTMLMediaElement::NoSupportedMediaSourceError( ShutdownDecoder(); } mErrorSink->SetError(MEDIA_ERR_SRC_NOT_SUPPORTED, aErrorDetails); - RemoveMediaTracks(); ChangeDelayLoadStatus(false); UpdateAudioChannelPlayingState(); RejectPromises(TakePendingPlayPromises(), @@ -2361,10 +2342,19 @@ void HTMLMediaElement::NotifyMediaTrackEnabled(dom::MediaTrack* aTrack) { } } } - } - // The set of enabled/selected tracks changed. - mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources); + if (mReadyState == HAVE_NOTHING) { + // No MediaStreamTracks are captured until we have metadata. + return; + } + for (OutputMediaStream& ms : mOutputStreams) { + if (aTrack->AsVideoTrack() && ms.mCapturingAudioOnly) { + // If the output stream is for audio only we ignore video tracks. + continue; + } + AddCaptureMediaTrackToOutputStream(aTrack, ms); + } + } } void HTMLMediaElement::NotifyMediaTrackDisabled(dom::MediaTrack* aTrack) { @@ -2372,13 +2362,14 @@ void HTMLMediaElement::NotifyMediaTrackDisabled(dom::MediaTrack* aTrack) { if (!aTrack) { return; } - +#ifdef DEBUG nsString id; aTrack->GetId(id); LOG(LogLevel::Debug, ("MediaElement %p %sTrack with id %s disabled", this, aTrack->AsAudioTrack() ? "Audio" : "Video", NS_ConvertUTF16toUTF8(id).get())); +#endif MOZ_ASSERT((!aTrack->AsAudioTrack() || !aTrack->AsAudioTrack()->Enabled()) && (!aTrack->AsVideoTrack() || !aTrack->AsVideoTrack()->Selected())); @@ -2420,8 +2411,45 @@ void HTMLMediaElement::NotifyMediaTrackDisabled(dom::MediaTrack* aTrack) { } } - // The set of enabled/selected tracks changed. - mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources); + if (mReadyState == HAVE_NOTHING) { + // No MediaStreamTracks are captured until we have metadata, and code + // below doesn't do anything for captured decoders. + return; + } + + for (OutputMediaStream& ms : mOutputStreams) { + if (ms.mCapturingDecoder) { + MOZ_ASSERT(!ms.mCapturingMediaStream); + continue; + } + if (ms.mCapturingAudioOnly && aTrack->AsVideoTrack()) { + continue; + } + MOZ_ASSERT(ms.mCapturingMediaStream); + for (int32_t i = ms.mTracks.Length() - 1; i >= 0; --i) { + if (ms.mTracks[i].first() != aTrack->GetId()) { + continue; + } + // The source of this track just ended. Force-notify that it ended. + // If we bounce it to the MediaTrackGraph it might not be picked up, + // for instance if the MediaInputPort was destroyed in the same + // iteration as it was added. + mMainThreadEventTarget->Dispatch(NewRunnableMethod( + "StreamCaptureTrackSource::OverrideEnded", + static_cast(ms.mTracks[i].second().get()), + &StreamCaptureTrackSource::OverrideEnded)); + + ms.mTracks.RemoveElementAt(i); + break; + } +#ifdef DEBUG + for (auto pair : ms.mTracks) { + MOZ_ASSERT(pair.first() != aTrack->GetId(), + "The same MediaTrack was forwarded to the output stream more " + "than once. This shouldn't happen."); + } +#endif + } } void HTMLMediaElement::DealWithFailedElement(nsIContent* aSourceElement) { @@ -2443,8 +2471,6 @@ void HTMLMediaElement::LoadFromSourceChildren() { AddMutationObserverUnlessExists(this); - RemoveMediaTracks(); - while (true) { Element* child = GetNextSource(); if (!child) { @@ -3141,257 +3167,109 @@ void HTMLMediaElement::SetMuted(bool aMuted) { PauseIfShouldNotBePlaying(); } -void HTMLMediaElement::GetAllEnabledMediaTracks( - nsTArray>& aTracks) { - if (AudioTrackList* tracks = AudioTracks()) { - for (size_t i = 0; i < tracks->Length(); ++i) { - AudioTrack* track = (*tracks)[i]; - if (track->Enabled()) { - aTracks.AppendElement(track); - } - } - } - if (IsVideo()) { - if (VideoTrackList* tracks = VideoTracks()) { - for (size_t i = 0; i < tracks->Length(); ++i) { - VideoTrack* track = (*tracks)[i]; - if (track->Selected()) { - aTracks.AppendElement(track); - } - } - } - } -} - void HTMLMediaElement::SetCapturedOutputStreamsEnabled(bool aEnabled) { - for (auto& entry : mOutputTrackSources) { - entry.GetData()->SetEnabled(aEnabled); + for (OutputMediaStream& ms : mOutputStreams) { + if (ms.mCapturingDecoder) { + MOZ_ASSERT(!ms.mCapturingMediaStream); + continue; + } + for (auto pair : ms.mTracks) { + static_cast(pair.second().get()) + ->SetEnabled(aEnabled); + + LOG(LogLevel::Debug, ("%s track %p for captured MediaStream %p", + aEnabled ? "Enabled" : "Disabled", + pair.second().get(), ms.mStream.get())); + } } } -void HTMLMediaElement::AddOutputTrackSourceToOutputStream( - MediaElementTrackSource* aSource, OutputMediaStream& aOutputStream, - AddTrackMode aMode) { +void HTMLMediaElement::AddCaptureMediaTrackToOutputStream( + dom::MediaTrack* aTrack, OutputMediaStream& aOutputStream, + bool aAsyncAddtrack) { + if (aOutputStream.mCapturingDecoder) { + MOZ_ASSERT(!aOutputStream.mCapturingMediaStream); + return; + } + aOutputStream.mCapturingMediaStream = true; + if (aOutputStream.mStream == mSrcStream) { // Cycle detected. This can happen since tracks are added async. // We avoid forwarding it to the output here or we'd get into an infloop. - LOG(LogLevel::Warning, - ("NOT adding output track source %p to output stream " - "%p -- cycle detected", - aSource, aOutputStream.mStream.get())); return; } - LOG(LogLevel::Debug, ("Adding output track source %p to output stream %p", - aSource, aOutputStream.mStream.get())); - - RefPtr domTrack; - if (aSource->Track()->mType == MediaSegment::AUDIO) { - domTrack = new AudioStreamTrack(aOutputStream.mStream->GetParentObject(), - aSource->Track(), aSource); - } else { - domTrack = new VideoStreamTrack(aOutputStream.mStream->GetParentObject(), - aSource->Track(), aSource); + if (!aTrack) { + MOZ_ASSERT(false, "Bad MediaTrack"); + return; } - switch (aMode) { - case AddTrackMode::ASYNC: - mMainThreadEventTarget->Dispatch( - NewRunnableMethod>( - "DOMMediaStream::AddTrackInternal", aOutputStream.mStream, - &DOMMediaStream::AddTrackInternal, domTrack)); - break; - case AddTrackMode::SYNC: - aOutputStream.mStream->AddTrackInternal(domTrack); - break; - default: - MOZ_CRASH("Unexpected mode"); + MediaStreamTrack* inputTrack = mSrcStream->GetTrackById(aTrack->GetId()); + MOZ_ASSERT(inputTrack); + if (!inputTrack) { + NS_ERROR("Input track not found in source stream"); + return; } + MOZ_DIAGNOSTIC_ASSERT(!inputTrack->Ended()); - LOG(LogLevel::Debug, - ("Created capture %s track %p", - domTrack->AsAudioStreamTrack() ? "audio" : "video", domTrack.get())); -} - -void HTMLMediaElement::UpdateOutputTrackSources() { - // This updates the track sources in mOutputTrackSources so they're in sync - // with the tracks being currently played, and state saying whether we should - // be capturing tracks. This method is long so here is a breakdown: - // - Figure out the tracks that should be captured - // - Diff those against currently captured tracks (mOutputTrackSources), into - // tracks-to-add, and tracks-to-remove - // - Remove the tracks in tracks-to-remove and dispatch "removetrack" and - // "ended" events for them - // - If playback has ended, or there is no longer a media provider object, - // remove any OutputMediaStreams that have the finish-when-ended flag set - // - Create track sources for, and add to OutputMediaStreams, the tracks in - // tracks-to-add - - const bool shouldHaveTrackSources = mTracksCaptured.Ref() && - !IsPlaybackEnded() && - mReadyState >= HAVE_METADATA; - - // Add track sources for all enabled/selected MediaTracks. nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow(); if (!window) { return; } - if (mDecoder) { - mDecoder->SetOutputCaptured(mTracksCaptured.Ref()); + MediaSegment::Type type = inputTrack->AsAudioStreamTrack() + ? MediaSegment::AUDIO + : MediaSegment::VIDEO; + ProcessedMediaTrack* track = + inputTrack->Graph()->CreateForwardedInputTrack(type); + RefPtr port = inputTrack->ForwardTrackContentsTo(track); + auto source = MakeRefPtr(&inputTrack->GetSource(), + track, port); + + // Track is muted initially, so we don't leak data if it's added while paused + // and an MTG iteration passes before the mute comes into effect. + source->SetEnabled(mSrcStreamIsPlaying); + + RefPtr domTrack; + if (inputTrack->AsAudioStreamTrack()) { + domTrack = new AudioStreamTrack(window, track, source); + } else { + domTrack = new VideoStreamTrack(window, track, source); } - // Start with all MediaTracks - AutoTArray, 4> mediaTracksToAdd; - if (shouldHaveTrackSources) { - GetAllEnabledMediaTracks(mediaTracksToAdd); - } + aOutputStream.mTracks.AppendElement( + Pair>(aTrack->GetId(), + source.get())); - // ...and all MediaElementTrackSources. - AutoTArray trackSourcesToRemove; - for (const auto& entry : mOutputTrackSources) { - trackSourcesToRemove.AppendElement(entry.GetKey()); - } - - // Then work out the differences. - for (const auto& track : - AutoTArray, 4>(mediaTracksToAdd)) { - if (mOutputTrackSources.GetWeak(track->GetId())) { - mediaTracksToAdd.RemoveElement(track); - trackSourcesToRemove.RemoveElement(track->GetId()); - } - } - - // First remove stale track sources. - for (const auto& id : trackSourcesToRemove) { - RefPtr source = mOutputTrackSources.GetWeak(id); - - LOG(LogLevel::Debug, ("Removing output track source %p for track %s", - source.get(), NS_ConvertUTF16toUTF8(id).get())); - - if (mDecoder) { - mDecoder->RemoveOutputTrack(source->Track()); - } - - // The source of this track just ended. Force-notify that it ended. - // If we bounce it to the MediaTrackGraph it might not be picked up, - // for instance if the MediaInputPort was destroyed in the same - // iteration as it was added. + if (aAsyncAddtrack) { mMainThreadEventTarget->Dispatch( - NewRunnableMethod("MediaElementTrackSource::OverrideEnded", source, - &MediaElementTrackSource::OverrideEnded)); - - mOutputTrackSources.Remove(id); + NewRunnableMethod>( + "DOMMediaStream::AddTrackInternal", aOutputStream.mStream, + &DOMMediaStream::AddTrackInternal, domTrack)); + } else { + aOutputStream.mStream->AddTrackInternal(domTrack); } - // Then update finish-when-ended output streams as needed. + LOG(LogLevel::Debug, + ("Created %s track %p from track %p through MediaInputPort %p", + inputTrack->AsAudioStreamTrack() ? "audio" : "video", domTrack.get(), + inputTrack, port.get())); +} + +void HTMLMediaElement::DiscardFinishWhenEndedOutputStreams() { + // Discard all output streams that have finished now. for (int32_t i = mOutputStreams.Length() - 1; i >= 0; --i) { if (!mOutputStreams[i].mFinishWhenEnded) { continue; } - - if (!mOutputStreams[i].mFinishWhenEndedLoadingSrc && - !mOutputStreams[i].mFinishWhenEndedAttrStream) { - // This finish-when-ended stream has not seen any source loaded yet. - // Update the loading src if it's time. - if (!IsPlaybackEnded()) { - if (mLoadingSrc) { - mOutputStreams[i].mFinishWhenEndedLoadingSrc = mLoadingSrc; - } else if (mSrcAttrStream) { - mOutputStreams[i].mFinishWhenEndedAttrStream = mSrcAttrStream; - } - } - continue; - } - - // Discard finish-when-ended output streams with a loading src set as - // needed. - if (!IsPlaybackEnded() && - mLoadingSrc == mOutputStreams[i].mFinishWhenEndedLoadingSrc) { - continue; - } - if (!IsPlaybackEnded() && - mSrcAttrStream == mOutputStreams[i].mFinishWhenEndedAttrStream) { - continue; - } LOG(LogLevel::Debug, - ("Playback ended or source changed. Discarding stream %p", + ("Playback ended. Letting output stream %p go inactive", mOutputStreams[i].mStream.get())); + mOutputStreams[i].mStream->SetFinishedOnInactive(true); + if (mOutputStreams[i].mCapturingDecoder) { + mDecoder->RemoveOutputStream(mOutputStreams[i].mStream); + } mOutputStreams.RemoveElementAt(i); - if (mOutputStreams.IsEmpty()) { - mTracksCaptured = nullptr; - } - } - - // Finally add new MediaTracks. - for (const auto& mediaTrack : mediaTracksToAdd) { - nsAutoString id; - mediaTrack->GetId(id); - - MediaSegment::Type type; - if (mediaTrack->AsAudioTrack()) { - type = MediaSegment::AUDIO; - } else if (mediaTrack->AsVideoTrack()) { - type = MediaSegment::VIDEO; - } else { - MOZ_CRASH("Unknown track type"); - } - - RefPtr track; - RefPtr source; - if (mDecoder) { - track = mTracksCaptured.Ref()->mTrack->Graph()->CreateForwardedInputTrack( - type); - RefPtr principal = GetCurrentPrincipal(); - if (!principal || IsCORSSameOrigin()) { - principal = NodePrincipal(); - } - source = MakeAndAddRef(track, principal); - mDecoder->AddOutputTrack(track); - } else if (mSrcStream) { - MediaStreamTrack* inputTrack; - if (AudioTrack* t = mediaTrack->AsAudioTrack()) { - inputTrack = t->GetAudioStreamTrack(); - } else if (VideoTrack* t = mediaTrack->AsVideoTrack()) { - inputTrack = t->GetVideoStreamTrack(); - } else { - MOZ_CRASH("Unknown track type"); - } - MOZ_ASSERT(inputTrack); - if (!inputTrack) { - NS_ERROR("Input track not found in source stream"); - return; - } - MOZ_DIAGNOSTIC_ASSERT(!inputTrack->Ended()); - - track = inputTrack->Graph()->CreateForwardedInputTrack(type); - RefPtr port = inputTrack->ForwardTrackContentsTo(track); - source = MakeAndAddRef(&inputTrack->GetSource(), - track, port); - - // Track is muted initially, so we don't leak data if it's added while - // paused and an MTG iteration passes before the mute comes into effect. - source->SetEnabled(mSrcStreamIsPlaying); - } else { - MOZ_CRASH("Unknown source"); - } - - LOG(LogLevel::Debug, ("Adding output track source %p for track %s", - source.get(), NS_ConvertUTF16toUTF8(id).get())); - - track->QueueSetAutoend(false); - MOZ_DIAGNOSTIC_ASSERT(!mOutputTrackSources.GetWeak(id)); - mOutputTrackSources.Put(id, source); - - // Add the new track source to any existing output streams - for (OutputMediaStream& ms : mOutputStreams) { - if (source->Track()->mType == MediaSegment::VIDEO && - ms.mCapturingAudioOnly) { - // If the output stream is for audio only we ignore video sources. - continue; - } - AddOutputTrackSourceToOutputStream(source, ms); - } } } @@ -3413,48 +3291,31 @@ bool HTMLMediaElement::CanBeCaptured(StreamCaptureType aCaptureType) { already_AddRefed HTMLMediaElement::CaptureStreamInternal( StreamCaptureBehavior aFinishBehavior, StreamCaptureType aStreamCaptureType, MediaTrackGraph* aGraph) { + MOZ_RELEASE_ASSERT(aGraph); MOZ_ASSERT(CanBeCaptured(aStreamCaptureType)); MarkAsContentSource(CallerAPI::CAPTURE_STREAM); MarkAsTainted(); - if (mTracksCaptured.Ref() && - aGraph != mTracksCaptured.Ref()->mTrack->Graph()) { + // We don't support routing to a different graph. + if (!mOutputStreams.IsEmpty() && + aGraph != mOutputStreams[0].mGraphKeepAliveDummyStream->mTrack->Graph()) { return nullptr; } - if (!mTracksCaptured.Ref()) { - // This is the first output stream, or there are no tracks. If the former, - // start capturing all tracks. If the latter, they will be added later. - mTracksCaptured = MakeRefPtr( - aGraph->CreateSourceTrack(MediaSegment::AUDIO)); - UpdateOutputTrackSources(); - } - + OutputMediaStream* out = mOutputStreams.AppendElement(); nsPIDOMWindowInner* window = OwnerDoc()->GetInnerWindow(); - OutputMediaStream* out = mOutputStreams.AppendElement(OutputMediaStream( - MakeRefPtr(window), - aStreamCaptureType == StreamCaptureType::CAPTURE_AUDIO, - aFinishBehavior == StreamCaptureBehavior::FINISH_WHEN_ENDED)); - - if (aFinishBehavior == StreamCaptureBehavior::FINISH_WHEN_ENDED && - !mOutputTrackSources.IsEmpty()) { - // This output stream won't receive any more tracks when playback of the - // current src of this media element ends, or when the src of this media - // element changes. If we're currently playing something (i.e., if there are - // tracks currently captured), set the current src on the output stream so - // this can be tracked. If we're not playing anything, - // UpdateOutputTrackSources will set the current src when it becomes - // available later. - if (mLoadingSrc) { - out->mFinishWhenEndedLoadingSrc = mLoadingSrc; - } - if (mSrcAttrStream) { - out->mFinishWhenEndedAttrStream = mSrcAttrStream; - } - MOZ_ASSERT(out->mFinishWhenEndedLoadingSrc || - out->mFinishWhenEndedAttrStream); - } + out->mGraphKeepAliveDummyStream = + mOutputStreams.Length() == 1 + ? MakeRefPtr( + aGraph->CreateSourceTrack(MediaSegment::AUDIO)) + : mOutputStreams[0].mGraphKeepAliveDummyStream; + out->mStream = MakeAndAddRef(window); + out->mStream->SetFinishedOnInactive(false); + out->mFinishWhenEnded = + aFinishBehavior == StreamCaptureBehavior::FINISH_WHEN_ENDED; + out->mCapturingAudioOnly = + aStreamCaptureType == StreamCaptureType::CAPTURE_AUDIO; if (aStreamCaptureType == StreamCaptureType::CAPTURE_AUDIO) { if (mSrcStream) { @@ -3470,22 +3331,41 @@ already_AddRefed HTMLMediaElement::CaptureStreamInternal( mAudioCaptured = true; } - for (const auto& entry : mOutputTrackSources) { - const RefPtr& source = entry.GetData(); - if (source->Track()->mType == MediaSegment::VIDEO) { - // Only add video tracks if we're a video element and the output stream - // wants video. - if (!IsVideo()) { - continue; - } - if (out->mCapturingAudioOnly) { - continue; - } - } - AddOutputTrackSourceToOutputStream(source, *out, AddTrackMode::SYNC); + if (mDecoder) { + out->mCapturingDecoder = true; + mDecoder->AddOutputStream(out->mStream, out->mGraphKeepAliveDummyStream); + } else if (mSrcStream) { + out->mCapturingMediaStream = true; } - return do_AddRef(out->mStream); + if (mReadyState == HAVE_NOTHING) { + // Do not expose the tracks until we have metadata. + RefPtr result = out->mStream; + return result.forget(); + } + + if (mSrcStream) { + MOZ_DIAGNOSTIC_ASSERT(AudioTracks(), "Element can't have been unlinked"); + for (size_t i = 0; i < AudioTracks()->Length(); ++i) { + AudioTrack* t = (*AudioTracks())[i]; + if (t->Enabled()) { + AddCaptureMediaTrackToOutputStream(t, *out, false); + } + } + if (IsVideo() && !out->mCapturingAudioOnly) { + MOZ_DIAGNOSTIC_ASSERT(VideoTracks(), "Element can't have been unlinked"); + // Only add video tracks if we're a video element and the output stream + // wants video. + for (size_t i = 0; i < VideoTracks()->Length(); ++i) { + VideoTrack* t = (*VideoTracks())[i]; + if (t->Selected()) { + AddCaptureMediaTrackToOutputStream(t, *out, false); + } + } + } + } + RefPtr result = out->mStream; + return result.forget(); } already_AddRefed HTMLMediaElement::CaptureAudio( @@ -3771,7 +3651,7 @@ HTMLMediaElement::HTMLMediaElement( OwnerDoc()->AbstractMainThreadFor(TaskCategory::Other)), mShutdownObserver(new ShutdownObserver), mPlayed(new TimeRanges(ToSupports(OwnerDoc()))), - mTracksCaptured(nullptr, "HTMLMediaElement::mTracksCaptured"), + mPaused(true, "HTMLMediaElement::mPaused"), mErrorSink(new ErrorSink(this)), mAudioChannelWrapper(new AudioChannelAgentCallback(this)), mSink(MakePair(nsString(), RefPtr())), @@ -3799,17 +3679,6 @@ void HTMLMediaElement::Init() { mWatchManager.Watch(mPaused, &HTMLMediaElement::UpdateWakeLock); - mWatchManager.Watch(mTracksCaptured, - &HTMLMediaElement::UpdateOutputTrackSources); - mWatchManager.Watch(mReadyState, &HTMLMediaElement::UpdateOutputTrackSources); - - mWatchManager.Watch(mDownloadSuspendedByCache, - &HTMLMediaElement::UpdateReadyStateInternal); - mWatchManager.Watch(mFirstFrameLoaded, - &HTMLMediaElement::UpdateReadyStateInternal); - mWatchManager.Watch(mSrcStreamPlaybackEnded, - &HTMLMediaElement::UpdateReadyStateInternal); - ErrorResult rv; double defaultVolume = Preferences::GetFloat("media.default_volume", 1.0); @@ -3835,8 +3704,6 @@ HTMLMediaElement::~HTMLMediaElement() { !mHasSelfReference, "How can we be destroyed if we're still holding a self reference?"); - mWatchManager.Shutdown(); - mShutdownObserver->Unsubscribe(); if (mVideoFrameContainer) { @@ -4179,6 +4046,14 @@ void HTMLMediaElement::ReleaseAudioWakeLockIfExists() { void HTMLMediaElement::WakeLockRelease() { ReleaseAudioWakeLockIfExists(); } +HTMLMediaElement::OutputMediaStream::OutputMediaStream() + : mFinishWhenEnded(false), + mCapturingAudioOnly(false), + mCapturingDecoder(false), + mCapturingMediaStream(false) {} + +HTMLMediaElement::OutputMediaStream::~OutputMediaStream() = default; + void HTMLMediaElement::GetEventTargetParent(EventChainPreVisitor& aVisitor) { if (!this->Controls() || !aVisitor.mEvent->mFlags.mIsTrusted) { nsGenericHTMLElement::GetEventTargetParent(aVisitor); @@ -4771,6 +4646,16 @@ nsresult HTMLMediaElement::FinishDecoderSetup(MediaDecoder* aDecoder) { #endif } + for (OutputMediaStream& ms : mOutputStreams) { + if (ms.mCapturingMediaStream) { + MOZ_ASSERT(!ms.mCapturingDecoder); + continue; + } + + ms.mCapturingDecoder = true; + aDecoder->AddOutputStream(ms.mStream, ms.mGraphKeepAliveDummyStream); + } + if (mMediaKeys) { if (mMediaKeys->GetCDMProxy()) { mDecoder->SetCDMProxy(mMediaKeys->GetCDMProxy()); @@ -4904,6 +4789,7 @@ class HTMLMediaElement::MediaStreamTrackListener mElement->mSrcStream.get())); mElement->PlaybackEnded(); + mElement->UpdateReadyStateInternal(); } void NotifyInactive() override { @@ -5091,10 +4977,18 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback() { mSrcStream->UnregisterTrackListener(mMediaStreamTrackListener.get()); mMediaStreamTrackListener = nullptr; + mSrcStreamTracksAvailable = false; mSrcStreamPlaybackEnded = false; mSrcStreamReportPlaybackEnded = false; mSrcStreamVideoPrincipal = nullptr; +#ifdef DEBUG + for (OutputMediaStream& ms : mOutputStreams) { + // These tracks were removed by clearing AudioTracks() and VideoTracks(). + MOZ_ASSERT(ms.mTracks.IsEmpty()); + } +#endif + mSrcStream = nullptr; } @@ -5131,7 +5025,7 @@ void HTMLMediaElement::NotifyMediaStreamTrackAdded( } #ifdef DEBUG - nsAutoString id; + nsString id; aTrack->GetId(id); LOG(LogLevel::Debug, ("%p, Adding %sTrack with id %s", this, @@ -5161,11 +5055,25 @@ void HTMLMediaElement::NotifyMediaStreamTrackAdded( } } - // The set of enabled AudioTracks and selected video track might have changed. - mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal); - mAbstractMainThread->TailDispatcher().AddDirectTask( - NewRunnableMethod("HTMLMediaElement::FirstFrameLoaded", this, - &HTMLMediaElement::FirstFrameLoaded)); + UpdateReadyStateInternal(); + + if (!mSrcStreamTracksAvailable) { + mAbstractMainThread->Dispatch(NS_NewRunnableFunction( + "HTMLMediaElement::NotifyMediaStreamTrackAdded->FirstFrameLoaded", + [this, self = RefPtr(this), stream = mSrcStream]() { + if (!mSrcStream || mSrcStream != stream) { + return; + } + + LOG(LogLevel::Debug, + ("MediaElement %p MediaStream tracks available", this)); + + mSrcStreamTracksAvailable = true; + + FirstFrameLoaded(); + UpdateReadyStateInternal(); + })); + } } void HTMLMediaElement::NotifyMediaStreamTrackRemoved( @@ -5211,10 +5119,6 @@ void HTMLMediaElement::MetadataLoaded(const MediaInfo* aInfo, UniquePtr aTags) { MOZ_ASSERT(NS_IsMainThread()); - if (mDecoder) { - ConstructMediaTracks(aInfo); - } - SetMediaInfo(*aInfo); mIsEncrypted = @@ -5223,10 +5127,6 @@ void HTMLMediaElement::MetadataLoaded(const MediaInfo* aInfo, mLoadedDataFired = false; ChangeReadyState(HAVE_METADATA); - // Add output tracks synchronously now to be sure they're available in - // "loadedmetadata" event handlers. - UpdateOutputTrackSources(); - DispatchAsyncEvent(NS_LITERAL_STRING("durationchange")); if (IsVideo() && HasVideo()) { DispatchAsyncEvent(NS_LITERAL_STRING("resize")); @@ -5267,18 +5167,44 @@ void HTMLMediaElement::MetadataLoaded(const MediaInfo* aInfo, mDefaultPlaybackStartPosition = 0.0; } - mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal); + UpdateReadyStateInternal(); + + if (!mSrcStream) { + return; + } + + for (OutputMediaStream& ms : mOutputStreams) { + if (AudioTracks()) { + for (size_t i = 0; i < AudioTracks()->Length(); ++i) { + AudioTrack* t = (*AudioTracks())[i]; + if (t->Enabled()) { + AddCaptureMediaTrackToOutputStream(t, ms); + } + } + } + if (VideoTracks() && IsVideo() && !ms.mCapturingAudioOnly) { + // Only add video tracks if we're a video element and the output stream + // wants video. + for (size_t i = 0; i < VideoTracks()->Length(); ++i) { + VideoTrack* t = (*VideoTracks())[i]; + if (t->Selected()) { + AddCaptureMediaTrackToOutputStream(t, ms); + } + } + } + } } void HTMLMediaElement::FirstFrameLoaded() { LOG(LogLevel::Debug, ("%p, FirstFrameLoaded() mFirstFrameLoaded=%d mWaitingForKey=%d", this, - mFirstFrameLoaded.Ref(), mWaitingForKey)); + mFirstFrameLoaded, mWaitingForKey)); NS_ASSERTION(!mSuspendedAfterFirstFrame, "Should not have already suspended"); if (!mFirstFrameLoaded) { mFirstFrameLoaded = true; + UpdateReadyStateInternal(); } ChangeDelayLoadStatus(false); @@ -5308,6 +5234,12 @@ void HTMLMediaElement::DecodeError(const MediaResult& aError) { DecoderDoctorDiagnostics diagnostics; diagnostics.StoreDecodeError(OwnerDoc(), aError, src, __func__); + if (AudioTracks()) { + AudioTracks()->EmptyTracks(); + } + if (VideoTracks()) { + VideoTracks()->EmptyTracks(); + } if (mIsLoadingFromSourceChildren) { mErrorSink->ResetError(); if (mSourceLoadCandidate) { @@ -5348,8 +5280,7 @@ void HTMLMediaElement::PlaybackEnded() { NS_ASSERTION(!mDecoder || mDecoder->IsEnded(), "Decoder fired ended, but not in ended state"); - // IsPlaybackEnded() became true. - mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources); + DiscardFinishWhenEndedOutputStreams(); if (mSrcStream) { LOG(LogLevel::Debug, @@ -5431,6 +5362,7 @@ void HTMLMediaElement::SeekAborted() { void HTMLMediaElement::NotifySuspendedByCache(bool aSuspendedByCache) { mDownloadSuspendedByCache = aSuspendedByCache; + UpdateReadyStateInternal(); } void HTMLMediaElement::DownloadSuspended() { @@ -5485,7 +5417,7 @@ void HTMLMediaElement::CheckProgress(bool aHaveNewProgress) { } // Download statistics may have been updated, force a recheck of the // readyState. - mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal); + UpdateReadyStateInternal(); } if (now - mDataTime >= TimeDuration::FromMilliseconds(STALL_MS)) { @@ -5578,12 +5510,14 @@ void HTMLMediaElement::UpdateReadyStateInternal() { return; } - if (mDecoder) { - // IsPlaybackEnded() might have become false. - mWatchManager.ManualNotify(&HTMLMediaElement::UpdateOutputTrackSources); - } - if (mSrcStream && mReadyState < HAVE_METADATA) { + if (!mSrcStreamTracksAvailable) { + LOG(LogLevel::Debug, ("MediaElement %p UpdateReadyStateInternal() " + "MediaStreamTracks not available yet", + this)); + return; + } + bool hasAudioTracks = AudioTracks() && !AudioTracks()->IsEmpty(); bool hasVideoTracks = VideoTracks() && !VideoTracks()->IsEmpty(); if (!hasAudioTracks && !hasVideoTracks) { @@ -6171,14 +6105,8 @@ void HTMLMediaElement::NotifyDecoderPrincipalChanged() { RefPtr principal = GetCurrentPrincipal(); bool isSameOrigin = !principal || IsCORSSameOrigin(); mDecoder->UpdateSameOriginStatus(isSameOrigin); - - if (isSameOrigin) { - principal = NodePrincipal(); - } - for (const auto& entry : mOutputTrackSources) { - entry.GetData()->SetPrincipal(principal); - } - mDecoder->SetOutputTracksPrincipal(principal); + mDecoder->SetOutputStreamPrincipal(isSameOrigin ? NodePrincipal() + : principal.get()); } void HTMLMediaElement::Invalidate(bool aImageSizeChanged, @@ -6219,7 +6147,7 @@ void HTMLMediaElement::UpdateMediaSize(const nsIntSize& aSize) { } mMediaInfo.mVideo.mDisplay = aSize; - mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal); + UpdateReadyStateInternal(); if (mFirstFrameListener) { mSelectedVideoStreamTrack->RemoveVideoOutput(mFirstFrameListener); @@ -6962,9 +6890,7 @@ void HTMLMediaElement::NotifyWaitingForKey() { // Note: algorithm continues in UpdateReadyStateInternal() when all decoded // data enqueued in the MDSM is consumed. mWaitingForKey = WAITING_FOR_KEY; - // mWaitingForKey changed outside of UpdateReadyStateInternal. This may - // affect mReadyState. - mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal); + UpdateReadyStateInternal(); } } @@ -7002,9 +6928,7 @@ MediaDecoderOwner::NextFrameStatus HTMLMediaElement::NextFrameStatus() { return mDecoder->NextFrameStatus(); } if (mSrcStream) { - AutoTArray, 4> tracks; - GetAllEnabledMediaTracks(tracks); - if (!tracks.IsEmpty() && !mSrcStreamPlaybackEnded) { + if (mSrcStreamTracksAvailable && !mSrcStreamPlaybackEnded) { return NEXT_FRAME_AVAILABLE; } return NEXT_FRAME_UNAVAILABLE; @@ -7171,20 +7095,14 @@ void HTMLMediaElement::AudioCaptureTrackChange(bool aCapture) { } else if (!aCapture && mStreamWindowCapturer) { for (size_t i = 0; i < mOutputStreams.Length(); i++) { if (mOutputStreams[i].mStream == mStreamWindowCapturer->mStream) { - // We own this MediaStream, it is not exposed to JS. - AutoTArray, 2> tracks; - mStreamWindowCapturer->mStream->GetTracks(tracks); - for (auto& track : tracks) { - track->Stop(); + if (mOutputStreams[i].mCapturingDecoder && mDecoder) { + mDecoder->RemoveOutputStream(mOutputStreams[i].mStream); } mOutputStreams.RemoveElementAt(i); break; } } mStreamWindowCapturer = nullptr; - if (mOutputStreams.IsEmpty()) { - mTracksCaptured = nullptr; - } } } @@ -7323,10 +7241,12 @@ bool HTMLMediaElement::IsAudible() const { } void HTMLMediaElement::ConstructMediaTracks(const MediaInfo* aInfo) { - if (!aInfo) { + if (mMediaTracksConstructed || !aInfo) { return; } + mMediaTracksConstructed = true; + AudioTrackList* audioList = AudioTracks(); if (audioList && aInfo->HasAudio()) { const TrackInfo& info = aInfo->mAudio; @@ -7353,9 +7273,12 @@ void HTMLMediaElement::RemoveMediaTracks() { if (mAudioTrackList) { mAudioTrackList->RemoveTracks(); } + if (mVideoTrackList) { mVideoTrackList->RemoveTracks(); } + + mMediaTracksConstructed = false; } class MediaElementGMPCrashHelper : public GMPCrashHelper { diff --git a/dom/html/HTMLMediaElement.h b/dom/html/HTMLMediaElement.h index 9225562a00e6..ed7cdb1def5c 100644 --- a/dom/html/HTMLMediaElement.h +++ b/dom/html/HTMLMediaElement.h @@ -113,26 +113,6 @@ class HTMLMediaElement : public nsGenericHTMLElement, typedef mozilla::MediaDecoderOwner MediaDecoderOwner; typedef mozilla::MetadataTags MetadataTags; - // Helper struct to keep track of the MediaStreams returned by - // mozCaptureStream(). For each OutputMediaStream, dom::MediaTracks get - // captured into MediaStreamTracks which get added to - // OutputMediaStream::mStream. - struct OutputMediaStream { - OutputMediaStream(RefPtr aStream, bool aCapturingAudioOnly, - bool aFinishWhenEnded); - ~OutputMediaStream(); - - RefPtr mStream; - const bool mCapturingAudioOnly; - const bool mFinishWhenEnded; - // If mFinishWhenEnded is true, this is the URI of the first resource - // mStream got tracks for, if not a MediaStream. - nsCOMPtr mFinishWhenEndedLoadingSrc; - // If mFinishWhenEnded is true, this is the first MediaStream mStream got - // tracks for, if not a resource. - RefPtr mFinishWhenEndedAttrStream; - }; - MOZ_DECLARE_WEAKREFERENCE_TYPENAME(HTMLMediaElement) NS_DECL_NSIMUTATIONOBSERVER_CONTENTREMOVED @@ -271,9 +251,7 @@ class HTMLMediaElement : public nsGenericHTMLElement, void DispatchAsyncEvent(const nsAString& aName) final; // Triggers a recomputation of readyState. - void UpdateReadyState() override { - mWatchManager.ManualNotify(&HTMLMediaElement::UpdateReadyStateInternal); - } + void UpdateReadyState() override { UpdateReadyStateInternal(); } // Dispatch events that were raised while in the bfcache nsresult DispatchPendingMediaEvents(); @@ -715,6 +693,10 @@ class HTMLMediaElement : public nsGenericHTMLElement, Document* GetDocument() const override; + void ConstructMediaTracks(const MediaInfo* aInfo) override; + + void RemoveMediaTracks() override; + already_AddRefed CreateGMPCrashHelper() override; nsISerialEventTarget* MainThreadEventTarget() { @@ -747,17 +729,37 @@ class HTMLMediaElement : public nsGenericHTMLElement, class AudioChannelAgentCallback; class ChannelLoader; class ErrorSink; - class MediaElementTrackSource; class MediaLoadListener; class MediaStreamRenderer; class MediaStreamTrackListener; class FirstFrameListener; class ShutdownObserver; + class StreamCaptureTrackSource; MediaDecoderOwner::NextFrameStatus NextFrameStatus(); void SetDecoder(MediaDecoder* aDecoder); + // Holds references to the DOM wrappers for the MediaStreams that we're + // writing to. + struct OutputMediaStream { + OutputMediaStream(); + ~OutputMediaStream(); + + RefPtr mStream; + // Dummy stream to keep mGraph from shutting down when MediaDecoder shuts + // down. Shared across all OutputMediaStreams as one stream is enough to + // keep the graph alive. + RefPtr mGraphKeepAliveDummyStream; + bool mFinishWhenEnded; + bool mCapturingAudioOnly; + bool mCapturingDecoder; + bool mCapturingMediaStream; + + // The following members are keeping state for a captured MediaStream. + nsTArray>> mTracks; + }; + void PlayInternal(bool aHandlingUserInput); /** Use this method to change the mReadyState member, so required @@ -852,35 +854,28 @@ class HTMLMediaElement : public nsGenericHTMLElement, */ void NotifyMediaStreamTrackRemoved(const RefPtr& aTrack); - /** - * Convenience method to get in a single list all enabled AudioTracks and, if - * this is a video element, the selected VideoTrack. - */ - void GetAllEnabledMediaTracks(nsTArray>& aTracks); - /** * Enables or disables all tracks forwarded from mSrcStream to all * OutputMediaStreams. We do this for muting the tracks when pausing, * and unmuting when playing the media element again. + * + * If mSrcStream is unset, this does nothing. */ void SetCapturedOutputStreamsEnabled(bool aEnabled); /** - * Create a new MediaStreamTrack for the TrackSource corresponding to aTrack - * and add it to the DOMMediaStream in aOutputStream. This automatically sets - * the output track to enabled or disabled depending on our current playing - * state. + * Create a new MediaStreamTrack for aTrack and add it to the DOMMediaStream + * in aOutputStream. This automatically sets the output track to enabled or + * disabled depending on our current playing state. */ - enum class AddTrackMode { ASYNC, SYNC }; - void AddOutputTrackSourceToOutputStream( - MediaElementTrackSource* aSource, OutputMediaStream& aOutputStream, - AddTrackMode aMode = AddTrackMode::ASYNC); + void AddCaptureMediaTrackToOutputStream(dom::MediaTrack* aTrack, + OutputMediaStream& aOutputStream, + bool aAsyncAddtrack = true); /** - * Creates output track sources when this media element is captured, tracks - * exist, playback is not ended and readyState is >= HAVE_METADATA. + * Discard all output streams that are flagged to finish when playback ends. */ - void UpdateOutputTrackSources(); + void DiscardFinishWhenEndedOutputStreams(); /** * Returns an DOMMediaStream containing the played contents of this @@ -894,8 +889,8 @@ class HTMLMediaElement : public nsGenericHTMLElement, * reaching the stream. No video tracks will be captured in this case. */ already_AddRefed CaptureStreamInternal( - StreamCaptureBehavior aFinishBehavior, - StreamCaptureType aStreamCaptureType, MediaTrackGraph* aGraph); + StreamCaptureBehavior aBehavior, StreamCaptureType aType, + MediaTrackGraph* aGraph); /** * Initialize a decoder as a clone of an existing decoder in another @@ -1254,18 +1249,6 @@ class HTMLMediaElement : public nsGenericHTMLElement, // Pass information for deciding the video decode mode to decoder. void NotifyDecoderActivityChanges() const; - // Constructs an AudioTrack in mAudioTrackList if aInfo reports that audio is - // available, and a VideoTrack in mVideoTrackList if aInfo reports that video - // is available. - void ConstructMediaTracks(const MediaInfo* aInfo); - - // Removes all MediaTracks from mAudioTrackList and mVideoTrackList and fires - // "removetrack" on the lists accordingly. - // Note that by spec, this should not fire "removetrack". However, it appears - // other user agents do, per - // https://wpt.fyi/results/media-source/mediasource-avtracks.html. - void RemoveMediaTracks(); - // Mark the decoder owned by the element as tainted so that the // suspend-video-decoder is disabled. void MarkAsTainted(); @@ -1347,6 +1330,9 @@ class HTMLMediaElement : public nsGenericHTMLElement, // enabled audio tracks, while mSrcStream is set. RefPtr mMediaStreamRenderer; + // True once mSrcStream's initial set of tracks are known. + bool mSrcStreamTracksAvailable = false; + // True once PlaybackEnded() is called and we're playing a MediaStream. // Reset to false if we start playing mSrcStream again. Watchable mSrcStreamPlaybackEnded = { @@ -1366,12 +1352,6 @@ class HTMLMediaElement : public nsGenericHTMLElement, // writing to. nsTArray mOutputStreams; - // Mapping for output tracks, from dom::MediaTrack ids to the - // MediaElementTrackSource that represents the source of all corresponding - // MediaStreamTracks captured from this element. - nsRefPtrHashtable - mOutputTrackSources; - // Holds a reference to the first-frame-getting track listener attached to // mSelectedVideoStreamTrack. RefPtr mFirstFrameListener; @@ -1563,7 +1543,7 @@ class HTMLMediaElement : public nsGenericHTMLElement, // Playback of the video is paused either due to calling the // 'Pause' method, or playback not yet having started. - Watchable mPaused = {true, "HTMLMediaElement::mPaused"}; + Watchable mPaused; // The following two fields are here for the private storage of the builtin // video controls, and control 'casting' of the video to external devices @@ -1573,14 +1553,6 @@ class HTMLMediaElement : public nsGenericHTMLElement, // True if currently casting this video bool mIsCasting = false; - // Set while there are some OutputMediaStreams this media element's enabled - // and selected tracks are captured into. When set, all tracks are captured - // into the graph of this dummy track. - // NB: This is a SharedDummyTrack to allow non-default graphs (AudioContexts - // with an explicit sampleRate defined) to capture this element. When - // cross-graph tracks are supported, this can become a bool. - Watchable> mTracksCaptured; - // True if the sound is being captured. bool mAudioCaptured = false; @@ -1682,8 +1654,7 @@ class HTMLMediaElement : public nsGenericHTMLElement, EncryptionInfo mPendingEncryptedInitData; // True if the media's channel's download has been suspended. - Watchable mDownloadSuspendedByCache = { - false, "HTMLMediaElement::mDownloadSuspendedByCache"}; + bool mDownloadSuspendedByCache = false; // Disable the video playback by track selection. This flag might not be // enough if we ever expand the ability of supporting multi-tracks video @@ -1821,8 +1792,7 @@ class HTMLMediaElement : public nsGenericHTMLElement, bool mIsBlessed = false; // True if the first frame has been successfully loaded. - Watchable mFirstFrameLoaded = {false, - "HTMLMediaElement::mFirstFrameLoaded"}; + bool mFirstFrameLoaded = false; // Media elements also have a default playback start position, which must // initially be set to zero seconds. This time is used to allow the element to @@ -1837,6 +1807,10 @@ class HTMLMediaElement : public nsGenericHTMLElement, // For use by mochitests. Enabling pref "media.test.video-suspend" bool mForcedHidden = false; + // True if audio tracks and video tracks are constructed and added into the + // track list, false if all tracks are removed from the track list. + bool mMediaTracksConstructed = false; + Visibility mVisibilityState = Visibility::Untracked; UniquePtr mErrorSink; diff --git a/dom/media/ChannelMediaDecoder.cpp b/dom/media/ChannelMediaDecoder.cpp index 90450d28828e..e79b37913a70 100644 --- a/dom/media/ChannelMediaDecoder.cpp +++ b/dom/media/ChannelMediaDecoder.cpp @@ -221,27 +221,13 @@ void ChannelMediaDecoder::Shutdown() { mResourceCallback->Disconnect(); MediaDecoder::Shutdown(); + // Force any outstanding seek and byterange requests to complete + // to prevent shutdown from deadlocking. if (mResource) { - // Force any outstanding seek and byterange requests to complete - // to prevent shutdown from deadlocking. - mResourceClosePromise = mResource->Close(); + mResource->Close(); } } -void ChannelMediaDecoder::ShutdownInternal() { - if (!mResourceClosePromise) { - MediaShutdownManager::Instance().Unregister(this); - return; - } - - mResourceClosePromise->Then( - AbstractMainThread(), __func__, - [self = RefPtr(this)] { - MediaShutdownManager::Instance().Unregister(self); - }); - return; -} - nsresult ChannelMediaDecoder::Load(nsIChannel* aChannel, bool aIsPrivateBrowsing, nsIStreamListener** aStreamListener) { diff --git a/dom/media/ChannelMediaDecoder.h b/dom/media/ChannelMediaDecoder.h index ed53496d3be4..3a500890c3cd 100644 --- a/dom/media/ChannelMediaDecoder.h +++ b/dom/media/ChannelMediaDecoder.h @@ -59,7 +59,6 @@ class ChannelMediaDecoder }; protected: - void ShutdownInternal() override; void OnPlaybackEvent(MediaPlaybackEvent&& aEvent) override; void DurationChanged() override; void MetadataLoaded(UniquePtr aInfo, UniquePtr aTags, @@ -157,10 +156,6 @@ class ChannelMediaDecoder // True if we've been notified that the ChannelMediaResource has // a principal. bool mInitialChannelPrincipalKnown = false; - - // Set in Shutdown() when we start closing mResource, if mResource is set. - // Must resolve before we unregister the shutdown blocker. - RefPtr mResourceClosePromise; }; } // namespace mozilla diff --git a/dom/media/ChannelMediaResource.cpp b/dom/media/ChannelMediaResource.cpp index 8653269f36b0..06ab2c0d5011 100644 --- a/dom/media/ChannelMediaResource.cpp +++ b/dom/media/ChannelMediaResource.cpp @@ -589,15 +589,15 @@ nsresult ChannelMediaResource::SetupChannelHeaders(int64_t aOffset) { return NS_OK; } -RefPtr ChannelMediaResource::Close() { +nsresult ChannelMediaResource::Close() { NS_ASSERTION(NS_IsMainThread(), "Only call on main thread"); if (!mClosed) { CloseChannel(); + mCacheStream.Close(); mClosed = true; - return mCacheStream.Close(); } - return GenericPromise::CreateAndResolve(true, __func__); + return NS_OK; } already_AddRefed ChannelMediaResource::GetCurrentPrincipal() { diff --git a/dom/media/ChannelMediaResource.h b/dom/media/ChannelMediaResource.h index a6e6459b6a8a..a9e07e80691c 100644 --- a/dom/media/ChannelMediaResource.h +++ b/dom/media/ChannelMediaResource.h @@ -117,7 +117,7 @@ class ChannelMediaResource // Main thread nsresult Open(nsIStreamListener** aStreamListener) override; - RefPtr Close() override; + nsresult Close() override; void Suspend(bool aCloseImmediately) override; void Resume() override; already_AddRefed GetCurrentPrincipal() override; diff --git a/dom/media/CloneableWithRangeMediaResource.cpp b/dom/media/CloneableWithRangeMediaResource.cpp index 74d9cb4ce5b1..bf2839ba876e 100644 --- a/dom/media/CloneableWithRangeMediaResource.cpp +++ b/dom/media/CloneableWithRangeMediaResource.cpp @@ -148,9 +148,7 @@ nsresult CloneableWithRangeMediaResource::Open( return NS_OK; } -RefPtr CloneableWithRangeMediaResource::Close() { - return GenericPromise::CreateAndResolve(true, __func__); -} +nsresult CloneableWithRangeMediaResource::Close() { return NS_OK; } already_AddRefed CloneableWithRangeMediaResource::GetCurrentPrincipal() { diff --git a/dom/media/CloneableWithRangeMediaResource.h b/dom/media/CloneableWithRangeMediaResource.h index d3aafce731f2..4f78f798ab44 100644 --- a/dom/media/CloneableWithRangeMediaResource.h +++ b/dom/media/CloneableWithRangeMediaResource.h @@ -27,7 +27,7 @@ class CloneableWithRangeMediaResource : public BaseMediaResource { // Main thread nsresult Open(nsIStreamListener** aStreamListener) override; - RefPtr Close() override; + nsresult Close() override; void Suspend(bool aCloseImmediately) override {} void Resume() override {} already_AddRefed GetCurrentPrincipal() override; diff --git a/dom/media/DOMMediaStream.cpp b/dom/media/DOMMediaStream.cpp index e5d89a8451d4..850cc700f741 100644 --- a/dom/media/DOMMediaStream.cpp +++ b/dom/media/DOMMediaStream.cpp @@ -373,7 +373,6 @@ already_AddRefed DOMMediaStream::Clone() { } bool DOMMediaStream::Active() const { return mActive; } -bool DOMMediaStream::Audible() const { return mAudible; } MediaStreamTrack* DOMMediaStream::GetTrackById(const nsAString& aId) const { for (const auto& track : mTracks) { @@ -456,6 +455,20 @@ void DOMMediaStream::UnregisterTrackListener(TrackListener* aListener) { mTrackListeners.RemoveElement(aListener); } +void DOMMediaStream::SetFinishedOnInactive(bool aFinishedOnInactive) { + MOZ_ASSERT(NS_IsMainThread()); + + if (mFinishedOnInactive == aFinishedOnInactive) { + return; + } + + mFinishedOnInactive = aFinishedOnInactive; + + if (mFinishedOnInactive && !ContainsLiveTracks(mTracks)) { + NotifyTrackRemoved(nullptr); + } +} + void DOMMediaStream::NotifyTrackAdded(const RefPtr& aTrack) { MOZ_ASSERT(NS_IsMainThread()); @@ -504,6 +517,10 @@ void DOMMediaStream::NotifyTrackRemoved( } } + if (!mFinishedOnInactive) { + return; + } + if (mAudible) { // Check if we became inaudible. if (!ContainsLiveAudioTracks(mTracks)) { diff --git a/dom/media/DOMMediaStream.h b/dom/media/DOMMediaStream.h index 9a4a02d01c8c..68f14a6572c4 100644 --- a/dom/media/DOMMediaStream.h +++ b/dom/media/DOMMediaStream.h @@ -144,9 +144,6 @@ class DOMMediaStream : public DOMEventTargetHelper, // NON-WebIDL - // Returns true if this stream contains a live audio track. - bool Audible() const; - /** * Returns true if this DOMMediaStream has aTrack in mTracks. */ @@ -189,6 +186,10 @@ class DOMMediaStream : public DOMEventTargetHelper, // a dead pointer. Main thread only. void UnregisterTrackListener(TrackListener* aListener); + // Tells this MediaStream whether it can go inactive as soon as no tracks + // are live anymore. + void SetFinishedOnInactive(bool aFinishedOnInactive); + protected: virtual ~DOMMediaStream(); @@ -239,6 +240,10 @@ class DOMMediaStream : public DOMEventTargetHelper, // True if this stream has live audio tracks. bool mAudible = false; + + // For compatibility with mozCaptureStream, we in some cases do not go + // inactive until the MediaDecoder lets us. (Remove this in Bug 1302379) + bool mFinishedOnInactive = true; }; NS_DEFINE_STATIC_IID_ACCESSOR(DOMMediaStream, NS_DOMMEDIASTREAM_IID) diff --git a/dom/media/FileMediaResource.cpp b/dom/media/FileMediaResource.cpp index 09ee243bf01f..94671d502178 100644 --- a/dom/media/FileMediaResource.cpp +++ b/dom/media/FileMediaResource.cpp @@ -95,7 +95,7 @@ nsresult FileMediaResource::Open(nsIStreamListener** aStreamListener) { return NS_OK; } -RefPtr FileMediaResource::Close() { +nsresult FileMediaResource::Close() { NS_ASSERTION(NS_IsMainThread(), "Only call on main thread"); // Since mChennel is only accessed by main thread, there is no necessary to @@ -105,7 +105,7 @@ RefPtr FileMediaResource::Close() { mChannel = nullptr; } - return GenericPromise::CreateAndResolve(true, __func__); + return NS_OK; } already_AddRefed FileMediaResource::GetCurrentPrincipal() { diff --git a/dom/media/FileMediaResource.h b/dom/media/FileMediaResource.h index 18b1828b13d2..a1a61b56ca1a 100644 --- a/dom/media/FileMediaResource.h +++ b/dom/media/FileMediaResource.h @@ -23,7 +23,7 @@ class FileMediaResource : public BaseMediaResource { // Main thread nsresult Open(nsIStreamListener** aStreamListener) override; - RefPtr Close() override; + nsresult Close() override; void Suspend(bool aCloseImmediately) override {} void Resume() override {} already_AddRefed GetCurrentPrincipal() override; diff --git a/dom/media/MediaCache.cpp b/dom/media/MediaCache.cpp index 91fe23889bf8..6cb0f34763c3 100644 --- a/dom/media/MediaCache.cpp +++ b/dom/media/MediaCache.cpp @@ -161,7 +161,7 @@ class MediaCache { // file backing will be provided. static RefPtr GetMediaCache(int64_t aContentLength); - nsISerialEventTarget* OwnerThread() const { return sThread; } + nsIEventTarget* OwnerThread() const { return sThread; } // Brutally flush the cache contents. Main thread only. void Flush(); @@ -2196,18 +2196,17 @@ bool MediaCacheStream::AreAllStreamsForResourceSuspended(AutoLock& aLock) { return true; } -RefPtr MediaCacheStream::Close() { +void MediaCacheStream::Close() { MOZ_ASSERT(NS_IsMainThread()); if (!mMediaCache) { - return GenericPromise::CreateAndResolve(true, __func__); + return; } - - return InvokeAsync(OwnerThread(), "MediaCacheStream::Close", - [this, client = RefPtr(mClient)] { - AutoLock lock(mMediaCache->Monitor()); - CloseInternal(lock); - return GenericPromise::CreateAndResolve(true, __func__); - }); + OwnerThread()->Dispatch(NS_NewRunnableFunction( + "MediaCacheStream::Close", + [this, client = RefPtr(mClient)]() { + AutoLock lock(mMediaCache->Monitor()); + CloseInternal(lock); + })); } void MediaCacheStream::CloseInternal(AutoLock& aLock) { @@ -2735,7 +2734,7 @@ void MediaCacheStream::InitAsCloneInternal(MediaCacheStream* aOriginal) { lock.NotifyAll(); } -nsISerialEventTarget* MediaCacheStream::OwnerThread() const { +nsIEventTarget* MediaCacheStream::OwnerThread() const { return mMediaCache->OwnerThread(); } diff --git a/dom/media/MediaCache.h b/dom/media/MediaCache.h index c19669de8a70..5e639275e0b2 100644 --- a/dom/media/MediaCache.h +++ b/dom/media/MediaCache.h @@ -217,12 +217,12 @@ class MediaCacheStream : public DecoderDoctorLifeLogger { // on this class. void InitAsClone(MediaCacheStream* aOriginal); - nsISerialEventTarget* OwnerThread() const; + nsIEventTarget* OwnerThread() const; // These are called on the main thread. - // This must be called (and resolve) before the ChannelMediaResource + // This must be called (and return) before the ChannelMediaResource // used to create this MediaCacheStream is deleted. - RefPtr Close(); + void Close(); // This returns true when the stream has been closed. bool IsClosed(AutoLock&) const { return mClosed; } // Returns true when this stream is can be shared by a new resource load. diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp index 2e544410a511..8eb8ee98d8a0 100644 --- a/dom/media/MediaDecoder.cpp +++ b/dom/media/MediaDecoder.cpp @@ -6,7 +6,6 @@ #include "MediaDecoder.h" -#include "AudioDeviceInfo.h" #include "DOMMediaStream.h" #include "DecoderBenchmark.h" #include "ImageContainer.h" @@ -226,46 +225,36 @@ void MediaDecoder::SetVolume(double aVolume) { mVolume = aVolume; } -RefPtr MediaDecoder::SetSink(AudioDeviceInfo* aSinkDevice) { +RefPtr MediaDecoder::SetSink(AudioDeviceInfo* aSink) { MOZ_ASSERT(NS_IsMainThread()); AbstractThread::AutoEnter context(AbstractMainThread()); - mSinkDevice = aSinkDevice; - return GetStateMachine()->InvokeSetSink(aSinkDevice); + return GetStateMachine()->InvokeSetSink(aSink); } -void MediaDecoder::SetOutputCaptured(bool aCaptured) { +void MediaDecoder::AddOutputStream(DOMMediaStream* aStream, + SharedDummyTrack* aDummyStream) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); AbstractThread::AutoEnter context(AbstractMainThread()); - mOutputCaptured = aCaptured; -} - -void MediaDecoder::AddOutputTrack(RefPtr aTrack) { - MOZ_ASSERT(NS_IsMainThread()); - MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); - AbstractThread::AutoEnter context(AbstractMainThread()); - nsTArray> tracks = mOutputTracks; - tracks.AppendElement(std::move(aTrack)); - mOutputTracks = tracks; -} - -void MediaDecoder::RemoveOutputTrack( - const RefPtr& aTrack) { - MOZ_ASSERT(NS_IsMainThread()); - MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); - AbstractThread::AutoEnter context(AbstractMainThread()); - nsTArray> tracks = mOutputTracks; - if (tracks.RemoveElement(aTrack)) { - mOutputTracks = tracks; + mDecoderStateMachine->EnsureOutputStreamManager(aDummyStream); + if (mInfo) { + mDecoderStateMachine->EnsureOutputStreamManagerHasTracks(*mInfo); } + mDecoderStateMachine->AddOutputStream(aStream); } -void MediaDecoder::SetOutputTracksPrincipal( - const RefPtr& aPrincipal) { +void MediaDecoder::RemoveOutputStream(DOMMediaStream* aStream) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); AbstractThread::AutoEnter context(AbstractMainThread()); - mOutputPrincipal = MakePrincipalHandle(aPrincipal); + mDecoderStateMachine->RemoveOutputStream(aStream); +} + +void MediaDecoder::SetOutputStreamPrincipal(nsIPrincipal* aPrincipal) { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); + AbstractThread::AutoEnter context(AbstractMainThread()); + mDecoderStateMachine->SetOutputStreamPrincipal(aPrincipal); } double MediaDecoder::GetDuration() { @@ -311,10 +300,6 @@ MediaDecoder::MediaDecoder(MediaDecoderInit& aInit) INIT_CANONICAL(mVolume, aInit.mVolume), INIT_CANONICAL(mPreservesPitch, aInit.mPreservesPitch), INIT_CANONICAL(mLooping, aInit.mLooping), - INIT_CANONICAL(mSinkDevice, nullptr), - INIT_CANONICAL(mOutputCaptured, false), - INIT_CANONICAL(mOutputTracks, nsTArray>()), - INIT_CANONICAL(mOutputPrincipal, PRINCIPAL_HANDLE_NONE), INIT_CANONICAL(mPlayState, PLAY_STATE_LOADING), mSameOriginMedia(false), mVideoDecodingOberver( @@ -390,11 +375,14 @@ void MediaDecoder::Shutdown() { nsCOMPtr r = NS_NewRunnableFunction("MediaDecoder::Shutdown", [self]() { self->mVideoFrameContainer = nullptr; - self->ShutdownInternal(); + MediaShutdownManager::Instance().Unregister(self); }); mAbstractMainThread->Dispatch(r.forget()); } + // Ask the owner to remove its audio/video tracks. + GetOwner()->RemoveMediaTracks(); + ChangeState(PLAY_STATE_SHUTDOWN); mVideoDecodingOberver->UnregisterEvent(); mVideoDecodingOberver = nullptr; @@ -531,16 +519,11 @@ void MediaDecoder::OnStoreDecoderBenchmark(const VideoInfo& aInfo) { } } -void MediaDecoder::ShutdownInternal() { - MOZ_ASSERT(NS_IsMainThread()); - MediaShutdownManager::Instance().Unregister(this); -} - void MediaDecoder::FinishShutdown() { MOZ_ASSERT(NS_IsMainThread()); SetStateMachine(nullptr); mVideoFrameContainer = nullptr; - ShutdownInternal(); + MediaShutdownManager::Instance().Unregister(this); } nsresult MediaDecoder::InitializeStateMachine() { @@ -659,6 +642,7 @@ double MediaDecoder::GetCurrentTime() { void MediaDecoder::OnMetadataUpdate(TimedMetadata&& aMetadata) { MOZ_ASSERT(NS_IsMainThread()); AbstractThread::AutoEnter context(AbstractMainThread()); + GetOwner()->RemoveMediaTracks(); MetadataLoaded(MakeUnique(*aMetadata.mInfo), UniquePtr(std::move(aMetadata.mTags)), MediaDecoderEventVisibility::Observable); @@ -681,6 +665,8 @@ void MediaDecoder::MetadataLoaded( mMediaSeekableOnlyInBufferedRanges = aInfo->mMediaSeekableOnlyInBufferedRanges; mInfo = aInfo.release(); + GetOwner()->ConstructMediaTracks(mInfo); + mDecoderStateMachine->EnsureOutputStreamManagerHasTracks(*mInfo); // Make sure the element and the frame (if any) are told about // our new size. @@ -871,6 +857,12 @@ void MediaDecoder::ChangeState(PlayState aState) { DDLOG(DDLogCategory::Property, "play_state", ToPlayStateStr(aState)); } mPlayState = aState; + + if (mPlayState == PLAY_STATE_PLAYING) { + GetOwner()->ConstructMediaTracks(mInfo); + } else if (IsEnded()) { + GetOwner()->RemoveMediaTracks(); + } } bool MediaDecoder::IsLoopingBack(double aPrevPos, double aCurPos) const { diff --git a/dom/media/MediaDecoder.h b/dom/media/MediaDecoder.h index 01a4d6735fd3..7a445bc1a503 100644 --- a/dom/media/MediaDecoder.h +++ b/dom/media/MediaDecoder.h @@ -43,12 +43,12 @@ class MediaMemoryInfo; class AbstractThread; class DOMMediaStream; class DecoderBenchmark; -class ProcessedMediaTrack; class FrameStatistics; class VideoFrameContainer; class MediaFormatReader; class MediaDecoderStateMachine; struct MediaPlaybackEvent; +struct SharedDummyTrack; enum class Visibility : uint8_t; @@ -155,7 +155,7 @@ class MediaDecoder : public DecoderDoctorLifeLogger { void SetLooping(bool aLooping); // Set the given device as the output device. - RefPtr SetSink(AudioDeviceInfo* aSinkDevice); + RefPtr SetSink(AudioDeviceInfo* aSink); bool GetMinimizePreroll() const { return mMinimizePreroll; } @@ -166,23 +166,15 @@ class MediaDecoder : public DecoderDoctorLifeLogger { // replaying after the input as ended. In the latter case, the new source is // not connected to streams created by captureStreamUntilEnded. - // Turn output capturing of this decoder on or off. If it is on, the - // MediaDecoderStateMachine's media sink will only play after output tracks - // have been set. This is to ensure that it doesn't skip over any data - // while the owner has intended to capture the full output, thus missing to - // capture some of it. The owner of the MediaDecoder is responsible for adding - // output tracks in a timely fashion while the output is captured. - void SetOutputCaptured(bool aCaptured); - // Add an output track. All decoder output for the track's media type will be - // sent to the track. - // Note that only one audio track and one video track is supported by - // MediaDecoder at this time. Passing in more of one type, or passing in a - // type that metadata says we are not decoding, is an error. - void AddOutputTrack(RefPtr aTrack); - // Remove an output track added with AddOutputTrack. - void RemoveOutputTrack(const RefPtr& aTrack); - // Update the principal for any output tracks. - void SetOutputTracksPrincipal(const RefPtr& aPrincipal); + // Add an output stream. All decoder output will be sent to the stream. + // The stream is initially blocked. The decoder is responsible for unblocking + // it while it is playing back. + void AddOutputStream(DOMMediaStream* aStream, SharedDummyTrack* aDummyStream); + // Remove an output stream added with AddOutputStream. + void RemoveOutputStream(DOMMediaStream* aStream); + + // Update the principal for any output streams and their tracks. + void SetOutputStreamPrincipal(nsIPrincipal* aPrincipal); // Return the duration of the video in seconds. virtual double GetDuration(); @@ -403,11 +395,6 @@ class MediaDecoder : public DecoderDoctorLifeLogger { void SetStateMachineParameters(); - // Called when MediaDecoder shutdown is finished. Subclasses use this to clean - // up internal structures, and unregister potential shutdown blockers when - // they're done. - virtual void ShutdownInternal(); - bool IsShutdown() const; // Called to notify the decoder that the duration has changed. @@ -619,20 +606,6 @@ class MediaDecoder : public DecoderDoctorLifeLogger { Canonical mLooping; - // The device used with SetSink, or nullptr if no explicit device has been - // set. - Canonical> mSinkDevice; - - // Whether this MediaDecoder's output is captured. When captured, all decoded - // data must be played out through mOutputTracks. - Canonical mOutputCaptured; - - // Tracks that, if set, will get data routed through them. - Canonical>> mOutputTracks; - - // PrincipalHandle to be used when feeding data into mOutputTracks. - Canonical mOutputPrincipal; - // Media duration set explicitly by JS. At present, this is only ever present // for MSE. Maybe mExplicitDuration; @@ -665,19 +638,6 @@ class MediaDecoder : public DecoderDoctorLifeLogger { return &mPreservesPitch; } AbstractCanonical* CanonicalLooping() { return &mLooping; } - AbstractCanonical>* CanonicalSinkDevice() { - return &mSinkDevice; - } - AbstractCanonical* CanonicalOutputCaptured() { - return &mOutputCaptured; - } - AbstractCanonical>>* - CanonicalOutputTracks() { - return &mOutputTracks; - } - AbstractCanonical* CanonicalOutputPrincipal() { - return &mOutputPrincipal; - } AbstractCanonical* CanonicalPlayState() { return &mPlayState; } private: diff --git a/dom/media/MediaDecoderOwner.h b/dom/media/MediaDecoderOwner.h index b6fe58c37783..9926fd4e6450 100644 --- a/dom/media/MediaDecoderOwner.h +++ b/dom/media/MediaDecoderOwner.h @@ -139,6 +139,14 @@ class MediaDecoderOwner { virtual void DispatchEncrypted(const nsTArray& aInitData, const nsAString& aInitDataType) = 0; + // Called by the media decoder to create audio/video tracks and add to its + // owner's track list. + virtual void ConstructMediaTracks(const MediaInfo* aInfo) = 0; + + // Called by the media decoder to removes all audio/video tracks from its + // owner's track list. + virtual void RemoveMediaTracks() = 0; + // Notified by the decoder that a decryption key is required before emitting // further output. virtual void NotifyWaitingForKey() {} diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index 2ee0a8de0c9a..ef89062b6189 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -11,6 +11,7 @@ #include "mediasink/AudioSink.h" #include "mediasink/AudioSinkWrapper.h" #include "mediasink/DecodedStream.h" +#include "mediasink/OutputStreamManager.h" #include "mediasink/VideoSink.h" #include "mozilla/Logging.h" #include "mozilla/MathAlgorithms.h" @@ -2591,10 +2592,6 @@ RefPtr MediaDecoderStateMachine::ShutdownState::Enter() { master->mVolume.DisconnectIfConnected(); master->mPreservesPitch.DisconnectIfConnected(); master->mLooping.DisconnectIfConnected(); - master->mSinkDevice.DisconnectIfConnected(); - master->mOutputCaptured.DisconnectIfConnected(); - master->mOutputTracks.DisconnectIfConnected(); - master->mOutputPrincipal.DisconnectIfConnected(); master->mDuration.DisconnectAll(); master->mCurrentPosition.DisconnectAll(); @@ -2630,10 +2627,12 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mReader(new ReaderProxy(mTaskQueue, aReader)), mPlaybackRate(1.0), mAmpleAudioThreshold(detail::AMPLE_AUDIO_THRESHOLD), + mAudioCaptured(false), mMinimizePreroll(aDecoder->GetMinimizePreroll()), mSentFirstFrameLoadedEvent(false), mVideoDecodeSuspended(false), mVideoDecodeSuspendTimer(mTaskQueue), + mOutputStreamManager(nullptr), mVideoDecodeMode(VideoDecodeMode::Normal), mIsMSE(aDecoder->IsMSE()), mSeamlessLoopingAllowed(false), @@ -2642,16 +2641,10 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, INIT_MIRROR(mVolume, 1.0), INIT_MIRROR(mPreservesPitch, true), INIT_MIRROR(mLooping, false), - INIT_MIRROR(mSinkDevice, nullptr), - INIT_MIRROR(mOutputCaptured, false), - INIT_MIRROR(mOutputTracks, nsTArray>()), - INIT_MIRROR(mOutputPrincipal, PRINCIPAL_HANDLE_NONE), - INIT_CANONICAL(mCanonicalOutputTracks, - nsTArray>()), - INIT_CANONICAL(mCanonicalOutputPrincipal, PRINCIPAL_HANDLE_NONE), INIT_CANONICAL(mDuration, NullableTimeUnit()), INIT_CANONICAL(mCurrentPosition, TimeUnit::Zero()), - INIT_CANONICAL(mIsAudioDataAudible, false) { + INIT_CANONICAL(mIsAudioDataAudible, false), + mSetSinkRequestsCount(0) { MOZ_COUNT_CTOR(MediaDecoderStateMachine); NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); @@ -2678,10 +2671,6 @@ void MediaDecoderStateMachine::InitializationTask(MediaDecoder* aDecoder) { mVolume.Connect(aDecoder->CanonicalVolume()); mPreservesPitch.Connect(aDecoder->CanonicalPreservesPitch()); mLooping.Connect(aDecoder->CanonicalLooping()); - mSinkDevice.Connect(aDecoder->CanonicalSinkDevice()); - mOutputCaptured.Connect(aDecoder->CanonicalOutputCaptured()); - mOutputTracks.Connect(aDecoder->CanonicalOutputTracks()); - mOutputPrincipal.Connect(aDecoder->CanonicalOutputPrincipal()); // Initialize watchers. mWatchManager.Watch(mBuffered, @@ -2691,16 +2680,6 @@ void MediaDecoderStateMachine::InitializationTask(MediaDecoder* aDecoder) { &MediaDecoderStateMachine::PreservesPitchChanged); mWatchManager.Watch(mPlayState, &MediaDecoderStateMachine::PlayStateChanged); mWatchManager.Watch(mLooping, &MediaDecoderStateMachine::LoopingChanged); - mWatchManager.Watch(mOutputCaptured, - &MediaDecoderStateMachine::UpdateOutputCaptured); - mWatchManager.Watch(mOutputTracks, - &MediaDecoderStateMachine::UpdateOutputCaptured); - mWatchManager.Watch(mOutputTracks, - &MediaDecoderStateMachine::OutputTracksChanged); - mWatchManager.Watch(mOutputPrincipal, - &MediaDecoderStateMachine::OutputPrincipalChanged); - - mMediaSink = CreateMediaSink(); MOZ_ASSERT(!mStateObj); auto* s = new DecodeMetadataState(this); @@ -2718,24 +2697,23 @@ MediaSink* MediaDecoderStateMachine::CreateAudioSink() { MOZ_ASSERT(self->OnTaskQueue()); AudioSink* audioSink = new AudioSink(self->mTaskQueue, self->mAudioQueue, self->GetMediaTime(), - self->Info().mAudio, self->mSinkDevice.Ref()); + self->Info().mAudio); self->mAudibleListener = audioSink->AudibleEvent().Connect( self->mTaskQueue, self.get(), &MediaDecoderStateMachine::AudioAudibleChanged); return audioSink; }; - return new AudioSinkWrapper(mTaskQueue, mAudioQueue, audioSinkCreator, - mVolume, mPlaybackRate, mPreservesPitch); + return new AudioSinkWrapper(mTaskQueue, mAudioQueue, audioSinkCreator); } -already_AddRefed MediaDecoderStateMachine::CreateMediaSink() { - MOZ_ASSERT(OnTaskQueue()); +already_AddRefed MediaDecoderStateMachine::CreateMediaSink( + bool aAudioCaptured, OutputStreamManager* aManager) { + MOZ_ASSERT_IF(aAudioCaptured, aManager); RefPtr audioSink = - mOutputCaptured - ? new DecodedStream(this, mOutputTracks, mVolume, mPlaybackRate, - mPreservesPitch, mAudioQueue, mVideoQueue) - : CreateAudioSink(); + aAudioCaptured ? new DecodedStream(mTaskQueue, mAbstractMainThread, + mAudioQueue, mVideoQueue, aManager) + : CreateAudioSink(); RefPtr mediaSink = new VideoSink(mTaskQueue, audioSink, mVideoQueue, mVideoFrameContainer, @@ -2825,6 +2803,8 @@ nsresult MediaDecoderStateMachine::Init(MediaDecoder* aDecoder) { mOnMediaNotSeekable = mReader->OnMediaNotSeekable().Connect( OwnerThread(), this, &MediaDecoderStateMachine::SetMediaNotSeekable); + mMediaSink = CreateMediaSink(mAudioCaptured, mOutputStreamManager); + nsresult rv = mReader->Init(); NS_ENSURE_SUCCESS(rv, rv); @@ -3360,6 +3340,9 @@ void MediaDecoderStateMachine::FinishDecodeFirstFrame() { RefPtr MediaDecoderStateMachine::BeginShutdown() { MOZ_ASSERT(NS_IsMainThread()); + if (mOutputStreamManager) { + mOutputStreamManager->Disconnect(); + } return InvokeAsync(OwnerThread(), this, __func__, &MediaDecoderStateMachine::Shutdown); } @@ -3448,7 +3431,7 @@ void MediaDecoderStateMachine::UpdatePlaybackPositionPeriodically() { } } // Note we have to update playback position before releasing the monitor. - // Otherwise, MediaDecoder::AddOutputTrack could kick in when we are outside + // Otherwise, MediaDecoder::AddOutputStream could kick in when we are outside // the monitor and get a staled value from GetCurrentTimeUs() which hits the // assertion in GetClock(). @@ -3534,75 +3517,47 @@ void MediaDecoderStateMachine::LoopingChanged() { } } -void MediaDecoderStateMachine::UpdateOutputCaptured() { - MOZ_ASSERT(OnTaskQueue()); - - // Reset these flags so they are consistent with the status of the sink. - // TODO: Move these flags into MediaSink to improve cohesion so we don't need - // to reset these flags when switching MediaSinks. - mAudioCompleted = false; - mVideoCompleted = false; - - // Stop and shut down the existing sink. - StopMediaSink(); - mMediaSink->Shutdown(); - - // Create a new sink according to whether output is captured. - mMediaSink = CreateMediaSink(); - - // Don't buffer as much when audio is captured because we don't need to worry - // about high latency audio devices. - mAmpleAudioThreshold = mOutputCaptured ? detail::AMPLE_AUDIO_THRESHOLD / 2 - : detail::AMPLE_AUDIO_THRESHOLD; - - mStateObj->HandleAudioCaptured(); -} - -void MediaDecoderStateMachine::OutputTracksChanged() { - MOZ_ASSERT(OnTaskQueue()); - LOG("OutputTracksChanged, tracks=%zu", mOutputTracks.Ref().Length()); - mCanonicalOutputTracks = mOutputTracks; -} - -void MediaDecoderStateMachine::OutputPrincipalChanged() { - MOZ_ASSERT(OnTaskQueue()); - mCanonicalOutputPrincipal = mOutputPrincipal; -} - RefPtr MediaDecoderStateMachine::InvokeSetSink( RefPtr aSink) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(aSink); + Unused << ++mSetSinkRequestsCount; return InvokeAsync(OwnerThread(), this, __func__, &MediaDecoderStateMachine::SetSink, aSink); } RefPtr MediaDecoderStateMachine::SetSink( - RefPtr aSinkDevice) { + RefPtr aSink) { MOZ_ASSERT(OnTaskQueue()); - if (mOutputCaptured) { + if (mAudioCaptured) { // Not supported yet. return GenericPromise::CreateAndReject(NS_ERROR_ABORT, __func__); } - if (mSinkDevice.Ref() != aSinkDevice) { - // A new sink was set before this ran. - return GenericPromise::CreateAndResolve(IsPlaying(), __func__); + // Backup current playback parameters. + bool wasPlaying = mMediaSink->IsPlaying(); + + if (--mSetSinkRequestsCount > 0) { + MOZ_ASSERT(mSetSinkRequestsCount > 0); + return GenericPromise::CreateAndResolve(wasPlaying, __func__); } - if (mMediaSink->AudioDevice() == aSinkDevice) { - // The sink has not changed. - return GenericPromise::CreateAndResolve(IsPlaying(), __func__); - } + MediaSink::PlaybackParams params = mMediaSink->GetPlaybackParams(); + params.mSink = std::move(aSink); - const bool wasPlaying = IsPlaying(); + if (!mMediaSink->IsStarted()) { + mMediaSink->SetPlaybackParams(params); + return GenericPromise::CreateAndResolve(false, __func__); + } // Stop and shutdown the existing sink. StopMediaSink(); mMediaSink->Shutdown(); // Create a new sink according to whether audio is captured. - mMediaSink = CreateMediaSink(); + mMediaSink = CreateMediaSink(false); + // Restore playback parameters. + mMediaSink->SetPlaybackParams(params); // Start the new sink if (wasPlaying) { nsresult rv = StartMediaSink(); @@ -3701,6 +3656,43 @@ void MediaDecoderStateMachine::OnMediaSinkAudioError(nsresult aResult) { DecodeError(MediaResult(NS_ERROR_DOM_MEDIA_MEDIASINK_ERR, __func__)); } +void MediaDecoderStateMachine::SetAudioCaptured(bool aCaptured, + OutputStreamManager* aManager) { + MOZ_ASSERT(OnTaskQueue()); + + if (aCaptured == mAudioCaptured) { + return; + } + + // Rest these flags so they are consistent with the status of the sink. + // TODO: Move these flags into MediaSink to improve cohesion so we don't need + // to reset these flags when switching MediaSinks. + mAudioCompleted = false; + mVideoCompleted = false; + + // Backup current playback parameters. + MediaSink::PlaybackParams params = mMediaSink->GetPlaybackParams(); + + // Stop and shut down the existing sink. + StopMediaSink(); + mMediaSink->Shutdown(); + + // Create a new sink according to whether audio is captured. + mMediaSink = CreateMediaSink(aCaptured, aManager); + + // Restore playback parameters. + mMediaSink->SetPlaybackParams(params); + + mAudioCaptured = aCaptured; + + // Don't buffer as much when audio is captured because we don't need to worry + // about high latency audio devices. + mAmpleAudioThreshold = mAudioCaptured ? detail::AMPLE_AUDIO_THRESHOLD / 2 + : detail::AMPLE_AUDIO_THRESHOLD; + + mStateObj->HandleAudioCaptured(); +} + uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const { MOZ_ASSERT(OnTaskQueue()); return mReader->VideoIsHardwareAccelerated() @@ -3744,6 +3736,86 @@ RefPtr MediaDecoderStateMachine::RequestDebugInfo( return p.forget(); } +void MediaDecoderStateMachine::SetOutputStreamPrincipal( + nsIPrincipal* aPrincipal) { + MOZ_ASSERT(NS_IsMainThread()); + mOutputStreamPrincipal = aPrincipal; + if (mOutputStreamManager) { + mOutputStreamManager->SetPrincipal(mOutputStreamPrincipal); + } +} + +void MediaDecoderStateMachine::AddOutputStream(DOMMediaStream* aStream) { + MOZ_ASSERT(NS_IsMainThread()); + LOG("AddOutputStream aStream=%p!", aStream); + mOutputStreamManager->Add(aStream); + nsCOMPtr r = + NS_NewRunnableFunction("MediaDecoderStateMachine::SetAudioCaptured", + [self = RefPtr(this), + manager = mOutputStreamManager]() { + self->SetAudioCaptured(true, manager); + }); + nsresult rv = OwnerThread()->Dispatch(r.forget()); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; +} + +void MediaDecoderStateMachine::RemoveOutputStream(DOMMediaStream* aStream) { + MOZ_ASSERT(NS_IsMainThread()); + LOG("RemoveOutputStream=%p!", aStream); + mOutputStreamManager->Remove(aStream); + if (mOutputStreamManager->IsEmpty()) { + mOutputStreamManager->Disconnect(); + mOutputStreamManager = nullptr; + nsCOMPtr r = NS_NewRunnableFunction( + "MediaDecoderStateMachine::SetAudioCaptured", + [self = RefPtr(this)]() { + self->SetAudioCaptured(false); + }); + nsresult rv = OwnerThread()->Dispatch(r.forget()); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + } +} + +void MediaDecoderStateMachine::EnsureOutputStreamManager( + SharedDummyTrack* aDummyStream) { + MOZ_ASSERT(NS_IsMainThread()); + if (mOutputStreamManager) { + return; + } + mOutputStreamManager = new OutputStreamManager( + aDummyStream, mOutputStreamPrincipal, mAbstractMainThread); +} + +void MediaDecoderStateMachine::EnsureOutputStreamManagerHasTracks( + const MediaInfo& aLoadedInfo) { + MOZ_ASSERT(NS_IsMainThread()); + if (!mOutputStreamManager) { + return; + } + if ((!aLoadedInfo.HasAudio() || + mOutputStreamManager->HasTrackType(MediaSegment::AUDIO)) && + (!aLoadedInfo.HasVideo() || + mOutputStreamManager->HasTrackType(MediaSegment::VIDEO))) { + return; + } + if (aLoadedInfo.HasAudio()) { + MOZ_ASSERT(!mOutputStreamManager->HasTrackType(MediaSegment::AUDIO)); + RefPtr dummy = + mOutputStreamManager->AddTrack(MediaSegment::AUDIO); + LOG("Pre-created audio track with underlying track %p", dummy.get()); + Unused << dummy; + } + if (aLoadedInfo.HasVideo()) { + MOZ_ASSERT(!mOutputStreamManager->HasTrackType(MediaSegment::VIDEO)); + RefPtr dummy = + mOutputStreamManager->AddTrack(MediaSegment::VIDEO); + LOG("Pre-created video track with underlying track %p", dummy.get()); + Unused << dummy; + } +} + class VideoQueueMemoryFunctor : public nsDequeFunctor { public: VideoQueueMemoryFunctor() : mSize(0) {} diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h index c75cc115dbf1..414a379e9d79 100644 --- a/dom/media/MediaDecoderStateMachine.h +++ b/dom/media/MediaDecoderStateMachine.h @@ -106,6 +106,7 @@ class AbstractThread; class AudioSegment; class DecodedStream; class DOMMediaStream; +class OutputStreamManager; class ReaderProxy; class TaskQueue; @@ -185,6 +186,19 @@ class MediaDecoderStateMachine RefPtr RequestDebugInfo( dom::MediaDecoderStateMachineDebugInfo& aInfo); + void SetOutputStreamPrincipal(nsIPrincipal* aPrincipal); + // If an OutputStreamManager does not exist, one will be created. + void EnsureOutputStreamManager(SharedDummyTrack* aDummyStream); + // If an OutputStreamManager exists, tracks matching aLoadedInfo will be + // created unless they already exist in the manager. + void EnsureOutputStreamManagerHasTracks(const MediaInfo& aLoadedInfo); + // Add an output stream to the output stream manager. The manager must have + // been created through EnsureOutputStreamManager() before this. + void AddOutputStream(DOMMediaStream* aStream); + // Remove an output stream added with AddOutputStream. If the last output + // stream was removed, we will also tear down the OutputStreamManager. + void RemoveOutputStream(DOMMediaStream* aStream); + // Seeks to the decoder to aTarget asynchronously. RefPtr InvokeSeek(const SeekTarget& aTarget); @@ -302,6 +316,11 @@ class MediaDecoderStateMachine // constructor immediately after the task queue is created. void InitializationTask(MediaDecoder* aDecoder); + // Sets the audio-captured state and recreates the media sink if needed. + // A manager must be passed in if setting the audio-captured state to true. + void SetAudioCaptured(bool aCaptured, + OutputStreamManager* aManager = nullptr); + RefPtr Seek(const SeekTarget& aTarget); RefPtr Shutdown(); @@ -375,9 +394,6 @@ class MediaDecoderStateMachine void SetPlaybackRate(double aPlaybackRate); void PreservesPitchChanged(); void LoopingChanged(); - void UpdateOutputCaptured(); - void OutputTracksChanged(); - void OutputPrincipalChanged(); MediaQueue& AudioQueue() { return mAudioQueue; } MediaQueue& VideoQueue() { return mVideoQueue; } @@ -422,9 +438,10 @@ class MediaDecoderStateMachine MediaSink* CreateAudioSink(); - // Always create mediasink which contains an AudioSink or DecodedStream - // inside. - already_AddRefed CreateMediaSink(); + // Always create mediasink which contains an AudioSink or StreamSink inside. + // A manager must be passed in if aAudioCaptured is true. + already_AddRefed CreateMediaSink( + bool aAudioCaptured, OutputStreamManager* aManager = nullptr); // Stops the media sink and shut it down. // The decoder monitor must be held with exactly one lock count. @@ -609,6 +626,11 @@ class MediaDecoderStateMachine bool mIsLiveStream = false; + // True if we shouldn't play our audio (but still write it to any capturing + // streams). When this is true, the audio thread will never start again after + // it has stopped. + bool mAudioCaptured; + // True if all audio frames are already rendered. bool mAudioCompleted = false; @@ -650,6 +672,13 @@ class MediaDecoderStateMachine // Track enabling video decode suspension via timer DelayedScheduler mVideoDecodeSuspendTimer; + // Data about MediaStreams that are being fed by the decoder. + // Main thread only. + RefPtr mOutputStreamManager; + + // Principal used by output streams. Main thread only. + nsCOMPtr mOutputStreamPrincipal; + // Track the current video decode mode. VideoDecodeMode mVideoDecodeMode; @@ -704,23 +733,6 @@ class MediaDecoderStateMachine // upon reaching the end. Mirror mLooping; - // The device used with SetSink, or nullptr if no explicit device has been - // set. - Mirror> mSinkDevice; - - // Whether all output should be captured into mOutputTracks. While true, the - // media sink will only play if there are output tracks. - Mirror mOutputCaptured; - - // Tracks to capture data into. - Mirror>> mOutputTracks; - - // PrincipalHandle to feed with data captured into mOutputTracks. - Mirror mOutputPrincipal; - - Canonical>> mCanonicalOutputTracks; - Canonical mCanonicalOutputPrincipal; - // Duration of the media. This is guaranteed to be non-null after we finish // decoding the first frame. Canonical mDuration; @@ -733,16 +745,12 @@ class MediaDecoderStateMachine // Used to distinguish whether the audio is producing sound. Canonical mIsAudioDataAudible; + // Used to count the number of pending requests to set a new sink. + Atomic mSetSinkRequestsCount; + public: AbstractCanonical* CanonicalBuffered() const; - AbstractCanonical>>* - CanonicalOutputTracks() { - return &mCanonicalOutputTracks; - } - AbstractCanonical* CanonicalOutputPrincipal() { - return &mCanonicalOutputPrincipal; - } AbstractCanonical* CanonicalDuration() { return &mDuration; } diff --git a/dom/media/MediaResource.h b/dom/media/MediaResource.h index 280953e679d8..e80632bbfc79 100644 --- a/dom/media/MediaResource.h +++ b/dom/media/MediaResource.h @@ -60,11 +60,8 @@ class MediaResource : public DecoderDoctorLifeLogger { // Close the resource, stop any listeners, channels, etc. // Cancels any currently blocking Read request and forces that request to - // return an error. This must be called (and resolve) before the MediaResource - // is deleted. - virtual RefPtr Close() { - return GenericPromise::CreateAndResolve(true, __func__); - } + // return an error. + virtual nsresult Close() { return NS_OK; } // These methods are called off the main thread. // Read up to aCount bytes from the stream. The read starts at diff --git a/dom/media/MediaStreamTrack.h b/dom/media/MediaStreamTrack.h index 73e002ef2229..c2e610d5257d 100644 --- a/dom/media/MediaStreamTrack.h +++ b/dom/media/MediaStreamTrack.h @@ -308,7 +308,7 @@ class MediaStreamTrackSource : public nsISupports { } // Principal identifying who may access the contents of this source. - RefPtr mPrincipal; + nsCOMPtr mPrincipal; // Currently registered sinks. nsTArray> mSinks; diff --git a/dom/media/mediasink/AudioSink.cpp b/dom/media/mediasink/AudioSink.cpp index 41f613cc30d7..7854cf223849 100644 --- a/dom/media/mediasink/AudioSink.cpp +++ b/dom/media/mediasink/AudioSink.cpp @@ -6,7 +6,6 @@ #include "AudioSink.h" #include "AudioConverter.h" -#include "AudioDeviceInfo.h" #include "MediaQueue.h" #include "VideoUtils.h" #include "mozilla/CheckedInt.h" @@ -35,11 +34,9 @@ using media::TimeUnit; AudioSink::AudioSink(AbstractThread* aThread, MediaQueue& aAudioQueue, - const TimeUnit& aStartTime, const AudioInfo& aInfo, - AudioDeviceInfo* aAudioDevice) + const TimeUnit& aStartTime, const AudioInfo& aInfo) : mStartTime(aStartTime), mInfo(aInfo), - mAudioDevice(aAudioDevice), mPlaying(true), mMonitor("AudioSink"), mWritten(0), @@ -186,7 +183,7 @@ nsresult AudioSink::InitializeAudioStream(const PlaybackParams& aParams) { // StaticPrefs::accessibility_monoaudio_enable() or // StaticPrefs::media_forcestereo_enabled() is applied. nsresult rv = mAudioStream->Init(mOutputChannels, channelMap, mOutputRate, - mAudioDevice); + aParams.mSink); if (NS_FAILED(rv)) { mAudioStream->Shutdown(); mAudioStream = nullptr; diff --git a/dom/media/mediasink/AudioSink.h b/dom/media/mediasink/AudioSink.h index 77604d11e958..d1b5449fe61d 100644 --- a/dom/media/mediasink/AudioSink.h +++ b/dom/media/mediasink/AudioSink.h @@ -23,20 +23,11 @@ namespace mozilla { class AudioConverter; class AudioSink : private AudioStream::DataSource { - public: - struct PlaybackParams { - PlaybackParams(double aVolume, double aPlaybackRate, bool aPreservesPitch) - : mVolume(aVolume), - mPlaybackRate(aPlaybackRate), - mPreservesPitch(aPreservesPitch) {} - double mVolume; - double mPlaybackRate; - bool mPreservesPitch; - }; + using PlaybackParams = MediaSink::PlaybackParams; + public: AudioSink(AbstractThread* aThread, MediaQueue& aAudioQueue, - const media::TimeUnit& aStartTime, const AudioInfo& aInfo, - AudioDeviceInfo* aAudioDevice); + const media::TimeUnit& aStartTime, const AudioInfo& aInfo); ~AudioSink(); @@ -68,8 +59,6 @@ class AudioSink : private AudioStream::DataSource { void GetDebugInfo(dom::MediaSinkDebugInfo& aInfo); - const RefPtr& AudioDevice() { return mAudioDevice; } - private: // Allocate and initialize mAudioStream. Returns NS_OK on success. nsresult InitializeAudioStream(const PlaybackParams& aParams); @@ -98,10 +87,6 @@ class AudioSink : private AudioStream::DataSource { const AudioInfo mInfo; - // The output device this AudioSink is playing data to. The system's default - // device is used if this is null. - const RefPtr mAudioDevice; - // Used on the task queue of MDSM only. bool mPlaying; diff --git a/dom/media/mediasink/AudioSinkWrapper.cpp b/dom/media/mediasink/AudioSinkWrapper.cpp index f29d9b1f2382..817e132ccf6e 100644 --- a/dom/media/mediasink/AudioSinkWrapper.cpp +++ b/dom/media/mediasink/AudioSinkWrapper.cpp @@ -21,6 +21,21 @@ void AudioSinkWrapper::Shutdown() { mCreator = nullptr; } +const MediaSink::PlaybackParams& AudioSinkWrapper::GetPlaybackParams() const { + AssertOwnerThread(); + return mParams; +} + +void AudioSinkWrapper::SetPlaybackParams(const PlaybackParams& aParams) { + AssertOwnerThread(); + if (mAudioSink) { + mAudioSink->SetVolume(aParams.mVolume); + mAudioSink->SetPlaybackRate(aParams.mPlaybackRate); + mAudioSink->SetPreservesPitch(aParams.mPreservesPitch); + } + mParams = aParams; +} + RefPtr AudioSinkWrapper::OnEnded(TrackType aType) { AssertOwnerThread(); MOZ_ASSERT(mIsStarted, "Must be called after playback starts."); @@ -139,11 +154,6 @@ void AudioSinkWrapper::SetPlaying(bool aPlaying) { } } -double AudioSinkWrapper::PlaybackRate() const { - AssertOwnerThread(); - return mParams.mPlaybackRate; -} - nsresult AudioSinkWrapper::Start(const TimeUnit& aStartTime, const MediaInfo& aInfo) { AssertOwnerThread(); diff --git a/dom/media/mediasink/AudioSinkWrapper.h b/dom/media/mediasink/AudioSinkWrapper.h index 93cc73e58784..51fcead89cce 100644 --- a/dom/media/mediasink/AudioSinkWrapper.h +++ b/dom/media/mediasink/AudioSinkWrapper.h @@ -24,8 +24,6 @@ class MediaQueue; * A wrapper around AudioSink to provide the interface of MediaSink. */ class AudioSinkWrapper : public MediaSink { - using PlaybackParams = AudioSink::PlaybackParams; - // An AudioSink factory. class Creator { public: @@ -48,18 +46,19 @@ class AudioSinkWrapper : public MediaSink { template AudioSinkWrapper(AbstractThread* aOwnerThread, const MediaQueue& aAudioQueue, - const Function& aFunc, double aVolume, double aPlaybackRate, - bool aPreservesPitch) + const Function& aFunc) : mOwnerThread(aOwnerThread), mCreator(new CreatorImpl(aFunc)), mIsStarted(false), - mParams(aVolume, aPlaybackRate, aPreservesPitch), // Give an invalid value to facilitate debug if used before playback // starts. mPlayDuration(media::TimeUnit::Invalid()), mAudioEnded(true), mAudioQueue(aAudioQueue) {} + const PlaybackParams& GetPlaybackParams() const override; + void SetPlaybackParams(const PlaybackParams& aParams) override; + RefPtr OnEnded(TrackType aType) override; media::TimeUnit GetEndTime(TrackType aType) const override; media::TimeUnit GetPosition(TimeStamp* aTimeStamp = nullptr) const override; @@ -70,8 +69,6 @@ class AudioSinkWrapper : public MediaSink { void SetPreservesPitch(bool aPreservesPitch) override; void SetPlaying(bool aPlaying) override; - double PlaybackRate() const override; - nsresult Start(const media::TimeUnit& aStartTime, const MediaInfo& aInfo) override; void Stop() override; diff --git a/dom/media/mediasink/DecodedStream.cpp b/dom/media/mediasink/DecodedStream.cpp index ce137b683f4d..08737553c581 100644 --- a/dom/media/mediasink/DecodedStream.cpp +++ b/dom/media/mediasink/DecodedStream.cpp @@ -7,10 +7,10 @@ #include "DecodedStream.h" #include "AudioSegment.h" #include "MediaData.h" -#include "MediaDecoderStateMachine.h" #include "MediaQueue.h" #include "MediaTrackGraph.h" #include "MediaTrackListener.h" +#include "OutputStreamManager.h" #include "SharedBuffer.h" #include "VideoSegment.h" #include "VideoUtils.h" @@ -54,32 +54,34 @@ class DecodedStreamGraphListener { NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DecodedStreamGraphListener) public: DecodedStreamGraphListener( - SourceMediaTrack* aAudioTrack, + SourceMediaTrack* aAudioStream, MozPromiseHolder&& aAudioEndedHolder, - SourceMediaTrack* aVideoTrack, - MozPromiseHolder&& aVideoEndedHolder) + SourceMediaTrack* aVideoStream, + MozPromiseHolder&& aVideoEndedHolder, + AbstractThread* aMainThread) : mAudioTrackListener( - aAudioTrack - ? MakeRefPtr(this, aAudioTrack) + aAudioStream + ? MakeRefPtr(this, aAudioStream) : nullptr), mAudioEndedHolder(std::move(aAudioEndedHolder)), mVideoTrackListener( - aVideoTrack - ? MakeRefPtr(this, aVideoTrack) + aVideoStream + ? MakeRefPtr(this, aVideoStream) : nullptr), mVideoEndedHolder(std::move(aVideoEndedHolder)), - mAudioTrack(aAudioTrack), - mVideoTrack(aVideoTrack) { + mAudioStream(aAudioStream), + mVideoStream(aVideoStream), + mAbstractMainThread(aMainThread) { MOZ_ASSERT(NS_IsMainThread()); if (mAudioTrackListener) { - mAudioTrack->AddListener(mAudioTrackListener); + mAudioStream->AddListener(mAudioTrackListener); } else { mAudioEnded = true; mAudioEndedHolder.ResolveIfExists(true, __func__); } if (mVideoTrackListener) { - mVideoTrack->AddListener(mVideoTrackListener); + mVideoStream->AddListener(mVideoTrackListener); } else { mVideoEnded = true; mVideoEndedHolder.ResolveIfExists(true, __func__); @@ -87,30 +89,30 @@ class DecodedStreamGraphListener { } void NotifyOutput(SourceMediaTrack* aTrack, TrackTime aCurrentTrackTime) { - if (aTrack == mAudioTrack) { + if (aTrack == mAudioStream) { if (aCurrentTrackTime >= mAudioEnd) { - mAudioTrack->End(); + mAudioStream->End(); } - } else if (aTrack == mVideoTrack) { + } else if (aTrack == mVideoStream) { if (aCurrentTrackTime >= mVideoEnd) { - mVideoTrack->End(); + mVideoStream->End(); } } else { MOZ_CRASH("Unexpected source track"); } - if (aTrack != mAudioTrack && mAudioTrack && !mAudioEnded) { + if (aTrack != mAudioStream && mAudioStream && !mAudioEnded) { // Only audio playout drives the clock forward, if present and live. return; } - MOZ_ASSERT_IF(aTrack == mAudioTrack, !mAudioEnded); - MOZ_ASSERT_IF(aTrack == mVideoTrack, !mVideoEnded); + MOZ_ASSERT_IF(aTrack == mAudioStream, !mAudioEnded); + MOZ_ASSERT_IF(aTrack == mVideoStream, !mVideoEnded); mOnOutput.Notify(aTrack->TrackTimeToMicroseconds(aCurrentTrackTime)); } void NotifyEnded(SourceMediaTrack* aTrack) { - if (aTrack == mAudioTrack) { + if (aTrack == mAudioStream) { mAudioEnded = true; - } else if (aTrack == mVideoTrack) { + } else if (aTrack == mVideoStream) { mVideoEnded = true; } else { MOZ_CRASH("Unexpected source track"); @@ -143,9 +145,9 @@ class DecodedStreamGraphListener { * Callable from any thread. */ void EndTrackAt(SourceMediaTrack* aTrack, TrackTime aEnd) { - if (aTrack == mAudioTrack) { + if (aTrack == mAudioStream) { mAudioEnd = aEnd; - } else if (aTrack == mVideoTrack) { + } else if (aTrack == mVideoStream) { mVideoEnd = aEnd; } else { MOZ_CRASH("Unexpected source track"); @@ -154,9 +156,9 @@ class DecodedStreamGraphListener { void DoNotifyTrackEnded(SourceMediaTrack* aTrack) { MOZ_ASSERT(NS_IsMainThread()); - if (aTrack == mAudioTrack) { + if (aTrack == mAudioStream) { mAudioEndedHolder.ResolveIfExists(true, __func__); - } else if (aTrack == mVideoTrack) { + } else if (aTrack == mVideoStream) { mVideoEndedHolder.ResolveIfExists(true, __func__); } else { MOZ_CRASH("Unexpected source track"); @@ -166,16 +168,16 @@ class DecodedStreamGraphListener { void Forget() { MOZ_ASSERT(NS_IsMainThread()); - if (mAudioTrackListener && !mAudioTrack->IsDestroyed()) { - mAudioTrack->End(); - mAudioTrack->RemoveListener(mAudioTrackListener); + if (mAudioTrackListener && !mAudioStream->IsDestroyed()) { + mAudioStream->End(); + mAudioStream->RemoveListener(mAudioTrackListener); } mAudioTrackListener = nullptr; mAudioEndedHolder.ResolveIfExists(false, __func__); - if (mVideoTrackListener && !mVideoTrack->IsDestroyed()) { - mVideoTrack->End(); - mVideoTrack->RemoveListener(mVideoTrackListener); + if (mVideoTrackListener && !mVideoStream->IsDestroyed()) { + mVideoStream->End(); + mVideoStream->RemoveListener(mVideoTrackListener); } mVideoTrackListener = nullptr; mVideoEndedHolder.ResolveIfExists(false, __func__); @@ -202,10 +204,11 @@ class DecodedStreamGraphListener { bool mVideoEnded = false; // Any thread. - const RefPtr mAudioTrack; - const RefPtr mVideoTrack; + const RefPtr mAudioStream; + const RefPtr mVideoStream; Atomic mAudioEnd{TRACK_TIME_MAX}; Atomic mVideoEnd{TRACK_TIME_MAX}; + const RefPtr mAbstractMainThread; }; DecodedStreamTrackListener::DecodedStreamTrackListener( @@ -223,7 +226,7 @@ void DecodedStreamTrackListener::NotifyEnded(MediaTrackGraph* aGraph) { /** * All MediaStream-related data is protected by the decoder's monitor. We have - * at most one DecodedStreamData per MediaDecoder. XXX Its tracks are used as + * at most one DecodedStreamData per MediaDecoder. Its tracks are used as * inputs for all output tracks created by OutputStreamManager after calls to * captureStream/UntilEnded. Seeking creates new source tracks, as does * replaying after the input as ended. In the latter case, the new sources are @@ -232,11 +235,12 @@ void DecodedStreamTrackListener::NotifyEnded(MediaTrackGraph* aGraph) { class DecodedStreamData final { public: DecodedStreamData( - PlaybackInfoInit&& aInit, MediaTrackGraph* aGraph, - RefPtr aAudioOutputTrack, - RefPtr aVideoOutputTrack, + OutputStreamManager* aOutputStreamManager, PlaybackInfoInit&& aInit, + RefPtr aAudioStream, + RefPtr aVideoStream, MozPromiseHolder&& aAudioEndedPromise, - MozPromiseHolder&& aVideoEndedPromise); + MozPromiseHolder&& aVideoEndedPromise, + AbstractThread* aMainThread); ~DecodedStreamData(); MediaEventSource& OnOutput(); void Forget(); @@ -254,9 +258,9 @@ class DecodedStreamData final { // Count of audio frames written to the track int64_t mAudioFramesWritten; // Count of video frames written to the track in the track's rate - TrackTime mVideoTrackWritten; + TrackTime mVideoStreamWritten; // Count of audio frames written to the track in the track's rate - TrackTime mAudioTrackWritten; + TrackTime mAudioStreamWritten; // mNextAudioTime is the end timestamp for the last packet sent to the track. // Therefore audio packets starting at or after this time need to be copied // to the output track. @@ -279,66 +283,42 @@ class DecodedStreamData final { bool mHaveSentFinishAudio; bool mHaveSentFinishVideo; - const RefPtr mAudioTrack; - const RefPtr mVideoTrack; - const RefPtr mAudioOutputTrack; - const RefPtr mVideoOutputTrack; - const RefPtr mAudioPort; - const RefPtr mVideoPort; + const RefPtr mAudioStream; + const RefPtr mVideoStream; const RefPtr mListener; + + const RefPtr mOutputStreamManager; + const RefPtr mAbstractMainThread; }; DecodedStreamData::DecodedStreamData( - PlaybackInfoInit&& aInit, MediaTrackGraph* aGraph, - RefPtr aAudioOutputTrack, - RefPtr aVideoOutputTrack, + OutputStreamManager* aOutputStreamManager, PlaybackInfoInit&& aInit, + RefPtr aAudioStream, + RefPtr aVideoStream, MozPromiseHolder&& aAudioEndedPromise, - MozPromiseHolder&& aVideoEndedPromise) + MozPromiseHolder&& aVideoEndedPromise, + AbstractThread* aMainThread) : mAudioFramesWritten(0), - mVideoTrackWritten(0), - mAudioTrackWritten(0), + mVideoStreamWritten(0), + mAudioStreamWritten(0), mNextAudioTime(aInit.mStartTime), mHaveSentFinishAudio(false), mHaveSentFinishVideo(false), - mAudioTrack(aInit.mInfo.HasAudio() - ? aGraph->CreateSourceTrack(MediaSegment::AUDIO) - : nullptr), - mVideoTrack(aInit.mInfo.HasVideo() - ? aGraph->CreateSourceTrack(MediaSegment::VIDEO) - : nullptr), - mAudioOutputTrack(std::move(aAudioOutputTrack)), - mVideoOutputTrack(std::move(aVideoOutputTrack)), - mAudioPort((mAudioOutputTrack && mAudioTrack) - ? mAudioOutputTrack->AllocateInputPort(mAudioTrack) - : nullptr), - mVideoPort((mVideoOutputTrack && mVideoTrack) - ? mVideoOutputTrack->AllocateInputPort(mVideoTrack) - : nullptr), + mAudioStream(std::move(aAudioStream)), + mVideoStream(std::move(aVideoStream)), // DecodedStreamGraphListener will resolve these promises. mListener(MakeRefPtr( - mAudioTrack, std::move(aAudioEndedPromise), mVideoTrack, - std::move(aVideoEndedPromise))) { + mAudioStream, std::move(aAudioEndedPromise), mVideoStream, + std::move(aVideoEndedPromise), aMainThread)), + mOutputStreamManager(aOutputStreamManager), + mAbstractMainThread(aMainThread) { MOZ_ASSERT(NS_IsMainThread()); - if (mAudioTrack) { - mAudioTrack->SetAppendDataSourceRate(aInit.mInfo.mAudio.mRate); - } + MOZ_DIAGNOSTIC_ASSERT( + mOutputStreamManager->HasTracks(mAudioStream, mVideoStream), + "Tracks must be pre-created on main thread"); } -DecodedStreamData::~DecodedStreamData() { - MOZ_ASSERT(NS_IsMainThread()); - if (mAudioTrack) { - mAudioTrack->Destroy(); - } - if (mVideoTrack) { - mVideoTrack->Destroy(); - } - if (mAudioPort) { - mAudioPort->Destroy(); - } - if (mVideoPort) { - mVideoPort->Destroy(); - } -} +DecodedStreamData::~DecodedStreamData() { MOZ_ASSERT(NS_IsMainThread()); } MediaEventSource& DecodedStreamData::OnOutput() { return mListener->OnOutput(); @@ -349,7 +329,7 @@ void DecodedStreamData::Forget() { mListener->Forget(); } void DecodedStreamData::GetDebugInfo(dom::DecodedStreamDataDebugInfo& aInfo) { aInfo.mInstance = NS_ConvertUTF8toUTF16(nsPrintfCString("%p", this)); aInfo.mAudioFramesWritten = mAudioFramesWritten; - aInfo.mStreamAudioWritten = mAudioTrackWritten; + aInfo.mStreamAudioWritten = mAudioStreamWritten; aInfo.mNextAudioTime = mNextAudioTime.ToMicroseconds(); aInfo.mLastVideoStartTime = mLastVideoStartTime.valueOr(TimeUnit::FromMicroseconds(-1)) @@ -361,29 +341,40 @@ void DecodedStreamData::GetDebugInfo(dom::DecodedStreamDataDebugInfo& aInfo) { aInfo.mHaveSentFinishVideo = mHaveSentFinishVideo; } -DecodedStream::DecodedStream( - MediaDecoderStateMachine* aStateMachine, - nsTArray> aOutputTracks, double aVolume, - double aPlaybackRate, bool aPreservesPitch, - MediaQueue& aAudioQueue, MediaQueue& aVideoQueue) - : mOwnerThread(aStateMachine->OwnerThread()), +DecodedStream::DecodedStream(AbstractThread* aOwnerThread, + AbstractThread* aMainThread, + MediaQueue& aAudioQueue, + MediaQueue& aVideoQueue, + OutputStreamManager* aOutputStreamManager) + : mOwnerThread(aOwnerThread), + mAbstractMainThread(aMainThread), + mOutputStreamManager(aOutputStreamManager), mWatchManager(this, mOwnerThread), mPlaying(false, "DecodedStream::mPlaying"), - mPrincipalHandle(aStateMachine->OwnerThread(), PRINCIPAL_HANDLE_NONE, + mPrincipalHandle(aOwnerThread, PRINCIPAL_HANDLE_NONE, "DecodedStream::mPrincipalHandle (Mirror)"), - mOutputTracks(std::move(aOutputTracks)), - mVolume(aVolume), - mPlaybackRate(aPlaybackRate), - mPreservesPitch(aPreservesPitch), mAudioQueue(aAudioQueue), mVideoQueue(aVideoQueue) { - mPrincipalHandle.Connect(aStateMachine->CanonicalOutputPrincipal()); + mPrincipalHandle.Connect(mOutputStreamManager->CanonicalPrincipalHandle()); mWatchManager.Watch(mPlaying, &DecodedStream::PlayingChanged); + PlayingChanged(); // Notify of the initial state } DecodedStream::~DecodedStream() { MOZ_ASSERT(mStartTime.isNothing(), "playback should've ended."); + NS_ProxyRelease("DecodedStream::mOutputStreamManager", mAbstractMainThread, + do_AddRef(mOutputStreamManager)); +} + +const MediaSink::PlaybackParams& DecodedStream::GetPlaybackParams() const { + AssertOwnerThread(); + return mParams; +} + +void DecodedStream::SetPlaybackParams(const PlaybackParams& aParams) { + AssertOwnerThread(); + mParams = aParams; } RefPtr DecodedStream::OnEnded(TrackType aType) { @@ -402,7 +393,6 @@ nsresult DecodedStream::Start(const TimeUnit& aStartTime, const MediaInfo& aInfo) { AssertOwnerThread(); MOZ_ASSERT(mStartTime.isNothing(), "playback already started."); - MOZ_DIAGNOSTIC_ASSERT(!mOutputTracks.IsEmpty()); mStartTime.emplace(aStartTime); mLastOutputTime = TimeUnit::Zero(); @@ -414,55 +404,58 @@ nsresult DecodedStream::Start(const TimeUnit& aStartTime, typedef MozPromiseHolder Promise; public: - R(PlaybackInfoInit&& aInit, - nsTArray> aOutputTracks, - Promise&& aAudioEndedPromise, Promise&& aVideoEndedPromise) + R(PlaybackInfoInit&& aInit, Promise&& aAudioEndedPromise, + Promise&& aVideoEndedPromise, OutputStreamManager* aManager, + AbstractThread* aMainThread) : Runnable("CreateDecodedStreamData"), mInit(std::move(aInit)), - mOutputTracks(std::move(aOutputTracks)), mAudioEndedPromise(std::move(aAudioEndedPromise)), - mVideoEndedPromise(std::move(aVideoEndedPromise)) {} + mVideoEndedPromise(std::move(aVideoEndedPromise)), + mOutputStreamManager(aManager), + mAbstractMainThread(aMainThread) {} NS_IMETHOD Run() override { MOZ_ASSERT(NS_IsMainThread()); - RefPtr audioOutputTrack; - RefPtr videoOutputTrack; - for (const auto& track : mOutputTracks) { - if (track->mType == MediaSegment::AUDIO) { - MOZ_DIAGNOSTIC_ASSERT( - !audioOutputTrack, - "We only support capturing to one output track per kind"); - audioOutputTrack = track; - } else if (track->mType == MediaSegment::VIDEO) { - MOZ_DIAGNOSTIC_ASSERT( - !videoOutputTrack, - "We only support capturing to one output track per kind"); - videoOutputTrack = track; - } else { - MOZ_CRASH("Unknown media type"); - } - } - if ((!audioOutputTrack && !videoOutputTrack) || - (audioOutputTrack && audioOutputTrack->IsDestroyed()) || - (videoOutputTrack && videoOutputTrack->IsDestroyed())) { - // No output tracks yet, or they're going away. Halt playback by not - // creating DecodedStreamData. MDSM will try again with a new - // DecodedStream sink when tracks are available. + // No need to create a source track when there are no output tracks. + // This happens when RemoveOutput() is called immediately after + // StartPlayback(). + if (mOutputStreamManager->IsEmpty()) { + // Resolve the promise to indicate the end of playback. + mAudioEndedPromise.Resolve(true, __func__); + mVideoEndedPromise.Resolve(true, __func__); return NS_OK; } + RefPtr audioStream = + mOutputStreamManager->GetPrecreatedTrackOfType(MediaSegment::AUDIO); + if (mInit.mInfo.HasAudio() && !audioStream) { + MOZ_DIAGNOSTIC_ASSERT( + !mOutputStreamManager->HasTrackType(MediaSegment::AUDIO)); + audioStream = mOutputStreamManager->AddTrack(MediaSegment::AUDIO); + } + if (audioStream) { + audioStream->SetAppendDataSourceRate(mInit.mInfo.mAudio.mRate); + } + RefPtr videoStream = + mOutputStreamManager->GetPrecreatedTrackOfType(MediaSegment::VIDEO); + if (mInit.mInfo.HasVideo() && !videoStream) { + MOZ_DIAGNOSTIC_ASSERT( + !mOutputStreamManager->HasTrackType(MediaSegment::VIDEO)); + videoStream = mOutputStreamManager->AddTrack(MediaSegment::VIDEO); + } mData = MakeUnique( - std::move(mInit), mOutputTracks[0]->Graph(), - std::move(audioOutputTrack), std::move(videoOutputTrack), - std::move(mAudioEndedPromise), std::move(mVideoEndedPromise)); + mOutputStreamManager, std::move(mInit), std::move(audioStream), + std::move(videoStream), std::move(mAudioEndedPromise), + std::move(mVideoEndedPromise), mAbstractMainThread); return NS_OK; } UniquePtr ReleaseData() { return std::move(mData); } private: PlaybackInfoInit mInit; - const nsTArray> mOutputTracks; Promise mAudioEndedPromise; Promise mVideoEndedPromise; + RefPtr mOutputStreamManager; UniquePtr mData; + const RefPtr mAbstractMainThread; }; MozPromiseHolder audioEndedHolder; @@ -470,9 +463,9 @@ nsresult DecodedStream::Start(const TimeUnit& aStartTime, MozPromiseHolder videoEndedHolder; mVideoEndedPromise = videoEndedHolder.Ensure(__func__); PlaybackInfoInit init{aStartTime, aInfo}; - nsCOMPtr r = new R( - std::move(init), nsTArray>(mOutputTracks), - std::move(audioEndedHolder), std::move(videoEndedHolder)); + nsCOMPtr r = new R(std::move(init), std::move(audioEndedHolder), + std::move(videoEndedHolder), + mOutputStreamManager, mAbstractMainThread); SyncRunnable::DispatchToThread( SystemGroup::EventTargetFor(TaskCategory::Other), r); mData = static_cast(r.get())->ReleaseData(); @@ -525,9 +518,12 @@ void DecodedStream::DestroyData(UniquePtr&& aData) { mOutputListener.Disconnect(); - NS_DispatchToMainThread( - NS_NewRunnableFunction("DecodedStream::DestroyData", - [data = std::move(aData)]() { data->Forget(); })); + NS_DispatchToMainThread(NS_NewRunnableFunction( + "DecodedStream::DestroyData", + [data = std::move(aData), manager = mOutputStreamManager]() { + data->Forget(); + manager->RemoveTracks(); + })); } void DecodedStream::SetPlaying(bool aPlaying) { @@ -543,22 +539,17 @@ void DecodedStream::SetPlaying(bool aPlaying) { void DecodedStream::SetVolume(double aVolume) { AssertOwnerThread(); - mVolume = aVolume; + mParams.mVolume = aVolume; } void DecodedStream::SetPlaybackRate(double aPlaybackRate) { AssertOwnerThread(); - mPlaybackRate = aPlaybackRate; + mParams.mPlaybackRate = aPlaybackRate; } void DecodedStream::SetPreservesPitch(bool aPreservesPitch) { AssertOwnerThread(); - mPreservesPitch = aPreservesPitch; -} - -double DecodedStream::PlaybackRate() const { - AssertOwnerThread(); - return mPlaybackRate; + mParams.mPreservesPitch = aPreservesPitch; } static void SendStreamAudio(DecodedStreamData* aStream, @@ -637,11 +628,12 @@ void DecodedStream::SendAudio(double aVolume, // |mNextAudioTime| is updated as we process each audio sample in // SendStreamAudio(). if (output.GetDuration() > 0) { - mData->mAudioTrackWritten += mData->mAudioTrack->AppendData(&output); + mData->mAudioStreamWritten += mData->mAudioStream->AppendData(&output); } if (mAudioQueue.IsFinished() && !mData->mHaveSentFinishAudio) { - mData->mListener->EndTrackAt(mData->mAudioTrack, mData->mAudioTrackWritten); + mData->mListener->EndTrackAt(mData->mAudioStream, + mData->mAudioStreamWritten); mData->mHaveSentFinishAudio = true; } } @@ -652,9 +644,9 @@ void DecodedStreamData::WriteVideoToSegment( VideoSegment* aOutput, const PrincipalHandle& aPrincipalHandle) { RefPtr image = aImage; auto end = - mVideoTrack->MicrosecondsToTrackTimeRoundDown(aEnd.ToMicroseconds()); + mVideoStream->MicrosecondsToTrackTimeRoundDown(aEnd.ToMicroseconds()); auto start = - mVideoTrack->MicrosecondsToTrackTimeRoundDown(aStart.ToMicroseconds()); + mVideoStream->MicrosecondsToTrackTimeRoundDown(aStart.ToMicroseconds()); aOutput->AppendFrame(image.forget(), aIntrinsicSize, aPrincipalHandle, false, aTimeStamp); // Extend this so we get accurate durations for all frames. @@ -700,7 +692,7 @@ void DecodedStream::ResetVideo(const PrincipalHandle& aPrincipalHandle) { // for video tracks as part of bug 1493618. resetter.AppendFrame(nullptr, mData->mLastVideoImageDisplaySize, aPrincipalHandle, false, currentTime); - mData->mVideoTrack->AppendData(&resetter); + mData->mVideoStream->AppendData(&resetter); // Consumer buffers have been reset. We now set the next time to the start // time of the current frame, so that it can be displayed again on resuming. @@ -780,7 +772,7 @@ void DecodedStream::SendVideo(const PrincipalHandle& aPrincipalHandle) { TimeUnit end = std::max( v->GetEndTime(), lastEnd + TimeUnit::FromMicroseconds( - mData->mVideoTrack->TrackTimeToMicroseconds(1) + 1)); + mData->mVideoStream->TrackTimeToMicroseconds(1) + 1)); mData->mLastVideoImage = v->mImage; mData->mLastVideoImageDisplaySize = v->mDisplay; mData->WriteVideoToSegment(v->mImage, lastEnd, end, v->mDisplay, t, @@ -796,7 +788,7 @@ void DecodedStream::SendVideo(const PrincipalHandle& aPrincipalHandle) { } if (output.GetDuration() > 0) { - mData->mVideoTrackWritten += mData->mVideoTrack->AppendData(&output); + mData->mVideoStreamWritten += mData->mVideoStream->AppendData(&output); } if (mVideoQueue.IsFinished() && !mData->mHaveSentFinishVideo) { @@ -818,7 +810,7 @@ void DecodedStream::SendVideo(const PrincipalHandle& aPrincipalHandle) { // We round the nr of microseconds up, because WriteVideoToSegment // will round the conversion from microseconds to TrackTime down. auto deviation = TimeUnit::FromMicroseconds( - mData->mVideoTrack->TrackTimeToMicroseconds(1) + 1); + mData->mVideoStream->TrackTimeToMicroseconds(1) + 1); auto start = mData->mLastVideoEndTime.valueOr(mStartTime.ref()); mData->WriteVideoToSegment( mData->mLastVideoImage, start, start + deviation, @@ -829,9 +821,11 @@ void DecodedStream::SendVideo(const PrincipalHandle& aPrincipalHandle) { if (forceBlack) { endSegment.ReplaceWithDisabled(); } - mData->mVideoTrackWritten += mData->mVideoTrack->AppendData(&endSegment); + mData->mVideoStreamWritten += + mData->mVideoStream->AppendData(&endSegment); } - mData->mListener->EndTrackAt(mData->mVideoTrack, mData->mVideoTrackWritten); + mData->mListener->EndTrackAt(mData->mVideoStream, + mData->mVideoStreamWritten); mData->mHaveSentFinishVideo = true; } } @@ -848,7 +842,7 @@ void DecodedStream::SendData() { return; } - SendAudio(mVolume, mPrincipalHandle); + SendAudio(mParams.mVolume, mPrincipalHandle); SendVideo(mPrincipalHandle); } @@ -902,6 +896,10 @@ void DecodedStream::PlayingChanged() { // On seek or pause we discard future frames. ResetVideo(mPrincipalHandle); } + + mAbstractMainThread->Dispatch(NewRunnableMethod( + "OutputStreamManager::SetPlaying", mOutputStreamManager, + &OutputStreamManager::SetPlaying, mPlaying)); } void DecodedStream::ConnectListener() { diff --git a/dom/media/mediasink/DecodedStream.h b/dom/media/mediasink/DecodedStream.h index abc57127c9e0..c7d198540b42 100644 --- a/dom/media/mediasink/DecodedStream.h +++ b/dom/media/mediasink/DecodedStream.h @@ -22,9 +22,9 @@ namespace mozilla { class DecodedStreamData; -class MediaDecoderStateMachine; class AudioData; class VideoData; +class OutputStreamManager; struct PlaybackInfoInit; class ProcessedMediaTrack; class TimeStamp; @@ -33,12 +33,17 @@ template class MediaQueue; class DecodedStream : public MediaSink { + using MediaSink::PlaybackParams; + public: - DecodedStream(MediaDecoderStateMachine* aStateMachine, - nsTArray> aOutputTracks, - double aVolume, double aPlaybackRate, bool aPreservesPitch, + DecodedStream(AbstractThread* aOwnerThread, AbstractThread* aMainThread, MediaQueue& aAudioQueue, - MediaQueue& aVideoQueue); + MediaQueue& aVideoQueue, + OutputStreamManager* aOutputStreamManager); + + // MediaSink functions. + const PlaybackParams& GetPlaybackParams() const override; + void SetPlaybackParams(const PlaybackParams& aParams) override; RefPtr OnEnded(TrackType aType) override; media::TimeUnit GetEndTime(TrackType aType) const override; @@ -53,8 +58,6 @@ class DecodedStream : public MediaSink { void SetPreservesPitch(bool aPreservesPitch) override; void SetPlaying(bool aPlaying) override; - double PlaybackRate() const override; - nsresult Start(const media::TimeUnit& aStartTime, const MediaInfo& aInfo) override; void Stop() override; @@ -85,6 +88,14 @@ class DecodedStream : public MediaSink { const RefPtr mOwnerThread; + const RefPtr mAbstractMainThread; + + /* + * Main thread only members. + */ + // Data about MediaStreams that are being fed by the decoder. + const RefPtr mOutputStreamManager; + /* * Worker thread only members. */ @@ -95,11 +106,8 @@ class DecodedStream : public MediaSink { Watchable mPlaying; Mirror mPrincipalHandle; - const nsTArray> mOutputTracks; - double mVolume; - double mPlaybackRate; - bool mPreservesPitch; + PlaybackParams mParams; media::NullableTimeUnit mStartTime; media::TimeUnit mLastOutputTime; diff --git a/dom/media/mediasink/MediaSink.h b/dom/media/mediasink/MediaSink.h index 5f31564868c0..0f0a347cff1b 100644 --- a/dom/media/mediasink/MediaSink.h +++ b/dom/media/mediasink/MediaSink.h @@ -7,6 +7,7 @@ #ifndef MediaSink_h_ #define MediaSink_h_ +#include "AudioDeviceInfo.h" #include "MediaInfo.h" #include "mozilla/MozPromise.h" #include "mozilla/RefPtr.h" @@ -38,6 +39,23 @@ class MediaSink { NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSink); typedef mozilla::TrackInfo::TrackType TrackType; + struct PlaybackParams { + PlaybackParams() + : mVolume(1.0), mPlaybackRate(1.0), mPreservesPitch(true) {} + double mVolume; + double mPlaybackRate; + bool mPreservesPitch; + RefPtr mSink; + }; + + // Return the playback parameters of this sink. + // Can be called in any state. + virtual const PlaybackParams& GetPlaybackParams() const = 0; + + // Set the playback parameters of this sink. + // Can be called in any state. + virtual void SetPlaybackParams(const PlaybackParams& aParams) = 0; + // EndedPromise needs to be a non-exclusive promise as it is shared between // both the AudioSink and VideoSink. typedef MozPromise EndedPromise; @@ -82,10 +100,6 @@ class MediaSink { // Pause/resume the playback. Only work after playback starts. virtual void SetPlaying(bool aPlaying) = 0; - // Get the playback rate. - // Can be called in any state. - virtual double PlaybackRate() const = 0; - // Single frame rendering operation may need to be done before playback // started (1st frame) or right after seek completed or playback stopped. // Do nothing if this sink has no video track. Can be called in any state. @@ -108,10 +122,6 @@ class MediaSink { // Can be called in any state. virtual bool IsPlaying() const = 0; - // The audio output device this MediaSink is playing audio data to. The - // default device is used if this returns null. - virtual const AudioDeviceInfo* AudioDevice() { return nullptr; } - // Called on the state machine thread to shut down the sink. All resources // allocated by this sink should be released. // Must be called after playback stopped. diff --git a/dom/media/mediasink/OutputStreamManager.cpp b/dom/media/mediasink/OutputStreamManager.cpp new file mode 100644 index 000000000000..ec1f9d51bf01 --- /dev/null +++ b/dom/media/mediasink/OutputStreamManager.cpp @@ -0,0 +1,357 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "OutputStreamManager.h" + +#include "DOMMediaStream.h" +#include "../MediaTrackGraph.h" +#include "mozilla/dom/MediaStreamTrack.h" +#include "mozilla/dom/AudioStreamTrack.h" +#include "mozilla/dom/VideoStreamTrack.h" +#include "nsContentUtils.h" + +namespace mozilla { + +#define LOG(level, msg, ...) \ + MOZ_LOG(gMediaDecoderLog, level, (msg, ##__VA_ARGS__)) + +class DecodedStreamTrackSource : public dom::MediaStreamTrackSource { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(DecodedStreamTrackSource, + dom::MediaStreamTrackSource) + + explicit DecodedStreamTrackSource(SourceMediaTrack* aSourceStream, + nsIPrincipal* aPrincipal) + : dom::MediaStreamTrackSource(aPrincipal, nsString()), + mTrack(aSourceStream->Graph()->CreateForwardedInputTrack( + aSourceStream->mType)), + mPort(mTrack->AllocateInputPort(aSourceStream)) { + MOZ_ASSERT(NS_IsMainThread()); + } + + dom::MediaSourceEnum GetMediaSource() const override { + return dom::MediaSourceEnum::Other; + } + + void Stop() override { + MOZ_ASSERT(NS_IsMainThread()); + + // We don't notify the source that a track was stopped since it will keep + // producing tracks until the element ends. The decoder also needs the + // tracks it created to be live at the source since the decoder's clock is + // based on MediaStreams during capture. We do however, disconnect this + // track's underlying track. + if (!mTrack->IsDestroyed()) { + mTrack->Destroy(); + mPort->Destroy(); + } + } + + void Disable() override {} + + void Enable() override {} + + void SetPrincipal(nsIPrincipal* aPrincipal) { + MOZ_ASSERT(NS_IsMainThread()); + mPrincipal = aPrincipal; + PrincipalChanged(); + } + + void ForceEnded() { OverrideEnded(); } + + const RefPtr mTrack; + const RefPtr mPort; + + protected: + virtual ~DecodedStreamTrackSource() { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(mTrack->IsDestroyed()); + } +}; + +NS_IMPL_ADDREF_INHERITED(DecodedStreamTrackSource, dom::MediaStreamTrackSource) +NS_IMPL_RELEASE_INHERITED(DecodedStreamTrackSource, dom::MediaStreamTrackSource) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(DecodedStreamTrackSource) +NS_INTERFACE_MAP_END_INHERITING(dom::MediaStreamTrackSource) +NS_IMPL_CYCLE_COLLECTION_INHERITED(DecodedStreamTrackSource, + dom::MediaStreamTrackSource) + +OutputStreamData::OutputStreamData(OutputStreamManager* aManager, + AbstractThread* aAbstractMainThread, + DOMMediaStream* aDOMStream) + : mManager(aManager), + mAbstractMainThread(aAbstractMainThread), + mDOMStream(aDOMStream) { + MOZ_ASSERT(NS_IsMainThread()); +} + +OutputStreamData::~OutputStreamData() = default; + +void OutputStreamData::AddTrack(SourceMediaTrack* aTrack, + MediaSegment::Type aType, + nsIPrincipal* aPrincipal, bool aAsyncAddTrack) { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(mDOMStream); + + LOG(LogLevel::Debug, + "Adding output %s track sourced from track %p to MediaStream %p%s", + aType == MediaSegment::AUDIO ? "audio" : "video", aTrack, + mDOMStream.get(), aAsyncAddTrack ? " (async)" : ""); + + auto source = MakeRefPtr(aTrack, aPrincipal); + RefPtr track; + if (aType == MediaSegment::AUDIO) { + track = new dom::AudioStreamTrack(mDOMStream->GetParentObject(), + source->mTrack, source); + } else { + MOZ_ASSERT(aType == MediaSegment::VIDEO); + track = new dom::VideoStreamTrack(mDOMStream->GetParentObject(), + source->mTrack, source); + } + mTracks.AppendElement(track.get()); + if (aAsyncAddTrack) { + GetMainThreadEventTarget()->Dispatch( + NewRunnableMethod>( + "DOMMediaStream::AddTrackInternal", mDOMStream.get(), + &DOMMediaStream::AddTrackInternal, track)); + } else { + mDOMStream->AddTrackInternal(track); + } +} + +void OutputStreamData::RemoveTrack(SourceMediaTrack* aTrack) { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(mDOMStream); + + LOG(LogLevel::Debug, + "Removing output track sourced by track %p from MediaStream %p", aTrack, + mDOMStream.get()); + + for (const auto& t : nsTArray>(mTracks)) { + mTracks.RemoveElement(t); + if (!t || t->Ended()) { + continue; + } + DecodedStreamTrackSource& source = + static_cast(t->GetSource()); + GetMainThreadEventTarget()->Dispatch( + NewRunnableMethod("DecodedStreamTrackSource::ForceEnded", &source, + &DecodedStreamTrackSource::ForceEnded)); + } +} + +void OutputStreamData::SetPrincipal(nsIPrincipal* aPrincipal) { + MOZ_DIAGNOSTIC_ASSERT(mDOMStream); + for (const WeakPtr& track : mTracks) { + if (!track || track->Ended()) { + continue; + } + DecodedStreamTrackSource& source = + static_cast(track->GetSource()); + source.SetPrincipal(aPrincipal); + } +} + +OutputStreamManager::OutputStreamManager(SharedDummyTrack* aDummyStream, + nsIPrincipal* aPrincipal, + AbstractThread* aAbstractMainThread) + : mAbstractMainThread(aAbstractMainThread), + mDummyStream(aDummyStream), + mPrincipalHandle( + aAbstractMainThread, + aPrincipal ? MakePrincipalHandle(aPrincipal) : PRINCIPAL_HANDLE_NONE, + "OutputStreamManager::mPrincipalHandle (Canonical)") { + MOZ_ASSERT(NS_IsMainThread()); +} + +void OutputStreamManager::Add(DOMMediaStream* aDOMStream) { + MOZ_ASSERT(NS_IsMainThread()); + + LOG(LogLevel::Info, "Adding MediaStream %p", aDOMStream); + + OutputStreamData* p = mStreams + .AppendElement(new OutputStreamData( + this, mAbstractMainThread, aDOMStream)) + ->get(); + for (const auto& lt : mLiveTracks) { + p->AddTrack(lt->mSourceTrack, lt->mType, mPrincipalHandle.Ref(), false); + } +} + +void OutputStreamManager::Remove(DOMMediaStream* aDOMStream) { + MOZ_ASSERT(NS_IsMainThread()); + + LOG(LogLevel::Info, "Removing MediaStream %p", aDOMStream); + + AutoRemoveDestroyedStreams(); + mStreams.ApplyIf( + aDOMStream, 0, StreamComparator(), + [&](const UniquePtr& aData) { + for (const auto& lt : mLiveTracks) { + aData->RemoveTrack(lt->mSourceTrack); + } + }, + []() { MOZ_ASSERT_UNREACHABLE("Didn't exist"); }); + DebugOnly rv = mStreams.RemoveElement(aDOMStream, StreamComparator()); + MOZ_ASSERT(rv); +} + +bool OutputStreamManager::HasTrackType(MediaSegment::Type aType) { + MOZ_ASSERT(NS_IsMainThread()); + + return mLiveTracks.Contains(aType, TrackTypeComparator()); +} + +bool OutputStreamManager::HasTracks(SourceMediaTrack* aAudioStream, + SourceMediaTrack* aVideoStream) { + MOZ_ASSERT(NS_IsMainThread()); + + size_t nrExpectedTracks = 0; + bool asExpected = true; + if (aAudioStream) { + Unused << ++nrExpectedTracks; + asExpected = asExpected && mLiveTracks.Contains( + MakePair(aAudioStream, MediaSegment::AUDIO), + TrackComparator()); + } + if (aVideoStream) { + Unused << ++nrExpectedTracks; + asExpected = asExpected && mLiveTracks.Contains( + MakePair(aVideoStream, MediaSegment::VIDEO), + TrackComparator()); + } + asExpected = asExpected && mLiveTracks.Length() == nrExpectedTracks; + return asExpected; +} + +SourceMediaTrack* OutputStreamManager::GetPrecreatedTrackOfType( + MediaSegment::Type aType) const { + auto i = mLiveTracks.IndexOf(aType, 0, PrecreatedTrackTypeComparator()); + return i == nsTArray>::NoIndex + ? nullptr + : mLiveTracks[i]->mSourceTrack.get(); +} + +size_t OutputStreamManager::NumberOfTracks() { + MOZ_ASSERT(NS_IsMainThread()); + return mLiveTracks.Length(); +} + +already_AddRefed OutputStreamManager::AddTrack( + MediaSegment::Type aType) { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(!HasTrackType(aType), + "Cannot have two tracks of the same type at the same time"); + + RefPtr track = + mDummyStream->mTrack->Graph()->CreateSourceTrack(aType); + if (!mPlaying) { + track->Suspend(); + } + + LOG(LogLevel::Info, "Adding %s track sourced by track %p", + aType == MediaSegment::AUDIO ? "audio" : "video", track.get()); + + mLiveTracks.AppendElement(MakeUnique(track, aType)); + AutoRemoveDestroyedStreams(); + for (const auto& data : mStreams) { + data->AddTrack(track, aType, mPrincipalHandle.Ref(), true); + } + + return track.forget(); +} + +OutputStreamManager::LiveTrack::LiveTrack(SourceMediaTrack* aSourceTrack, + MediaSegment::Type aType) + : mSourceTrack(aSourceTrack), mType(aType) {} + +OutputStreamManager::LiveTrack::~LiveTrack() { mSourceTrack->Destroy(); } + +void OutputStreamManager::AutoRemoveDestroyedStreams() { + MOZ_ASSERT(NS_IsMainThread()); + for (size_t i = mStreams.Length(); i > 0; --i) { + const auto& data = mStreams[i - 1]; + if (!data->mDOMStream) { + // If the mDOMStream WeakPtr is now null, mDOMStream has been destructed. + mStreams.RemoveElementAt(i - 1); + } + } +} + +void OutputStreamManager::RemoveTrack(SourceMediaTrack* aTrack) { + MOZ_ASSERT(NS_IsMainThread()); + LOG(LogLevel::Info, "Removing track with source track %p", aTrack); + DebugOnly rv = + mLiveTracks.RemoveElement(aTrack, TrackStreamComparator()); + MOZ_ASSERT(rv); + AutoRemoveDestroyedStreams(); + for (const auto& data : mStreams) { + data->RemoveTrack(aTrack); + } +} + +void OutputStreamManager::RemoveTracks() { + MOZ_ASSERT(NS_IsMainThread()); + for (size_t i = mLiveTracks.Length(); i > 0; --i) { + RemoveTrack(mLiveTracks[i - 1]->mSourceTrack); + } +} + +void OutputStreamManager::Disconnect() { + MOZ_ASSERT(NS_IsMainThread()); + RemoveTracks(); + MOZ_ASSERT(mLiveTracks.IsEmpty()); + AutoRemoveDestroyedStreams(); + nsTArray> domStreams(mStreams.Length()); + for (const auto& data : mStreams) { + domStreams.AppendElement(data->mDOMStream); + } + for (auto& domStream : domStreams) { + Remove(domStream); + } + MOZ_ASSERT(mStreams.IsEmpty()); +} + +AbstractCanonical* +OutputStreamManager::CanonicalPrincipalHandle() { + return &mPrincipalHandle; +} + +void OutputStreamManager::SetPrincipal(nsIPrincipal* aPrincipal) { + MOZ_ASSERT(NS_IsMainThread()); + nsCOMPtr principal = GetPrincipalFromHandle(mPrincipalHandle); + if (nsContentUtils::CombineResourcePrincipals(&principal, aPrincipal)) { + AutoRemoveDestroyedStreams(); + for (const UniquePtr& data : mStreams) { + data->SetPrincipal(principal); + } + mPrincipalHandle = MakePrincipalHandle(principal); + } +} + +void OutputStreamManager::SetPlaying(bool aPlaying) { + MOZ_ASSERT(NS_IsMainThread()); + if (mPlaying == aPlaying) { + return; + } + + mPlaying = aPlaying; + for (auto& lt : mLiveTracks) { + if (mPlaying) { + lt->mSourceTrack->Resume(); + lt->mEverPlayed = true; + } else { + lt->mSourceTrack->Suspend(); + } + } +} + +OutputStreamManager::~OutputStreamManager() = default; + +#undef LOG + +} // namespace mozilla diff --git a/dom/media/mediasink/OutputStreamManager.h b/dom/media/mediasink/OutputStreamManager.h new file mode 100644 index 000000000000..a4459d9cec29 --- /dev/null +++ b/dom/media/mediasink/OutputStreamManager.h @@ -0,0 +1,161 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef OutputStreamManager_h +#define OutputStreamManager_h + +#include "mozilla/RefPtr.h" +#include "mozilla/StateMirroring.h" +#include "mozilla/WeakPtr.h" +#include "nsTArray.h" + +namespace mozilla { + +class DOMMediaStream; +class MediaInputPort; +class OutputStreamManager; +class ProcessedMediaTrack; +class SourceMediaTrack; + +namespace dom { +class MediaStreamTrack; +} + +class OutputStreamData { + public: + OutputStreamData(OutputStreamManager* aManager, + AbstractThread* aAbstractMainThread, + DOMMediaStream* aDOMStream); + OutputStreamData(const OutputStreamData& aOther) = delete; + OutputStreamData(OutputStreamData&& aOther) = delete; + ~OutputStreamData(); + + // Creates and adds a MediaStreamTrack to mDOMStream so that we can feed data + // to it. For a true aAsyncAddTrack we will dispatch a task to add the + // created track to mDOMStream, as is required by spec for the "addtrack" + // event. + void AddTrack(SourceMediaTrack* aTrack, MediaSegment::Type aType, + nsIPrincipal* aPrincipal, bool aAsyncAddTrack); + // Ends any MediaStreamTracks sourced from aTrack. + void RemoveTrack(SourceMediaTrack* aTrack); + + void SetPrincipal(nsIPrincipal* aPrincipal); + + const RefPtr mManager; + const RefPtr mAbstractMainThread; + // The DOMMediaStream we add tracks to and represent. + const WeakPtr mDOMStream; + + private: + // Tracks that have been added and not yet removed. + nsTArray> mTracks; +}; + +class OutputStreamManager { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(OutputStreamManager); + + public: + OutputStreamManager(SharedDummyTrack* aDummyStream, nsIPrincipal* aPrincipal, + AbstractThread* aAbstractMainThread); + // Add the output stream to the collection. + void Add(DOMMediaStream* aDOMStream); + // Remove the output stream from the collection. + void Remove(DOMMediaStream* aDOMStream); + // Returns true if there's a live track of the given type. + bool HasTrackType(MediaSegment::Type aType); + // Returns true if the given tracks are sourcing all currently live tracks. + // Use nullptr to make it ignored for that type. + bool HasTracks(SourceMediaTrack* aAudioStream, + SourceMediaTrack* aVideoStream); + // Gets the underlying track for the given type if it has never been played, + // or nullptr if there is none. + SourceMediaTrack* GetPrecreatedTrackOfType(MediaSegment::Type aType) const; + // Returns the number of live tracks. + size_t NumberOfTracks(); + // Add a track sourced to all output tracks and return the MediaTrack that + // sources it. + already_AddRefed AddTrack(MediaSegment::Type aType); + // Remove all currently live tracks. + void RemoveTracks(); + // Remove all currently live tracks and all output streams. + void Disconnect(); + // The principal handle for the underlying decoder. + AbstractCanonical* CanonicalPrincipalHandle(); + // Called when the underlying decoder's principal has changed. + void SetPrincipal(nsIPrincipal* aPrincipal); + // Called by DecodedStream when its playing state changes. While not playing + // we suspend mSourceTrack. + void SetPlaying(bool aPlaying); + // Return true if the collection of output streams is empty. + bool IsEmpty() const { + MOZ_ASSERT(NS_IsMainThread()); + return mStreams.IsEmpty(); + } + + const RefPtr mAbstractMainThread; + + private: + ~OutputStreamManager(); + + class LiveTrack { + public: + LiveTrack(SourceMediaTrack* aSourceTrack, MediaSegment::Type aType); + ~LiveTrack(); + + const RefPtr mSourceTrack; + const MediaSegment::Type mType; + bool mEverPlayed = false; + }; + + struct StreamComparator { + static bool Equals(const UniquePtr& aData, + DOMMediaStream* aStream) { + return aData->mDOMStream == aStream; + } + }; + struct TrackStreamComparator { + static bool Equals(const UniquePtr& aLiveTrack, + SourceMediaTrack* aTrack) { + return aLiveTrack->mSourceTrack == aTrack; + } + }; + struct TrackTypeComparator { + static bool Equals(const UniquePtr& aLiveTrack, + MediaSegment::Type aType) { + return aLiveTrack->mType == aType; + } + }; + struct PrecreatedTrackTypeComparator { + static bool Equals(const UniquePtr& aLiveTrack, + MediaSegment::Type aType) { + return !aLiveTrack->mEverPlayed && aLiveTrack->mType == aType; + } + }; + struct TrackComparator { + static bool Equals( + const UniquePtr& aLiveTrack, + const Pair& aOther) { + return aLiveTrack->mSourceTrack == aOther.first() && + aLiveTrack->mType == aOther.second(); + } + }; + + // Goes through mStreams and removes any entries that have been destroyed. + void AutoRemoveDestroyedStreams(); + + // Remove tracks sourced from aTrack from all output tracks. + void RemoveTrack(SourceMediaTrack* aTrack); + + const RefPtr mDummyStream; + nsTArray> mStreams; + nsTArray> mLiveTracks; + Canonical mPrincipalHandle; + bool mPlaying = false; +}; + +} // namespace mozilla + +#endif // OutputStreamManager_h diff --git a/dom/media/mediasink/VideoSink.cpp b/dom/media/mediasink/VideoSink.cpp index 70eccfb7b8a9..917139f471ae 100644 --- a/dom/media/mediasink/VideoSink.cpp +++ b/dom/media/mediasink/VideoSink.cpp @@ -156,6 +156,18 @@ VideoSink::~VideoSink() { #endif } +const MediaSink::PlaybackParams& VideoSink::GetPlaybackParams() const { + AssertOwnerThread(); + + return mAudioSink->GetPlaybackParams(); +} + +void VideoSink::SetPlaybackParams(const PlaybackParams& aParams) { + AssertOwnerThread(); + + mAudioSink->SetPlaybackParams(aParams); +} + RefPtr VideoSink::OnEnded(TrackType aType) { AssertOwnerThread(); MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts."); @@ -211,12 +223,6 @@ void VideoSink::SetPreservesPitch(bool aPreservesPitch) { mAudioSink->SetPreservesPitch(aPreservesPitch); } -double VideoSink::PlaybackRate() const { - AssertOwnerThread(); - - return mAudioSink->PlaybackRate(); -} - void VideoSink::EnsureHighResTimersOnOnlyIfPlaying() { #ifdef XP_WIN const bool needed = IsPlaying(); @@ -434,8 +440,8 @@ void VideoSink::TryUpdateRenderedVideoFrames() { // If we send this future frame to the compositor now, it will be rendered // immediately and break A/V sync. Instead, we schedule a timer to send it // later. - int64_t delta = - (v->mTime - clockTime).ToMicroseconds() / mAudioSink->PlaybackRate(); + int64_t delta = (v->mTime - clockTime).ToMicroseconds() / + mAudioSink->GetPlaybackParams().mPlaybackRate; TimeStamp target = nowTime + TimeDuration::FromMicroseconds(delta); RefPtr self = this; mUpdateScheduler.Ensure( @@ -475,7 +481,7 @@ void VideoSink::RenderVideoFrames(int32_t aMaxFrames, int64_t aClockTime, AutoTArray images; TimeStamp lastFrameTime; - double playbackRate = mAudioSink->PlaybackRate(); + MediaSink::PlaybackParams params = mAudioSink->GetPlaybackParams(); for (uint32_t i = 0; i < frames.Length(); ++i) { VideoData* frame = frames[i]; bool wasSent = frame->IsSentToCompositor(); @@ -493,8 +499,8 @@ void VideoSink::RenderVideoFrames(int32_t aMaxFrames, int64_t aClockTime, MOZ_ASSERT(!aClockTimeStamp.IsNull()); int64_t delta = frame->mTime.ToMicroseconds() - aClockTime; - TimeStamp t = - aClockTimeStamp + TimeDuration::FromMicroseconds(delta / playbackRate); + TimeStamp t = aClockTimeStamp + + TimeDuration::FromMicroseconds(delta / params.mPlaybackRate); if (!lastFrameTime.IsNull() && t <= lastFrameTime) { // Timestamps out of order; drop the new frame. In theory we should // probably replace the previous frame with the new frame if the @@ -607,8 +613,9 @@ void VideoSink::UpdateRenderedVideoFrames() { int64_t nextFrameTime = frames[1]->mTime.ToMicroseconds(); int64_t delta = std::max(nextFrameTime - clockTime.ToMicroseconds(), MIN_UPDATE_INTERVAL_US); - TimeStamp target = nowTime + TimeDuration::FromMicroseconds( - delta / mAudioSink->PlaybackRate()); + TimeStamp target = + nowTime + TimeDuration::FromMicroseconds( + delta / mAudioSink->GetPlaybackParams().mPlaybackRate); RefPtr self = this; mUpdateScheduler.Ensure( @@ -640,7 +647,7 @@ void VideoSink::MaybeResolveEndPromise() { "end promise. clockTime=%" PRId64 ", endTime=%" PRId64, clockTime.ToMicroseconds(), mVideoFrameEndTime.ToMicroseconds()); int64_t delta = (mVideoFrameEndTime - clockTime).ToMicroseconds() / - mAudioSink->PlaybackRate(); + mAudioSink->GetPlaybackParams().mPlaybackRate; TimeStamp target = nowTime + TimeDuration::FromMicroseconds(delta); auto resolveEndPromise = [self = RefPtr(this)]() { self->mEndPromiseHolder.ResolveIfExists(true, __func__); diff --git a/dom/media/mediasink/VideoSink.h b/dom/media/mediasink/VideoSink.h index 6a9a9e38e80e..721cc5538c41 100644 --- a/dom/media/mediasink/VideoSink.h +++ b/dom/media/mediasink/VideoSink.h @@ -32,6 +32,10 @@ class VideoSink : public MediaSink { MediaQueue& aVideoQueue, VideoFrameContainer* aContainer, FrameStatistics& aFrameStats, uint32_t aVQueueSentToCompositerSize); + const PlaybackParams& GetPlaybackParams() const override; + + void SetPlaybackParams(const PlaybackParams& aParams) override; + RefPtr OnEnded(TrackType aType) override; TimeUnit GetEndTime(TrackType aType) const override; @@ -48,8 +52,6 @@ class VideoSink : public MediaSink { void SetPlaying(bool aPlaying) override; - double PlaybackRate() const override; - void Redraw(const VideoInfo& aInfo) override; nsresult Start(const TimeUnit& aStartTime, const MediaInfo& aInfo) override; diff --git a/dom/media/mediasink/moz.build b/dom/media/mediasink/moz.build index fba2abb269c5..383bce657182 100644 --- a/dom/media/mediasink/moz.build +++ b/dom/media/mediasink/moz.build @@ -8,6 +8,7 @@ UNIFIED_SOURCES += [ 'AudioSink.cpp', 'AudioSinkWrapper.cpp', 'DecodedStream.cpp', + 'OutputStreamManager.cpp', 'VideoSink.cpp', ] diff --git a/dom/media/mediasource/SourceBufferResource.cpp b/dom/media/mediasource/SourceBufferResource.cpp index 49447be01532..f0dedf390786 100644 --- a/dom/media/mediasource/SourceBufferResource.cpp +++ b/dom/media/mediasource/SourceBufferResource.cpp @@ -24,11 +24,11 @@ mozilla::LogModule* GetSourceBufferResourceLog() { namespace mozilla { -RefPtr SourceBufferResource::Close() { +nsresult SourceBufferResource::Close() { MOZ_ASSERT(OnThread()); SBR_DEBUG("Close"); mClosed = true; - return GenericPromise::CreateAndResolve(true, __func__); + return NS_OK; } nsresult SourceBufferResource::ReadAt(int64_t aOffset, char* aBuffer, diff --git a/dom/media/mediasource/SourceBufferResource.h b/dom/media/mediasource/SourceBufferResource.h index 5810412ea390..8649a1de9fbf 100644 --- a/dom/media/mediasource/SourceBufferResource.h +++ b/dom/media/mediasource/SourceBufferResource.h @@ -36,7 +36,7 @@ class SourceBufferResource final public DecoderDoctorLifeLogger { public: SourceBufferResource(); - RefPtr Close() override; + nsresult Close() override; nsresult ReadAt(int64_t aOffset, char* aBuffer, uint32_t aCount, uint32_t* aBytes) override; // Memory-based and no locks, caching discouraged. diff --git a/dom/media/test/test_mediatrack_consuming_mediaresource.html b/dom/media/test/test_mediatrack_consuming_mediaresource.html index 515df5c05378..b3d274499593 100644 --- a/dom/media/test/test_mediatrack_consuming_mediaresource.html +++ b/dom/media/test/test_mediatrack_consuming_mediaresource.html @@ -10,19 +10,19 @@ 
 
   
-  
+  
 
 
 
 
 
 
-
 

 
diff --git a/dom/media/webaudio/MediaElementAudioSourceNode.cpp b/dom/media/webaudio/MediaElementAudioSourceNode.cpp
index 09af8e0d23a8..a60e0cf6140d 100644
--- a/dom/media/webaudio/MediaElementAudioSourceNode.cpp
+++ b/dom/media/webaudio/MediaElementAudioSourceNode.cpp
@@ -9,7 +9,6 @@
 #include "AudioDestinationNode.h"
 #include "nsIScriptError.h"
 #include "AudioNodeTrack.h"
-#include "MediaStreamTrack.h"
 
 namespace mozilla {
 namespace dom {
diff --git a/dom/media/webaudio/MediaStreamAudioSourceNode.cpp b/dom/media/webaudio/MediaStreamAudioSourceNode.cpp
index 12048246f07e..c1663db0bc09 100644
--- a/dom/media/webaudio/MediaStreamAudioSourceNode.cpp
+++ b/dom/media/webaudio/MediaStreamAudioSourceNode.cpp
@@ -76,8 +76,8 @@ void MediaStreamAudioSourceNode::Init(DOMMediaStream* aMediaStream,
   mInputStream->AddConsumerToKeepAlive(ToSupports(this));
 
   mInputStream->RegisterTrackListener(this);
-  if (mInputStream->Audible()) {
-    NotifyAudible();
+  if (mInputStream->Active()) {
+    NotifyActive();
   }
   AttachToRightTrack(mInputStream, aRv);
 }
@@ -119,7 +119,6 @@ void MediaStreamAudioSourceNode::AttachToTrack(
   mInputPort = mInputTrack->ForwardTrackContentsTo(outputTrack);
   PrincipalChanged(mInputTrack);  // trigger enabling/disabling of the connector
   mInputTrack->AddPrincipalChangeObserver(this);
-  MarkActive();
 }
 
 void MediaStreamAudioSourceNode::DetachFromTrack() {
@@ -166,6 +165,7 @@ void MediaStreamAudioSourceNode::AttachToRightTrack(
 
     if (!track->Ended()) {
       AttachToTrack(track, aRv);
+      MarkActive();
     }
     return;
   }
@@ -202,7 +202,7 @@ void MediaStreamAudioSourceNode::NotifyTrackRemoved(
   }
 }
 
-void MediaStreamAudioSourceNode::NotifyAudible() {
+void MediaStreamAudioSourceNode::NotifyActive() {
   MOZ_ASSERT(mInputStream);
   Context()->StartBlockedAudioContextIfAllowed();
 }
diff --git a/dom/media/webaudio/MediaStreamAudioSourceNode.h b/dom/media/webaudio/MediaStreamAudioSourceNode.h
index dd77dfd4c482..c387a46f534f 100644
--- a/dom/media/webaudio/MediaStreamAudioSourceNode.h
+++ b/dom/media/webaudio/MediaStreamAudioSourceNode.h
@@ -91,7 +91,7 @@ class MediaStreamAudioSourceNode
   // From DOMMediaStream::TrackListener.
   void NotifyTrackAdded(const RefPtr& aTrack) override;
   void NotifyTrackRemoved(const RefPtr& aTrack) override;
-  void NotifyAudible() override;
+  void NotifyActive() override;
 
   // From PrincipalChangeObserver.
   void PrincipalChanged(MediaStreamTrack* aMediaStreamTrack) override;
diff --git a/testing/web-platform/meta/webaudio/the-audio-api/the-mediaelementaudiosourcenode-interface/mediaElementAudioSourceToScriptProcessorTest.html.ini b/testing/web-platform/meta/webaudio/the-audio-api/the-mediaelementaudiosourcenode-interface/mediaElementAudioSourceToScriptProcessorTest.html.ini
new file mode 100644
index 000000000000..767be080a590
--- /dev/null
+++ b/testing/web-platform/meta/webaudio/the-audio-api/the-mediaelementaudiosourcenode-interface/mediaElementAudioSourceToScriptProcessorTest.html.ini
@@ -0,0 +1,8 @@
+[mediaElementAudioSourceToScriptProcessorTest.html]
+  disabled:
+    if (os == "mac") and (version == "OS X 10.14"): new platform
+    if (os == "android") and debug: https://bugzilla.mozilla.org/show_bug.cgi?id=1546756
+  [All data processed correctly]
+    expected:
+      if processor == "aarch64": ["PASS", "FAIL"]
+