Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-25 05:41:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1829068 retain the same AudioSinkWrapper when switching output devices r=padenot

Browse Source 
This allows video playback to continue uninterrupted.

The AudioSinkWrapper now maintains the correct playback position in the media
data.  Previously the replacement AudioSinkWrapper started its clock at the
"current playback position" from the MediaDecoderStateMachine, which did not
include additional time from looping, so silence would be played until the
missing time had passed.

The AudioSinkWrapper now continues to periodically attempt to open a new
output device if not initially available.  Previously such attempts were only
performed on state changes such as seeking or unmuting.

The promise returned from MediaDecoderStateMachine::InvokeSetSink() now
resolves regardless of whether the new device can be opened successfully.
Previously the promise could be rejected if the device was necessary for audio
output at the time of the call but no check for device availability (beyond
that of device enumeration) was performed if playback was suspended or muted
or if a subsequent device change had already been queued.  Even previously the
underlying device was changed even when the promise was rejected and could be
used if a state change occurred.

Differential Revision: https://phabricator.services.mozilla.com/D181975
This commit is contained in:
Karl Tomlinson 2023-06-27 08:53:50 +00:00
parent 7fa10acf9d
commit d7940eeeb5
11 changed files with 153 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dom/html/HTMLMediaElement.cpp
View File

@ -5053,16 +5053,7 @@ nsresult HTMLMediaElement::FinishDecoderSetup(MediaDecoder* aDecoder) {
// Set sink device if we have one. Otherwise the default is used.
if (mSink.second) {
mDecoder
->SetSink(mSink.second)
#ifdef DEBUG
->Then(mAbstractMainThread, __func__,
[](const GenericPromise::ResolveOrRejectValue& aValue) {
MOZ_ASSERT(aValue.IsResolve() && !aValue.ResolveValue());
});
#else
;
#endif
mDecoder->SetSink(mSink.second);
}
if (mMediaKeys) {

35
dom/media/MediaDecoderStateMachine.cpp
View File

@ -4430,43 +4430,14 @@ RefPtr<GenericPromise> MediaDecoderStateMachine::InvokeSetSink(
}
RefPtr<GenericPromise> MediaDecoderStateMachine::SetSink(
const RefPtr<AudioDeviceInfo>& aDevice) {
RefPtr<AudioDeviceInfo> aDevice) {
MOZ_ASSERT(OnTaskQueue());
if (mIsMediaSinkSuspended) {
// Don't create a new media sink when suspended.
return GenericPromise::CreateAndResolve(false, __func__);
return GenericPromise::CreateAndResolve(true, __func__);
}
if (mOutputCaptureState != MediaDecoder::OutputCaptureState::None) {
// Nothing to do.
return GenericPromise::CreateAndResolve(IsPlaying(), __func__);
}
if (mSinkDevice.Ref() != aDevice) {
// A new sink was set before this ran.
return GenericPromise::CreateAndResolve(IsPlaying(), __func__);
}
if (mMediaSink->AudioDevice() == aDevice) {
// The sink has not changed.
return GenericPromise::CreateAndResolve(IsPlaying(), __func__);
}
const bool wasPlaying = IsPlaying();
// Stop and shutdown the existing sink.
StopMediaSink();
mMediaSink->Shutdown();
// Create a new sink according to whether audio is captured.
mMediaSink = CreateMediaSink();
// Start the new sink
if (wasPlaying) {
nsresult rv = StartMediaSink();
if (NS_FAILED(rv)) {
return GenericPromise::CreateAndReject(NS_ERROR_ABORT, __func__);
}
}
return GenericPromise::CreateAndResolve(wasPlaying, __func__);
return mMediaSink->SetAudioDevice(std::move(aDevice));
}
void MediaDecoderStateMachine::InvokeSuspendMediaSink() {

10
dom/media/MediaDecoderStateMachine.h
View File

@ -226,15 +226,7 @@ class MediaDecoderStateMachine
void SetVideoDecodeModeInternal(VideoDecodeMode aMode);
// Set new sink device and restart MediaSink if playback is started.
// Returned promise will be resolved with true if the playback is
// started and false if playback is stopped after setting the new sink.
// Returned promise will be rejected with value NS_ERROR_ABORT
// if the action fails or it is not supported.
// If there are multiple pending requests only the last one will be
// executed, for all previous requests the promise will be resolved
// with true or false similar to above.
RefPtr<GenericPromise> SetSink(const RefPtr<AudioDeviceInfo>& aDevice);
RefPtr<GenericPromise> SetSink(RefPtr<AudioDeviceInfo> aDevice);
// Shutdown MediaSink on suspend to clean up resources.
void SuspendMediaSink();

9
dom/media/MediaDecoderStateMachineBase.h
View File

@ -93,6 +93,15 @@ class MediaDecoderStateMachineBase {
// Sets the video decode mode. Used by the suspend-video-decoder feature.
virtual void SetVideoDecodeMode(VideoDecodeMode aMode) = 0;
// Set new sink device. ExternalEngineStateMachine will reject the returned
// promise with NS_ERROR_ABORT to indicate that the action is not supported.
// MediaDecoderStateMachine will resolve the promise when the previous
// device is no longer in use and an attempt to open the new device
// completes (successfully or not) or is deemed unnecessary because the
// device is not required for output at this time. MediaDecoderStateMachine
// will always consider the switch in underlying output device successful
// and continue attempting to open the new device even if opening initially
// fails.
virtual RefPtr<GenericPromise> InvokeSetSink(
const RefPtr<AudioDeviceInfo>& aSink) = 0;
virtual void InvokeSuspendMediaSink() = 0;

178
dom/media/mediasink/AudioSinkWrapper.cpp
View File

@ -128,8 +128,9 @@ TimeUnit AudioSinkWrapper::GetPosition(TimeStamp* aTimeStamp) {
}
mLastClockSource = ClockSource::SystemClock;
if (!mAudioSink && mAsyncCreateCount == 0 && t > mRetrySinkTime) {
MaybeAsyncCreateAudioSink();
if (!mAudioSink && mAsyncCreateCount == 0 && NeedAudioSink() &&
t > mRetrySinkTime) {
MaybeAsyncCreateAudioSink(mAudioDevice);
}
} else {
// Return how long we've played if we are not playing.
@ -195,7 +196,7 @@ void AudioSinkWrapper::OnMuted(bool aMuted) {
} else {
LOG("%p: AudioSinkWrapper unmuted, maybe re-creating an AudioStream.",
this);
MaybeAsyncCreateAudioSink();
MaybeAsyncCreateAudioSink(mAudioDevice);
}
}
@ -283,6 +284,11 @@ void AudioSinkWrapper::SetPlaying(bool aPlaying) {
}
}
RefPtr<GenericPromise> AudioSinkWrapper::SetAudioDevice(
RefPtr<AudioDeviceInfo> aDevice) {
return MaybeAsyncCreateAudioSink(std::move(aDevice));
}
double AudioSinkWrapper::PlaybackRate() const {
AssertOwnerThread();
return mParams.mPlaybackRate;
@ -342,89 +348,105 @@ void AudioSinkWrapper::ShutDownAudioSink() {
mAudioSink = nullptr;
}
void AudioSinkWrapper::MaybeAsyncCreateAudioSink() {
MOZ_ASSERT(!mAudioSink);
MOZ_ASSERT(!mAudioSinkEndedRequest.Exists());
if (!NeedAudioSink()) {
LOG("%p: AudioSinkWrapper::MaybeAsyncCreateAudioSink: AudioSink not needed",
RefPtr<GenericPromise> AudioSinkWrapper::MaybeAsyncCreateAudioSink(
RefPtr<AudioDeviceInfo> aDevice) {
UniquePtr<AudioSink> audioSink;
if (NeedAudioSink() && (!mAudioSink || aDevice != mAudioDevice)) {
LOG("%p: AudioSinkWrapper::MaybeAsyncCreateAudioSink: AudioSink needed",
this);
return;
if (mAudioSink) {
ShutDownAudioSink();
}
audioSink = mSinkCreator();
} else {
LOG("%p: AudioSinkWrapper::MaybeAsyncCreateAudioSink: no AudioSink change",
this);
// Bounce off the background thread to keep promise resolution in order.
}
LOG("%p: AudioSinkWrapper::MaybeAsyncCreateAudioSink: AudioSink needed",
this);
mAudioDevice = std::move(aDevice);
++mAsyncCreateCount;
UniquePtr<AudioSink> audioSink = mSinkCreator();
using Promise =
MozPromise<UniquePtr<AudioSink>, nsresult, /* IsExclusive = */ true>;
InvokeAsync(
mAsyncInitTaskQueue,
"MaybeAsyncCreateAudioSink (Async part: initialization)",
[self = RefPtr<AudioSinkWrapper>(this), audioSink{std::move(audioSink)},
this]() mutable {
LOG("AudioSink initialization on background thread");
// This can take about 200ms, e.g. on Windows, we don't want to do
// it on the MDSM thread, because it would make the clock not update
// for that amount of time, and the video would therefore not
// update. The Start() call is very cheap on the other hand, we can
// do it from the MDSM thread.
nsresult rv = audioSink->InitializeAudioStream(
mParams, mAudioDevice, AudioSink::InitializationType::UNMUTING);
if (NS_FAILED(rv)) {
LOG("Async AudioSink initialization failed");
return Promise::CreateAndReject(rv, __func__);
}
return Promise::CreateAndResolve(std::move(audioSink), __func__);
})
->Then(mOwnerThread,
"MaybeAsyncCreateAudioSink (Async part: start from MDSM thread)",
[self = RefPtr<AudioSinkWrapper>(this),
this](Promise::ResolveOrRejectValue&& aValue) mutable {
LOG("AudioSink async init done, back on MDSM thread");
ScopeExit decr([&] { --mAsyncCreateCount; });
return InvokeAsync(mAsyncInitTaskQueue,
"MaybeAsyncCreateAudioSink (Async part: initialization)",
[self = RefPtr<AudioSinkWrapper>(this),
audioSink{std::move(audioSink)},
audioDevice = mAudioDevice, this]() mutable {
if (!audioSink) {
return Promise::CreateAndResolve(nullptr, __func__);
}
if (aValue.IsReject()) {
if (mAudioDevice) {
// Device will be started when available again.
ScheduleRetrySink();
return;
}
// Default device not available. Report error.
mEndedPromiseHolder.RejectIfExists(aValue.RejectValue(),
__func__);
return;
}
LOG("AudioSink initialization on background thread");
// This can take about 200ms, e.g. on Windows, we don't
// want to do it on the MDSM thread, because it would
// make the clock not update for that amount of time, and
// the video would therefore not update. The Start() call
// is very cheap on the other hand, we can do it from the
// MDSM thread.
nsresult rv = audioSink->InitializeAudioStream(
mParams, audioDevice,
AudioSink::InitializationType::UNMUTING);
if (NS_FAILED(rv)) {
LOG("Async AudioSink initialization failed");
return Promise::CreateAndReject(rv, __func__);
}
return Promise::CreateAndResolve(std::move(audioSink),
__func__);
})
->Then(
mOwnerThread,
"MaybeAsyncCreateAudioSink (Async part: start from MDSM thread)",
[self = RefPtr<AudioSinkWrapper>(this), audioDevice = mAudioDevice,
this](Promise::ResolveOrRejectValue&& aValue) mutable {
LOG("AudioSink async init done, back on MDSM thread");
ScopeExit decr([&] { --mAsyncCreateCount; });
UniquePtr audioSink = std::move(aValue.ResolveValue());
// It's possible that the newly created isn't needed at this
// point, in some cases:
// 1. An AudioSink was created synchronously while this
// AudioSink was initialized asynchronously, bail out here. This
// happens when seeking (which does a synchronous
// initialization) right after unmuting.
// 2. The media element was muted while the async initialization
// was happening.
// 3. The AudioSinkWrapper was paused or stopped during
// asynchronous initialization.
// 4. The audio has ended during asynchronous initialization.
if (mAudioSink || !NeedAudioSink()) {
LOG("AudioSink initialized async isn't needed, shutting "
"it down.");
audioSink->ShutDown();
return;
}
if (aValue.IsReject()) {
if (audioDevice) {
// Device will be started when available again.
ScheduleRetrySink();
return GenericPromise::CreateAndResolve(true, __func__);
}
// Default device not available. Report error.
mEndedPromiseHolder.RejectIfExists(aValue.RejectValue(),
__func__);
return GenericPromise::CreateAndResolve(true, __func__);
}
MOZ_ASSERT(!mAudioSink);
TimeUnit switchTime = GetPosition();
DropAudioPacketsIfNeeded(switchTime);
if (mTreatUnderrunAsSilence) {
audioSink->EnableTreatAudioUnderrunAsSilence(
mTreatUnderrunAsSilence);
}
LOG("AudioSink async, start");
StartAudioSink(std::move(audioSink), switchTime);
});
UniquePtr audioSink = std::move(aValue.ResolveValue());
if (!audioSink) {
return GenericPromise::CreateAndResolve(true, __func__);
}
// It's possible that the newly created isn't needed at this
// point, in some cases:
// 1. An AudioSink was created synchronously while this
// AudioSink was initialized asynchronously, bail out here. This
// happens when seeking (which does a synchronous
// initialization) right after unmuting.
// 2. The media element was muted while the async initialization
// was happening.
// 3. The AudioSinkWrapper was paused or stopped during
// asynchronous initialization.
// 4. The audio has ended during asynchronous initialization.
// 5. A change to a potentially different sink device is pending.
if (mAudioSink || !NeedAudioSink() || audioDevice != mAudioDevice) {
LOG("AudioSink initialized async isn't needed, shutting "
"it down.");
audioSink->ShutDown();
return GenericPromise::CreateAndResolve(true, __func__);
}
MOZ_ASSERT(!mAudioSink);
TimeUnit switchTime = GetPosition();
DropAudioPacketsIfNeeded(switchTime);
if (mTreatUnderrunAsSilence) {
audioSink->EnableTreatAudioUnderrunAsSilence(
mTreatUnderrunAsSilence);
}
LOG("AudioSink async, start");
StartAudioSink(std::move(audioSink), switchTime);
return GenericPromise::CreateAndResolve(true, __func__);
});
}
nsresult AudioSinkWrapper::SyncCreateAudioSink(const TimeUnit& aStartTime) {

15
dom/media/mediasink/AudioSinkWrapper.h
View File

@ -54,6 +54,8 @@ class AudioSinkWrapper : public MediaSink {
void SetPlaybackRate(double aPlaybackRate) override;
void SetPreservesPitch(bool aPreservesPitch) override;
void SetPlaying(bool aPlaying) override;
RefPtr<GenericPromise> SetAudioDevice(
RefPtr<AudioDeviceInfo> aDevice) override;
double PlaybackRate() const override;
@ -63,8 +65,6 @@ class AudioSinkWrapper : public MediaSink {
bool IsStarted() const override;
bool IsPlaying() const override;
const AudioDeviceInfo* AudioDevice() const override { return mAudioDevice; }
void Shutdown() override;
void GetDebugInfo(dom::MediaSinkDebugInfo& aInfo) override;
@ -103,9 +103,14 @@ class AudioSinkWrapper : public MediaSink {
// and when seeking.
// In asynchronous mode, the clock will keep going forward (using the system
// clock) until the AudioSink is started, at which point the clock will use
// the AudioSink clock. This is used when unmuting a media element.
// the AudioSink clock. This is used when unmuting a media element or
// switching audio output devices. The promise is resolved when the
// previous device is no longer in use and an attempt to open the new device
// completes (successfully or not) or is deemed unnecessary because the
// device is not required for output at this time.
nsresult SyncCreateAudioSink(const media::TimeUnit& aStartTime);
void MaybeAsyncCreateAudioSink();
RefPtr<GenericPromise> MaybeAsyncCreateAudioSink(
RefPtr<AudioDeviceInfo> aDevice);
void ScheduleRetrySink();
// Get the current media position using the system clock. This is used when
@ -124,7 +129,7 @@ class AudioSinkWrapper : public MediaSink {
UniquePtr<AudioSink> mAudioSink;
// The output device this AudioSink is playing data to. The system's default
// device is used if this is null.
const RefPtr<AudioDeviceInfo> mAudioDevice;
RefPtr<AudioDeviceInfo> mAudioDevice;
// Will only exist when media has an audio track.
RefPtr<EndedPromise> mEndedPromise;
MozPromiseHolder<EndedPromise> mEndedPromiseHolder;

9
dom/media/mediasink/DecodedStream.cpp
View File

@ -465,8 +465,7 @@ DecodedStream::DecodedStream(
mPlaybackRate(aPlaybackRate),
mPreservesPitch(aPreservesPitch),
mAudioQueue(aAudioQueue),
mVideoQueue(aVideoQueue),
mAudioDevice(std::move(aAudioDevice)) {}
mVideoQueue(aVideoQueue) {}
DecodedStream::~DecodedStream() {
MOZ_ASSERT(mStartTime.isNothing(), "playback should've ended.");
@ -722,6 +721,12 @@ void DecodedStream::SetPreservesPitch(bool aPreservesPitch) {
}
}
RefPtr<GenericPromise> DecodedStream::SetAudioDevice(
RefPtr<AudioDeviceInfo> aDevice) {
// All audio is captured, so nothing is actually played out, so nothing to do.
return GenericPromise::CreateAndResolve(true, __func__);
}
double DecodedStream::PlaybackRate() const {
AssertOwnerThread();
return mPlaybackRate;

9
dom/media/mediasink/DecodedStream.h
View File

@ -60,6 +60,8 @@ class DecodedStream : public MediaSink {
void SetPlaybackRate(double aPlaybackRate) override;
void SetPreservesPitch(bool aPreservesPitch) override;
void SetPlaying(bool aPlaying) override;
RefPtr<GenericPromise> SetAudioDevice(
RefPtr<AudioDeviceInfo> aDevice) override;
double PlaybackRate() const override;
@ -70,7 +72,6 @@ class DecodedStream : public MediaSink {
bool IsPlaying() const override;
void Shutdown() override;
void GetDebugInfo(dom::MediaSinkDebugInfo& aInfo) override;
const AudioDeviceInfo* AudioDevice() const override { return mAudioDevice; }
MediaEventSource<bool>& AudibleEvent() { return mAudibleEvent; }
@ -136,12 +137,6 @@ class DecodedStream : public MediaSink {
MediaQueue<AudioData>& mAudioQueue;
MediaQueue<VideoData>& mVideoQueue;
// This is the audio device we were told to play out to.
// All audio is captured, so nothing is actually played out -- but we report
// this upwards as it could save us from being recreated when the sink
// changes.
const RefPtr<AudioDeviceInfo> mAudioDevice;
MediaEventListener mAudioPushListener;
MediaEventListener mVideoPushListener;
MediaEventListener mAudioFinishListener;

15
dom/media/mediasink/MediaSink.h
View File

@ -93,6 +93,17 @@ class MediaSink {
// Pause/resume the playback. Only work after playback starts.
virtual void SetPlaying(bool aPlaying) = 0;
// Set the audio output device. aDevice == nullptr indicates the default
// device. The returned promise is resolved when the previous device is no
// longer in use and an attempt to open the new device completes
// (successfully or not) or is deemed unnecessary because the device is not
// required for output at this time. The promise will be resolved whether
// or not opening the cubeb_stream succeeds because the aDevice is
// considered the new underlying device and will be used when it becomes
// available.
virtual RefPtr<GenericPromise> SetAudioDevice(
RefPtr<AudioDeviceInfo> aDevice) = 0;
// Get the playback rate.
// Can be called in any state.
virtual double PlaybackRate() const = 0;
@ -121,10 +132,6 @@ class MediaSink {
// Can be called in any state.
virtual bool IsPlaying() const = 0;
// The audio output device this MediaSink is playing audio data to. The
// default device is used if this returns null.
virtual const AudioDeviceInfo* AudioDevice() const = 0;
// Called on the state machine thread to shut down the sink. All resources
// allocated by this sink should be released.
// Must be called after playback stopped.

10
dom/media/mediasink/VideoSink.cpp
View File

@ -12,6 +12,7 @@
#include "VideoSink.h"
#include "AudioDeviceInfo.h"
#include "MediaQueue.h"
#include "VideoUtils.h"
@ -160,6 +161,11 @@ void VideoSink::SetPreservesPitch(bool aPreservesPitch) {
mAudioSink->SetPreservesPitch(aPreservesPitch);
}
RefPtr<GenericPromise> VideoSink::SetAudioDevice(
RefPtr<AudioDeviceInfo> aDevice) {
return mAudioSink->SetAudioDevice(std::move(aDevice));
}
double VideoSink::PlaybackRate() const {
AssertOwnerThread();
@ -297,10 +303,6 @@ bool VideoSink::IsPlaying() const {
return mAudioSink->IsPlaying();
}
const AudioDeviceInfo* VideoSink::AudioDevice() const {
return mAudioSink->AudioDevice();
}
void VideoSink::Shutdown() {
AssertOwnerThread();
MOZ_ASSERT(!mAudioSink->IsStarted(), "must be called after playback stops.");

5
dom/media/mediasink/VideoSink.h
View File

@ -51,6 +51,9 @@ class VideoSink : public MediaSink {
void SetPlaying(bool aPlaying) override;
RefPtr<GenericPromise> SetAudioDevice(
RefPtr<AudioDeviceInfo> aDevice) override;
double PlaybackRate() const override;
void Redraw(const VideoInfo& aInfo) override;
@ -64,8 +67,6 @@ class VideoSink : public MediaSink {
bool IsPlaying() const override;
const AudioDeviceInfo* AudioDevice() const override;
void Shutdown() override;
void SetSecondaryVideoContainer(VideoFrameContainer* aSecondary) override;