Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-13 21:35:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1397793 - Move to APM - Part 1 - UpdateSingleSource. r=pehrsons

Browse Source 
This part is about setting on/off audio processing feature. It's long, but
it's mostly mechanichal changes, from the old API to the new one.

This also covers reseting the processing in case of device changes (with
macros).

MozReview-Commit-ID: EI2TxHRicEr

--HG--
extra : rebase_source : 7044c2d1695cdf0d6a69b4faa19349e3261ef204
extra : histedit_source : f5ac61e7b90ab4d5280623095c443529fb36cde5%2C5c969f1833bdc425842f945a5a8a4702ca13cd56
This commit is contained in:
Paul Adenot 2017-10-31 18:25:41 +01:00
parent 0ade31e233
commit b76152fc03
3 changed files with 185 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dom/media/webrtc/MediaEngineWebRTC.h
View File

@ -513,9 +513,10 @@ private:
// Note: shared across all microphone sources
static int sChannelsOpen;
const UniquePtr<webrtc::AudioProcessing> mAudioProcessing;
// accessed from the GraphDriver thread except for deletion
nsAutoPtr<AudioPacketizer<AudioDataValue, int16_t>> mPacketizer;
nsAutoPtr<AudioPacketizer<AudioDataValue, AudioDataValue>> mPacketizer;
ScopedCustomReleasePtr<webrtc::VoEExternalMedia> mVoERenderListener;
// mMonitor protects mSources[] and mPrinicpalIds[] access/changes, and
@ -539,8 +540,6 @@ private:
uint64_t mTotalFrames;
uint64_t mLastLogFrames;
NullTransport *mNullTransport;
// mSkipProcessing is true if none of the processing passes are enabled,
// because of prefs or constraints. This allows simply copying the audio into
// the MSG, skipping resampling and the whole webrtc.org code.
@ -549,7 +548,8 @@ private:
// To only update microphone when needed, we keep track of previous settings.
MediaEnginePrefs mLastPrefs;
AlignedShortBuffer mInputDownmixBuffer;
AlignedFloatBuffer mInputBuffer;
AlignedFloatBuffer mInputDownmixBuffer;
};
class MediaEngineWebRTC : public MediaEngine

251
dom/media/webrtc/MediaEngineWebRTCAudio.cpp
View File

@ -16,6 +16,11 @@
#undef FF
#endif
#include "webrtc/modules/audio_device/opensl/single_rw_fifo.h"
#include "webrtc/voice_engine/voice_engine_defines.h"
#include "webrtc/modules/audio_processing/include/audio_processing.h"
#include "webrtc/common_audio/include/audio_util.h"
using namespace webrtc;
#define CHANNELS 1
#define ENCODING "L16"
@ -63,7 +68,7 @@ AudioOutputObserver::AudioOutputObserver()
, mDownmixBuffer(MAX_SAMPLING_FREQ * MAX_CHANNELS / 100)
{
// Buffers of 10ms chunks
mPlayoutFifo = new webrtc::SingleRwFifo(MAX_AEC_FIFO_DEPTH/10);
mPlayoutFifo = new SingleRwFifo(MAX_AEC_FIFO_DEPTH/10);
}
AudioOutputObserver::~AudioOutputObserver()
@ -191,6 +196,7 @@ MediaEngineWebRTCMicrophoneSource::MediaEngineWebRTCMicrophoneSource(
bool aExtendedFilter)
: MediaEngineAudioSource(kReleased)
, mAudioInput(aAudioInput)
, mAudioProcessing(AudioProcessing::Create())
, mMonitor("WebRTCMic.Monitor")
, mCapIndex(aIndex)
, mDelayAgnostic(aDelayAgnostic)
@ -266,6 +272,140 @@ bool operator == (const MediaEnginePrefs& a, const MediaEnginePrefs& b)
return !memcmp(&a, &b, sizeof(MediaEnginePrefs));
};
// This does an early return in case of error.
#define HANDLE_APM_ERROR(fn) \
do { \
int rv = fn; \
if (rv != AudioProcessing::kNoError) { \
MOZ_ASSERT_UNREACHABLE("APM error in " #fn); \
return; \
} \
} while(0);
void MediaEngineWebRTCMicrophoneSource::UpdateAECSettingsIfNeeded(bool aEnable, EcModes aMode)
{
using webrtc::EcModes;
EchoCancellation::SuppressionLevel level;
switch(aMode) {
case EcModes::kEcUnchanged:
level = mAudioProcessing->echo_cancellation()->suppression_level();
break;
case EcModes::kEcConference:
level = EchoCancellation::kHighSuppression;
break;
case EcModes::kEcDefault:
level = EchoCancellation::kModerateSuppression;
break;
case EcModes::kEcAec:
level = EchoCancellation::kModerateSuppression;
break;
case EcModes::kEcAecm:
// No suppression level to set for the mobile echo canceller
break;
default:
MOZ_LOG(GetMediaManagerLog(), LogLevel::Error, ("Bad EcMode value"));
MOZ_ASSERT_UNREACHABLE("Bad pref set in all.js or in about:config"
" for the echo cancelation mode.");
// fall back to something sensible in release
level = EchoCancellation::kModerateSuppression;
break;
}
// AECm and AEC are mutually exclusive.
if (aMode == EcModes::kEcAecm) {
HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->Enable(false));
HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->Enable(aEnable));
} else {
HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->Enable(false));
HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->Enable(aEnable));
HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->set_suppression_level(level));
}
}
void
MediaEngineWebRTCMicrophoneSource::UpdateAGCSettingsIfNeeded(bool aEnable, AgcModes aMode)
{
#if defined(WEBRTC_IOS) || defined(ATA) || defined(WEBRTC_ANDROID)
if (aMode == kAgcAdaptiveAnalog) {
MOZ_LOG(GetMediaManagerLog(),
LogLevel::Error,
("Invalid AGC mode kAgcAdaptiveAnalog on mobile"));
MOZ_ASSERT_UNREACHABLE("Bad pref set in all.js or in about:config"
" for the auto gain, on mobile.");
aMode = kAgcDefault;
}
#endif
GainControl::Mode mode = kDefaultAgcMode;
switch (aMode) {
case AgcModes::kAgcDefault:
mode = kDefaultAgcMode;
break;
case AgcModes::kAgcUnchanged:
mode = mAudioProcessing->gain_control()->mode();
break;
case AgcModes::kAgcFixedDigital:
mode = GainControl::Mode::kFixedDigital;
break;
case AgcModes::kAgcAdaptiveAnalog:
mode = GainControl::Mode::kAdaptiveAnalog;
break;
case AgcModes::kAgcAdaptiveDigital:
mode = GainControl::Mode::kAdaptiveDigital;
break;
default:
MOZ_ASSERT_UNREACHABLE("Bad pref set in all.js or in about:config"
" for the auto gain.");
// This is a good fallback, it works regardless of the platform.
mode = GainControl::Mode::kAdaptiveDigital;
break;
}
HANDLE_APM_ERROR(mAudioProcessing->gain_control()->set_mode(mode));
HANDLE_APM_ERROR(mAudioProcessing->gain_control()->Enable(aEnable));
}
void
MediaEngineWebRTCMicrophoneSource::UpdateNSSettingsIfNeeded(bool aEnable, NsModes aMode)
{
NoiseSuppression::Level nsLevel;
switch (aMode) {
case NsModes::kNsDefault:
nsLevel = kDefaultNsMode;
break;
case NsModes::kNsUnchanged:
nsLevel = mAudioProcessing->noise_suppression()->level();
break;
case NsModes::kNsConference:
nsLevel = NoiseSuppression::kHigh;
break;
case NsModes::kNsLowSuppression:
nsLevel = NoiseSuppression::kLow;
break;
case NsModes::kNsModerateSuppression:
nsLevel = NoiseSuppression::kModerate;
break;
case NsModes::kNsHighSuppression:
nsLevel = NoiseSuppression::kHigh;
break;
case NsModes::kNsVeryHighSuppression:
nsLevel = NoiseSuppression::kVeryHigh;
break;
default:
MOZ_ASSERT_UNREACHABLE("Bad pref set in all.js or in about:config"
" for the noise suppression.");
// Pick something sensible as a faillback in release.
nsLevel = NoiseSuppression::kModerate;
}
HANDLE_APM_ERROR(mAudioProcessing->noise_suppression()->set_level(nsLevel));
HANDLE_APM_ERROR(mAudioProcessing->noise_suppression()->Enable(aEnable));
}
#undef HANDLE_APM_ERROR
nsresult
MediaEngineWebRTCMicrophoneSource::UpdateSingleSource(
const AllocationHandle* aHandle,
@ -314,24 +454,12 @@ MediaEngineWebRTCMicrophoneSource::UpdateSingleSource(
// (Bug 1238038) fail allocation for a second device
return NS_ERROR_FAILURE;
}
if (mAudioInput->SetRecordingDevice(mCapIndex)) {
return NS_ERROR_FAILURE;
}
mAudioInput->SetUserChannelCount(prefs.mChannels);
if (!AllocChannel()) {
FreeChannel();
LOG(("Audio device is not initalized"));
return NS_ERROR_FAILURE;
}
LOG(("Audio device %d allocated", mCapIndex));
{
// Update with the actual applied channelCount in order
// to store it in settings.
uint32_t channelCount = 0;
mAudioInput->GetChannelCount(channelCount);
MOZ_ASSERT(channelCount > 0);
prefs.mChannels = channelCount;
}
break;
case kStarted:
@ -341,19 +469,19 @@ MediaEngineWebRTCMicrophoneSource::UpdateSingleSource(
if (prefs.mChannels != mLastPrefs.mChannels) {
MOZ_ASSERT(mSources.Length() > 0);
// If the channel count changed, tell the MSG to open a new driver with
// the correct channel count.
auto& source = mSources.LastElement();
mAudioInput->SetUserChannelCount(prefs.mChannels);
// Get validated number of channel
uint32_t channelCount = 0;
mAudioInput->GetChannelCount(channelCount);
MOZ_ASSERT(channelCount > 0 && mLastPrefs.mChannels > 0);
// Check if new validated channels is the same as previous
if (static_cast<uint32_t>(mLastPrefs.mChannels) != channelCount &&
if (mLastPrefs.mChannels != prefs.mChannels &&
!source->OpenNewAudioCallbackDriver(mListener)) {
MOZ_LOG(GetMediaManagerLog(), LogLevel::Error, ("Could not open a new AudioCallbackDriver for input"));
return NS_ERROR_FAILURE;
}
// Update settings
prefs.mChannels = channelCount;
}
if (MOZ_LOG_TEST(GetMediaManagerLog(), LogLevel::Debug)) {
@ -372,46 +500,22 @@ MediaEngineWebRTCMicrophoneSource::UpdateSingleSource(
}
if (sChannelsOpen > 0) {
int error;
UpdateAGCSettingsIfNeeded(prefs.mAgcOn, static_cast<AgcModes>(prefs.mAgc));
UpdateNSSettingsIfNeeded(prefs.mNoiseOn, static_cast<NsModes>(prefs.mNoise));
UpdateAECSettingsIfNeeded(prefs.mAecOn, static_cast<EcModes>(prefs.mAec));
error = mVoEProcessing->SetEcStatus(prefs.mAecOn, (webrtc::EcModes)prefs.mAec);
if (error) {
LOG(("%s Error setting Echo Status: %d ",__FUNCTION__, error));
// Overhead of capturing all the time is very low (<0.1% of an audio only call)
if (prefs.mAecOn) {
error = mVoEProcessing->SetEcMetricsStatus(true);
if (error) {
LOG(("%s Error setting Echo Metrics: %d ",__FUNCTION__, error));
}
}
}
error = mVoEProcessing->SetAgcStatus(prefs.mAgcOn, (webrtc::AgcModes)prefs.mAgc);
if (error) {
LOG(("%s Error setting AGC Status: %d ",__FUNCTION__, error));
}
error = mVoEProcessing->SetNsStatus(prefs.mNoiseOn, (webrtc::NsModes)prefs.mNoise);
if (error) {
LOG(("%s Error setting NoiseSuppression Status: %d ",__FUNCTION__, error));
}
webrtc::Config config;
config.Set<webrtc::ExtendedFilter>(new webrtc::ExtendedFilter(mExtendedFilter));
config.Set<webrtc::DelayAgnostic>(new webrtc::DelayAgnostic(mDelayAgnostic));
mAudioProcessing->SetExtraOptions(config);
}
// we don't allow switching from non-fast-path to fast-path on the fly yet
if (mState != kStarted) {
mSkipProcessing = !(prefs.mAecOn || prefs.mAgcOn || prefs.mNoiseOn);
if (mSkipProcessing) {
mSampleFrequency = MediaEngine::USE_GRAPH_RATE;
} else {
// make sure we route a copy of the mixed audio output of this MSG to the
// AEC
if (!mAudioOutputObserver) {
mAudioOutputObserver = new AudioOutputObserver();
}
}
}
SetLastPrefs(prefs);
return NS_OK;
}
#undef HANDLE_APM_ERROR
void
MediaEngineWebRTCMicrophoneSource::SetLastPrefs(
const MediaEnginePrefs& aPrefs)
@ -579,13 +683,25 @@ MediaEngineWebRTCMicrophoneSource::PacketizeAndProcess(MediaStreamGraph* aGraph,
TrackRate aRate,
uint32_t aChannels)
{
// This will call Process() with data coming out of the AEC/NS/AGC/etc chain
MOZ_ASSERT(!PassThrough(), "This should be bypassed when in PassThrough mode.");
size_t offset = 0;
if (!mPacketizer ||
mPacketizer->PacketSize() != aRate/100u ||
mPacketizer->Channels() != aChannels) {
// It's ok to drop the audio still in the packetizer here.
mPacketizer =
new AudioPacketizer<AudioDataValue, int16_t>(aRate/100, aChannels);
new AudioPacketizer<AudioDataValue, AudioDataValue>(aRate/100, aChannels);
}
// On initial capture, throw away all far-end data except the most recent sample
// since it's already irrelevant and we want to keep avoid confusing the AEC far-end
// input code with "old" audio.
if (!mStarted) {
mStarted = true;
while (mAudioOutputObserver->Size() > 1) {
free(mAudioOutputObserver->Pop()); // only call if size() > 0
}
}
mPacketizer->Input(aBuffer, static_cast<uint32_t>(aFrames));
@ -704,38 +820,31 @@ MediaEngineWebRTCMicrophoneSource::NotifyInputData(MediaStreamGraph* aGraph,
#define ResetProcessingIfNeeded(_processing) \
do { \
webrtc::_processing##Modes mode; \
int rv = mVoEProcessing->Get##_processing##Status(enabled, mode); \
if (rv) { \
NS_WARNING("Could not get the status of the " \
#_processing " on device change."); \
return; \
} \
bool enabled = mAudioProcessing->_processing()->is_enabled(); \
\
if (enabled) { \
rv = mVoEProcessing->Set##_processing##Status(!enabled); \
int rv = mAudioProcessing->_processing()->Enable(!enabled); \
if (rv) { \
NS_WARNING("Could not reset the status of the " \
#_processing " on device change."); \
return; \
} \
rv = mAudioProcessing->_processing()->Enable(enabled); \
if (rv) { \
NS_WARNING("Could not reset the status of the " \
#_processing " on device change."); \
return; \
} \
\
rv = mVoEProcessing->Set##_processing##Status(enabled); \
if (rv) { \
NS_WARNING("Could not reset the status of the " \
#_processing " on device change."); \
return; \
} \
} \
} while(0)
void
MediaEngineWebRTCMicrophoneSource::DeviceChanged() {
// Reset some processing
bool enabled;
ResetProcessingIfNeeded(Agc);
ResetProcessingIfNeeded(Ec);
ResetProcessingIfNeeded(Ns);
ResetProcessingIfNeeded(gain_control);
ResetProcessingIfNeeded(echo_cancellation);
ResetProcessingIfNeeded(noise_suppression);
}
// mState records if a channel is allocated (slightly redundantly to mChannel)
@ -796,8 +905,6 @@ MediaEngineWebRTCMicrophoneSource::Shutdown()
Deallocate(mRegisteredHandles[0].get());
}
MOZ_ASSERT(mState == kReleased);
mAudioInput = nullptr;
}
void

4
modules/libpref/init/all.js
View File

@ -496,7 +496,7 @@ pref("media.peerconnection.ice.no_host", false);
pref("media.peerconnection.ice.default_address_only", false);
pref("media.peerconnection.ice.proxy_only", false);
// These values (aec, agc, and noice) are from media/webrtc/trunk/webrtc/common_types.h
// These values (aec, agc, and noise) are from media/webrtc/trunk/webrtc/common_types.h
// kXxxUnchanged = 0, kXxxDefault = 1, and higher values are specific to each
// setting (for Xxx = Ec, Agc, or Ns). Defaults are all set to kXxxDefault here.
pref("media.peerconnection.turn.disable", false);
@ -510,7 +510,7 @@ pref("media.getusermedia.noise_enabled", true);
pref("media.getusermedia.aec_extended_filter", true);
pref("media.getusermedia.noise", 1);
pref("media.getusermedia.agc_enabled", false);
pref("media.getusermedia.agc", 1);
pref("media.getusermedia.agc", 3); // kAgcAdaptiveDigital
// capture_delay: Adjustments for OS-specific input delay (lower bound)
// playout_delay: Adjustments for OS-specific AudioStream+cubeb+output delay (lower bound)
// full_duplex: enable cubeb full-duplex capture/playback