Bug 1531833 - Add a way to tell the graph driver that the audio input is voice. r=pehrsons

Differential Revision: https://phabricator.services.mozilla.com/D21737

--HG--
extra : moz-landing-system : lando

dom/media/GraphDriver.cpp

@@ -469,7 +469,8 @@ StreamAndPromiseForOperation::StreamAndPromiseForOperation(
       mFlags(aFlags) {}
 
 AudioCallbackDriver::AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl,
-                                         uint32_t aInputChannelCount)
+                                         uint32_t aInputChannelCount,
+                                         AudioInputType aAudioInputType)
     : GraphDriver(aGraphImpl),
       mOutputChannels(0),
       mSampleRate(0),

dom/media/GraphDriver.h

@@ -323,6 +323,7 @@ struct StreamAndPromiseForOperation {
 };
 
 enum AsyncCubebOperation { INIT, SHUTDOWN };
+enum class AudioInputType { Unknown, Voice };
 
 /**
  * This is a graph driver that is based on callback functions called by the
@@ -354,7 +355,8 @@ class AudioCallbackDriver : public GraphDriver,
  public:
   /** If aInputChannelCount is zero, then this driver is output-only. */
   AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl,
-                      uint32_t aInputChannelCount);
+                      uint32_t aInputChannelCount,
+                      AudioInputType aAudioInputType);
   virtual ~AudioCallbackDriver();
 
   void Start() override;
@@ -402,6 +404,13 @@ class AudioCallbackDriver : public GraphDriver,
 
   uint32_t InputChannelCount() { return mInputChannelCount; }
 
+  AudioInputType InputDevicePreference() {
+    if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
+      return AudioInputType::Voice;
+    }
+    return AudioInputType::Unknown;
+  }
+
   /* Enqueue a promise that is going to be resolved when a specific operation
    * occurs on the cubeb stream. */
   void EnqueueStreamAndPromiseForOperation(
@@ -503,12 +512,12 @@ class AudioCallbackDriver : public GraphDriver,
   const RefPtr<SharedThreadPool> mInitShutdownThread;
   /* This must be accessed with the graph monitor held. */
   AutoTArray<StreamAndPromiseForOperation, 1> mPromisesForOperation;
+  cubeb_device_pref mInputDevicePreference;
   /* This is used to signal adding the mixer callback on first run
    * of audio callback. This is atomic because it is touched from different
    * threads, the audio callback thread and the state change thread. However,
    * the order of the threads does not allow concurent access. */
   Atomic<bool> mAddedMixer;
-
   /* Contains the id of the audio thread for as long as the callback
    * is taking place, after that it is reseted to an invalid value. */
   std::atomic<std::thread::id> mAudioThreadId;

dom/media/MediaStreamGraph.cpp

@@ -368,8 +368,8 @@ void MediaStreamGraphImpl::UpdateStreamOrder() {
       !CurrentDriver()->AsAudioCallbackDriver() && !switching) {
     MonitorAutoLock mon(mMonitor);
     if (LifecycleStateRef() == LIFECYCLE_RUNNING) {
-      AudioCallbackDriver* driver =
-          new AudioCallbackDriver(this, AudioInputChannelCount());
+      AudioCallbackDriver* driver = new AudioCallbackDriver(
+          this, AudioInputChannelCount(), AudioInputDevicePreference());
       CurrentDriver()->SwitchAtNextIteration(driver);
     }
   }
@@ -612,8 +612,8 @@ void MediaStreamGraphImpl::CreateOrDestroyAudioStreams(MediaStream* aStream) {
       if (!CurrentDriver()->AsAudioCallbackDriver() && !switching) {
         MonitorAutoLock mon(mMonitor);
         if (LifecycleStateRef() == LIFECYCLE_RUNNING) {
-          AudioCallbackDriver* driver =
-              new AudioCallbackDriver(this, AudioInputChannelCount());
+          AudioCallbackDriver* driver = new AudioCallbackDriver(
+              this, AudioInputChannelCount(), AudioInputDevicePreference());
           CurrentDriver()->SwitchAtNextIteration(driver);
         }
       }
@@ -746,8 +746,8 @@ void MediaStreamGraphImpl::OpenAudioInputImpl(CubebUtils::AudioDeviceID aID,
     // Switch Drivers since we're adding input (to input-only or full-duplex)
     MonitorAutoLock mon(mMonitor);
     if (LifecycleStateRef() == LIFECYCLE_RUNNING) {
-      AudioCallbackDriver* driver =
-          new AudioCallbackDriver(this, AudioInputChannelCount());
+      AudioCallbackDriver* driver = new AudioCallbackDriver(
+          this, AudioInputChannelCount(), AudioInputDevicePreference());
       LOG(LogLevel::Debug,
           ("%p OpenAudioInput: starting new AudioCallbackDriver(input) %p",
            this, driver));
@@ -830,7 +830,8 @@ void MediaStreamGraphImpl::CloseAudioInputImpl(
       LOG(LogLevel::Debug,
           ("%p: CloseInput: output present (AudioCallback)", this));
 
-      driver = new AudioCallbackDriver(this, AudioInputChannelCount());
+      driver = new AudioCallbackDriver(this, AudioInputChannelCount(),
+                                       AudioInputDevicePreference());
       CurrentDriver()->SwitchAtNextIteration(driver);
     } else if (CurrentDriver()->AsAudioCallbackDriver()) {
       LOG(LogLevel::Debug,
@@ -981,6 +982,10 @@ void MediaStreamGraphImpl::ReevaluateInputDevice() {
     if (audioCallbackDriver->InputChannelCount() != AudioInputChannelCount()) {
       needToSwitch = true;
     }
+    if (audioCallbackDriver->InputDevicePreference() !=
+        AudioInputDevicePreference()) {
+      needToSwitch = true;
+    }
   } else {
     // We're already in the process of switching to a audio callback driver,
     // which will happen at the next iteration.
@@ -993,8 +998,8 @@ void MediaStreamGraphImpl::ReevaluateInputDevice() {
     needToSwitch = true;
   }
   if (needToSwitch) {
-    AudioCallbackDriver* newDriver =
-        new AudioCallbackDriver(this, AudioInputChannelCount());
+    AudioCallbackDriver* newDriver = new AudioCallbackDriver(
+        this, AudioInputChannelCount(), AudioInputDevicePreference());
     {
       MonitorAutoLock lock(mMonitor);
       CurrentDriver()->SwitchAtNextIteration(newDriver);
@@ -3153,8 +3158,9 @@ MediaStreamGraphImpl::MediaStreamGraphImpl(GraphDriverType aDriverRequested,
       mMainThreadGraphTime(0, "MediaStreamGraphImpl::mMainThreadGraphTime") {
   if (mRealtime) {
     if (aDriverRequested == AUDIO_THREAD_DRIVER) {
-      // Always start with zero input channels.
-      mDriver = new AudioCallbackDriver(this, 0);
+      // Always start with zero input channels, and no particular preferences
+      // for the input channel.
+      mDriver = new AudioCallbackDriver(this, 0, AudioInputType::Unknown);
     } else {
       mDriver = new SystemClockDriver(this);
     }
@@ -3645,7 +3651,8 @@ void MediaStreamGraphImpl::ApplyAudioContextOperationImpl(
         MOZ_ASSERT(nextDriver->AsAudioCallbackDriver());
         driver = nextDriver->AsAudioCallbackDriver();
       } else {
-        driver = new AudioCallbackDriver(this, AudioInputChannelCount());
+        driver = new AudioCallbackDriver(this, AudioInputChannelCount(),
+                                         AudioInputDevicePreference());
         MonitorAutoLock lock(mMonitor);
         CurrentDriver()->SwitchAtNextIteration(driver);
       }

dom/media/MediaStreamGraph.h

@@ -122,6 +122,10 @@ class AudioDataListenerInterface {
    */
   virtual uint32_t RequestedInputChannelCount(MediaStreamGraphImpl* aGraph) = 0;
 
+  /**
+   * Whether the underlying audio device is used for voice input.
+   */
+  virtual bool IsVoiceInput(MediaStreamGraphImpl* aGraph) const = 0;
   /**
    * Called when the underlying audio device has changed.
    */

dom/media/MediaStreamGraphImpl.h

@@ -398,6 +398,7 @@ class MediaStreamGraphImpl : public MediaStreamGraph,
    * reevaluated, for example, if the channel count of the input stream should
    * be changed. */
   void ReevaluateInputDevice();
+
   /* Called on the graph thread when there is new output data for listeners.
    * This is the mixed audio output of this MediaStreamGraph. */
   void NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames,
@@ -485,6 +486,29 @@ class MediaStreamGraphImpl : public MediaStreamGraph,
     return maxInputChannels;
   }
 
+  AudioInputType AudioInputDevicePreference() {
+    MOZ_ASSERT(OnGraphThreadOrNotRunning());
+
+    if (!mInputDeviceUsers.GetValue(mInputDeviceID)) {
+      return AudioInputType::Unknown;
+    }
+    bool voiceInput = false;
+    // When/if we decide to support multiple input device per graph, this needs
+    // loop over them.
+    nsTArray<RefPtr<AudioDataListener>>* listeners =
+        mInputDeviceUsers.GetValue(mInputDeviceID);
+    MOZ_ASSERT(listeners);
+
+    // If at least one stream is considered to be voice,
+    for (const auto& listener : *listeners) {
+      voiceInput |= listener->IsVoiceInput(this);
+    }
+    if (voiceInput) {
+      return AudioInputType::Voice;
+    }
+    return AudioInputType::Unknown;
+  }
+
   CubebUtils::AudioDeviceID InputDeviceID() { return mInputDeviceID; }
 
   double MediaTimeToSeconds(GraphTime aTime) const {

dom/media/webrtc/MediaEngineWebRTCAudio.h

@@ -160,6 +160,12 @@ class AudioInputProcessing : public AudioDataListener {
   void NotifyInputData(MediaStreamGraphImpl* aGraph,
                        const AudioDataValue* aBuffer, size_t aFrames,
                        TrackRate aRate, uint32_t aChannels) override;
+  bool IsVoiceInput(MediaStreamGraphImpl* aGraph) const override {
+    // If we're passing data directly without AEC or any other process, this
+    // means that all voice-processing has been disabled intentionaly. In this
+    // case, consider that the device is not used for voice input.
+    return !PassThrough(aGraph);
+  }
 
   void Start();
   void Stop();