Bug 1221587: Implement switching of AudioCallbackDrivers for full-duplex r=padenot

--HG-- extra : commitid : 7b8MdK7aRfk
2024-10-17 15:25:52 +00:00 · 2016-01-21 11:51:36 -05:00 · 2016-01-21 11:51:36 -05:00 · c00d287af2
commit c00d287af2
parent 2501b5c4c1
3 changed files with 116 additions and 100 deletions
--- a/dom/media/GraphDriver.cpp
+++ b/dom/media/GraphDriver.cpp
@ -66,8 +66,14 @@ void GraphDriver::SetGraphTime(GraphDriver* aPreviousDriver,
  mIterationStart = aLastSwitchNextIterationStart;
  mIterationEnd = aLastSwitchNextIterationEnd;

-  STREAM_LOG(LogLevel::Debug, ("Setting previous driver: %p (%s)", PreviousDriver(), PreviousDriver()->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver"));
  MOZ_ASSERT(!PreviousDriver());
+  MOZ_ASSERT(aPreviousDriver);
+
+  STREAM_LOG(LogLevel::Debug, ("Setting previous driver: %p (%s)",
+                               aPreviousDriver,
+                               aPreviousDriver->AsAudioCallbackDriver()
+                                 ? "AudioCallbackDriver"
+                                 : "SystemClockDriver"));
  SetPreviousDriver(aPreviousDriver);
 }

@ -103,37 +109,6 @@ void GraphDriver::EnsureNextIteration()
  mGraphImpl->EnsureNextIteration();
 }

-class MediaStreamGraphShutdownThreadRunnable : public nsRunnable {
-public:
-  explicit MediaStreamGraphShutdownThreadRunnable(GraphDriver* aDriver)
-    : mDriver(aDriver)
-  {
-  }
-  NS_IMETHOD Run()
-  {
-    MOZ_ASSERT(NS_IsMainThread());
-
-    LIFECYCLE_LOG("MediaStreamGraphShutdownThreadRunnable for graph %p",
-        mDriver->GraphImpl());
-    // We can't release an audio driver on the main thread, because it can be
-    // blocking.
-    if (mDriver->AsAudioCallbackDriver()) {
-      LIFECYCLE_LOG("Releasing audio driver off main thread.");
-      RefPtr<AsyncCubebTask> releaseEvent =
-        new AsyncCubebTask(mDriver->AsAudioCallbackDriver(),
-                           AsyncCubebOperation::SHUTDOWN);
-      mDriver = nullptr;
-      releaseEvent->Dispatch();
-    } else {
-      LIFECYCLE_LOG("Dropping driver reference for SystemClockDriver.");
-      mDriver = nullptr;
-    }
-    return NS_OK;
-  }
-private:
-  RefPtr<GraphDriver> mDriver;
-};
-
 void GraphDriver::Shutdown()
 {
  if (AsAudioCallbackDriver()) {
@ -271,6 +246,11 @@ ThreadedDriver::Revive()
  }
 }

+void
+ThreadedDriver::RemoveCallback()
+{
+}
+
 void
 ThreadedDriver::Stop()
 {
@ -338,6 +318,7 @@ ThreadedDriver::RunThread()
    MonitorAutoLock lock(GraphImpl()->GetMonitor());
    if (NextDriver() && stillProcessing) {
      STREAM_LOG(LogLevel::Debug, ("Switching to AudioCallbackDriver"));
+      RemoveCallback();
      NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd);
      mGraphImpl->SetCurrentDriver(NextDriver());
      NextDriver()->Start();
@ -542,6 +523,7 @@ AudioCallbackDriver::AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl)
  , mStarted(false)
  , mAudioInput(nullptr)
  , mAudioChannel(aGraphImpl->AudioChannel())
+  , mAddedMixer(false)
  , mInCallback(false)
  , mMicrophoneActive(false)
 #ifdef XP_MACOSX
@ -616,8 +598,6 @@ AudioCallbackDriver::Init()
    SetNextDriver(new SystemClockDriver(GraphImpl()));
    NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd);
    mGraphImpl->SetCurrentDriver(NextDriver());
-    DebugOnly<bool> found = mGraphImpl->RemoveMixerCallback(this);
-    NS_WARN_IF_FALSE(!found, "Mixer callback not added when switching?");
    NextDriver()->Start();
    return;
  }
@ -650,35 +630,25 @@ AudioCallbackDriver::Resume()
 void
 AudioCallbackDriver::Start()
 {
-  // If this is running on the main thread, we can't open the stream directly,
-  // because it is a blocking operation.
-  if (NS_IsMainThread()) {
-    STREAM_LOG(LogLevel::Debug, ("Starting audio threads for MediaStreamGraph %p from a new thread.", mGraphImpl));
-    RefPtr<AsyncCubebTask> initEvent =
-      new AsyncCubebTask(this, AsyncCubebOperation::INIT);
-    initEvent->Dispatch();
-  } else {
-    STREAM_LOG(LogLevel::Debug, ("Starting audio threads for MediaStreamGraph %p from the previous driver's thread", mGraphImpl));
-    {
-      MonitorAutoUnlock mon(GraphImpl()->GetMonitor());
-      Init();
-    }
-
-    // Check if we need to resolve promises because the driver just got switched
-    // because of a resuming AudioContext
-    if (!mPromisesForOperation.IsEmpty()) {
-      // CompleteAudioContextOperations takes the lock as needed
-      MonitorAutoUnlock mon(GraphImpl()->GetMonitor());
-      CompleteAudioContextOperations(AsyncCubebOperation::INIT);
-    }
-
-    if (PreviousDriver()) {
-      nsCOMPtr<nsIRunnable> event =
-        new MediaStreamGraphShutdownThreadRunnable(PreviousDriver());
-      SetPreviousDriver(nullptr);
-      NS_DispatchToMainThread(event);
+  if (mPreviousDriver) {
+    if (mPreviousDriver->AsAudioCallbackDriver()) {
+      LIFECYCLE_LOG("Releasing audio driver off main thread.");
+      RefPtr<AsyncCubebTask> releaseEvent =
+        new AsyncCubebTask(mPreviousDriver->AsAudioCallbackDriver(),
+                           AsyncCubebOperation::SHUTDOWN);
+      releaseEvent->Dispatch();
+      mPreviousDriver = nullptr;
+    } else {
+      LIFECYCLE_LOG("Dropping driver reference for SystemClockDriver.");
+      mPreviousDriver = nullptr;
    }
  }
+
+  LIFECYCLE_LOG("Starting new audio driver off main thread, "
+                "to ensure it runs after previous shutdown.");
+  RefPtr<AsyncCubebTask> initEvent =
+    new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::INIT);
+  initEvent->Dispatch();
 }

 void
@ -712,6 +682,7 @@ AudioCallbackDriver::Revive()
  // If we were switching, switch now. Otherwise, start the audio thread again.
  MonitorAutoLock mon(mGraphImpl->GetMonitor());
  if (NextDriver()) {
+    RemoveCallback();
    NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd);
    mGraphImpl->SetCurrentDriver(NextDriver());
    NextDriver()->Start();
@ -723,7 +694,17 @@ AudioCallbackDriver::Revive()
  }
 }

-void AudioCallbackDriver::WaitForNextIteration()
+void
+AudioCallbackDriver::RemoveCallback()
+{
+  if (mAddedMixer) {
+    mGraphImpl->mMixer.RemoveCallback(this);
+    mAddedMixer = false;
+  }
+}
+
+void
+AudioCallbackDriver::WaitForNextIteration()
 {
 }

@ -813,6 +794,12 @@ AudioCallbackDriver::DataCallback(AudioDataValue* aInputBuffer,
 {
  bool stillProcessing;

+  // Don't add the callback until we're inited and ready
+  if (!mAddedMixer) {
+    mGraphImpl->mMixer.AddCallback(this);
+    mAddedMixer = true;
+  }
+
 #ifdef XP_MACOSX
  if (OSXDeviceSwitchingWorkaround()) {
    PodZero(aOutputBuffer, aFrames * mGraphImpl->AudioChannelCount());
@ -894,7 +881,7 @@ AudioCallbackDriver::DataCallback(AudioDataValue* aInputBuffer,

    stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime);
  } else {
-    NS_WARNING("DataCallback buffer filled entirely from scratch buffer, skipping iteration.");
+    STREAM_LOG(LogLevel::Verbose, ("DataCallback buffer filled entirely from scratch buffer, skipping iteration."));
    stillProcessing = true;
  }

@ -931,6 +918,7 @@ AudioCallbackDriver::DataCallback(AudioDataValue* aInputBuffer,
      return aFrames;
    }
    STREAM_LOG(LogLevel::Debug, ("Switching to system driver."));
+    RemoveCallback();
    NextDriver()->SetGraphTime(this, mIterationStart, mIterationEnd);
    mGraphImpl->SetCurrentDriver(NextDriver());
    NextDriver()->Start();
@ -1037,6 +1025,7 @@ AudioCallbackDriver::DeviceChangedCallback() {
  mSelfReference.Take(this);
  mCallbackReceivedWhileSwitching = 0;
  SetNextDriver(new SystemClockDriver(GraphImpl()));
+  RemoveCallback();
  mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd);
  mGraphImpl->SetCurrentDriver(mNextDriver);
  mNextDriver->Start();
--- a/dom/media/GraphDriver.h
+++ b/dom/media/GraphDriver.h
@ -122,6 +122,8 @@ public:
  virtual void Resume() = 0;
  /* Revive this driver, as more messages just arrived. */
  virtual void Revive() = 0;
+  /* Remove Mixer callbacks when switching */
+  virtual void RemoveCallback() = 0;
  void Shutdown();
  /* Rate at which the GraphDriver runs, in ms. This can either be user
   * controlled (because we are using a {System,Offline}ClockDriver, and decide
@ -270,6 +272,7 @@ public:
  void Stop() override;
  void Resume() override;
  void Revive() override;
+  void RemoveCallback() override;
  /**
   * Runs main control loop on the graph thread. Normally a single invocation
   * of this runs for the entire lifetime of the graph thread.
@ -381,6 +384,7 @@ public:
  void Stop() override;
  void Resume() override;
  void Revive() override;
+  void RemoveCallback() override;
  void WaitForNextIteration() override;
  void WakeUp() override;

@ -511,8 +515,11 @@ private:
  nsCOMPtr<nsIThread> mInitShutdownThread;
  /* This must be accessed with the graph monitor held. */
  nsAutoTArray<StreamAndPromiseForOperation, 1> mPromisesForOperation;
-  /* This is set during initialization, and ca be read safely afterwards. */
+  /* This is set during initialization, and can be read safely afterwards. */
  dom::AudioChannel mAudioChannel;
+  /* Used to queue us to add the mixer callback on first run. */
+  bool mAddedMixer;
+
  /* This is atomic and is set by the audio callback thread. It can be read by
   * any thread safely. */
  Atomic<bool> mInCallback;
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@ -349,6 +349,8 @@ MediaStreamGraphImpl::UpdateStreamOrder()
      }
    }
  }
+  // Note that this looks for any audio streams, input or output, and switches to a
+  // SystemClockDriver if there are none

  if (!audioTrackPresent && mRealtime &&
      CurrentDriver()->AsAudioCallbackDriver()) {
@ -356,7 +358,6 @@ MediaStreamGraphImpl::UpdateStreamOrder()
    if (CurrentDriver()->AsAudioCallbackDriver()->IsStarted()) {
      if (mLifecycleState == LIFECYCLE_RUNNING) {
        SystemClockDriver* driver = new SystemClockDriver(this);
-        mMixer.RemoveCallback(CurrentDriver()->AsAudioCallbackDriver());
        CurrentDriver()->SwitchAtNextIteration(driver);
      }
    }
@ -374,7 +375,6 @@ MediaStreamGraphImpl::UpdateStreamOrder()
    MonitorAutoLock mon(mMonitor);
    if (mLifecycleState == LIFECYCLE_RUNNING) {
      AudioCallbackDriver* driver = new AudioCallbackDriver(this);
-      mMixer.AddCallback(driver);
      CurrentDriver()->SwitchAtNextIteration(driver);
    }
  }
@ -640,7 +640,6 @@ MediaStreamGraphImpl::CreateOrDestroyAudioStreams(MediaStream* aStream)
        MonitorAutoLock mon(mMonitor);
        if (mLifecycleState == LIFECYCLE_RUNNING) {
          AudioCallbackDriver* driver = new AudioCallbackDriver(this);
-          mMixer.AddCallback(driver);
          CurrentDriver()->SwitchAtNextIteration(driver);
        }
      }
@ -927,27 +926,36 @@ void
 MediaStreamGraphImpl::OpenAudioInputImpl(CubebUtils::AudioDeviceID aID,
                                         AudioDataListener *aListener)
 {
+ // Bug 1238038 Need support for multiple mics at once
  MOZ_ASSERT(!mInputWanted);
-  mInputWanted = true;
-  mInputDeviceID = aID;
-  // XXX Switch Drivers
-  if (CurrentDriver()->AsAudioCallbackDriver()) {
-    CurrentDriver()->SetInputListener(aListener);
-  } else {
-    // XXX Switch to callback driver
+  if (mInputWanted) {
+    // Need to support separate input-only AudioCallback drivers; they'll
+    // call us back on "other" threads.  We will need to echo-cancel them, though.
+    return;
  }
+  mInputWanted = true;
+  // aID is a cubeb_devid, and we assume that opaque ptr is valid until
+  // we close cubeb.
+  mInputDeviceID = aID;
  mAudioInputs.AppendElement(aListener); // always monitor speaker data
+
+  // Switch Drivers since we're adding input (to input-only or full-duplex)
+  MonitorAutoLock mon(mMonitor);
+  if (mLifecycleState == LIFECYCLE_RUNNING) {
+    AudioCallbackDriver* driver = new AudioCallbackDriver(this);
+    CurrentDriver()->SwitchAtNextIteration(driver);
+  }
 }

 nsresult
 MediaStreamGraphImpl::OpenAudioInput(CubebUtils::AudioDeviceID aID,
                                     AudioDataListener *aListener)
 {
-  // XXX So, so, so annoying.  Can't AppendMessage except on Mainthread
+  // So, so, so annoying.  Can't AppendMessage except on Mainthread
  if (!NS_IsMainThread()) {
    NS_DispatchToMainThread(WrapRunnable(this,
                                         &MediaStreamGraphImpl::OpenAudioInput,
-                                         aID, aListener)); // XXX Fix! string need to copied
+                                         aID, aListener));
    return NS_OK;
  }
  class Message : public ControlMessage {
@ -960,6 +968,8 @@ MediaStreamGraphImpl::OpenAudioInput(CubebUtils::AudioDeviceID aID,
      mGraph->OpenAudioInputImpl(mID, mListener);
    }
    MediaStreamGraphImpl *mGraph;
+    // aID is a cubeb_devid, and we assume that opaque ptr is valid until
+    // we close cubeb.
    CubebUtils::AudioDeviceID mID;
    RefPtr<AudioDataListener> mListener;
  };
@ -973,14 +983,46 @@ MediaStreamGraphImpl::CloseAudioInputImpl(AudioDataListener *aListener)
  mInputDeviceID = nullptr;
  mInputWanted = false;
  CurrentDriver()->RemoveInputListener(aListener);
-  // XXX Switch Drivers
  mAudioInputs.RemoveElement(aListener);
+
+  // Switch Drivers since we're adding or removing an input (to nothing/system or output only)
+  bool audioTrackPresent = false;
+  for (uint32_t i = 0; i < mStreams.Length(); ++i) {
+    MediaStream* stream = mStreams[i];
+    // If this is a AudioNodeStream, force a AudioCallbackDriver.
+    if (stream->AsAudioNodeStream()) {
+      audioTrackPresent = true;
+    } else if (CurrentDriver()->AsAudioCallbackDriver()) {
+      // only if there's a real switch!
+      for (StreamBuffer::TrackIter tracks(stream->GetStreamBuffer(), MediaSegment::AUDIO);
+           !tracks.IsEnded(); tracks.Next()) {
+        audioTrackPresent = true;
+      }
+    }
+  }
+
+  MonitorAutoLock mon(mMonitor);
+  if (mLifecycleState == LIFECYCLE_RUNNING) {
+    GraphDriver* driver;
+    if (audioTrackPresent) {
+      // We still have audio output
+      STREAM_LOG(LogLevel::Debug, ("CloseInput: output present (AudioCallback)"));
+
+      driver = new AudioCallbackDriver(this);
+      CurrentDriver()->SwitchAtNextIteration(driver);
+    } else if (CurrentDriver()->AsAudioCallbackDriver()) {
+      STREAM_LOG(LogLevel::Debug, ("CloseInput: no output present (SystemClockCallback)"));
+
+      driver = new SystemClockDriver(this);
+      CurrentDriver()->SwitchAtNextIteration(driver);
+    } // else SystemClockDriver->SystemClockDriver, no switch
+  }
 }

 void
 MediaStreamGraphImpl::CloseAudioInput(AudioDataListener *aListener)
 {
-  // XXX So, so, so annoying.  Can't AppendMessage except on Mainthread
+  // So, so, so annoying.  Can't AppendMessage except on Mainthread
  if (!NS_IsMainThread()) {
    NS_DispatchToMainThread(WrapRunnable(this,
                                         &MediaStreamGraphImpl::CloseAudioInput,
@ -1265,25 +1307,6 @@ MediaStreamGraphImpl::Process()
    mMixer.FinishMixing();
  }

-  // If we are switching away from an AudioCallbackDriver, we don't need the
-  // mixer anymore.
-  bool switching = false;
-  {
-    MonitorAutoLock lock(mMonitor);
-    switching = CurrentDriver()->Switching();
-  }
-  if (CurrentDriver()->AsAudioCallbackDriver() &&
-      switching) {
-    bool isStarted;
-    {
-      MonitorAutoLock mon(mMonitor);
-      isStarted = CurrentDriver()->AsAudioCallbackDriver()->IsStarted();
-    }
-    if (isStarted) {
-      mMixer.RemoveCallback(CurrentDriver()->AsAudioCallbackDriver());
-    }
-  }
-
  if (!allBlockedForever) {
    EnsureNextIteration();
  }
@ -2754,7 +2777,6 @@ MediaStreamGraphImpl::MediaStreamGraphImpl(GraphDriverType aDriverRequested,
    if (aDriverRequested == AUDIO_THREAD_DRIVER) {
      AudioCallbackDriver* driver = new AudioCallbackDriver(this);
      mDriver = driver;
-      mMixer.AddCallback(driver);
    } else {
      mDriver = new SystemClockDriver(this);
    }
@ -3160,7 +3182,6 @@ MediaStreamGraphImpl::ApplyAudioContextOperationImpl(
        driver = nextDriver->AsAudioCallbackDriver();
      } else {
        driver = new AudioCallbackDriver(this);
-        mMixer.AddCallback(driver);
        MonitorAutoLock lock(mMonitor);
        CurrentDriver()->SwitchAtNextIteration(driver);
      }
@ -3199,7 +3220,6 @@ MediaStreamGraphImpl::ApplyAudioContextOperationImpl(
        MOZ_ASSERT(!nextDriver->AsAudioCallbackDriver());
      } else {
        driver = new SystemClockDriver(this);
-        mMixer.RemoveCallback(CurrentDriver()->AsAudioCallbackDriver());
        MonitorAutoLock lock(mMonitor);
        CurrentDriver()->SwitchAtNextIteration(driver);
      }