Bug 694814: Patch 3 - Add far-end mixer observer and insert far-end audio for AEC r=padenot

content/media/MediaStreamGraph.cpp

@@ -27,6 +27,7 @@
 #include "GeckoProfiler.h"
 #include "mozilla/unused.h"
 #include "speex/speex_resampler.h"
+#include "AudioOutputObserver.h"
 
 using namespace mozilla::layers;
 using namespace mozilla::dom;
@@ -583,6 +584,13 @@ static void AudioMixerCallback(AudioDataValue* aMixedBuffer,
                                uint32_t aFrames)
 {
   // Need an api to register mixer callbacks, bug 989921
+  if (aFrames > 0 && aChannels > 0) {
+    // XXX need Observer base class and registration API
+    if (gFarendObserver) {
+      gFarendObserver->InsertFarEnd(aMixedBuffer, aFrames, false,
+                                    IdealAudioRate(), aChannels, aFormat);
+    }
+  }
 }
 
 void

content/media/webrtc/AudioOutputObserver.h

@@ -0,0 +1,55 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef AUDIOOUTPUTOBSERVER_H_
+#define AUDIOOUTPUTOBSERVER_H_
+
+#include "mozilla/StaticPtr.h"
+
+namespace webrtc {
+class SingleRwFifo;
+}
+
+namespace mozilla {
+
+typedef struct FarEndAudioChunk_ {
+  uint16_t mSamples;
+  bool mOverrun;
+  int16_t mData[1]; // variable-length
+} FarEndAudioChunk;
+
+// XXX Really a singleton currently
+class AudioOutputObserver // : public MSGOutputObserver
+{
+public:
+  AudioOutputObserver();
+  virtual ~AudioOutputObserver();
+
+  void Clear();
+  void InsertFarEnd(const AudioDataValue *aBuffer, uint32_t aSamples, bool aOverran,
+                    int aFreq, int aChannels, AudioSampleFormat aFormat);
+  uint32_t PlayoutFrequency() { return mPlayoutFreq; }
+  uint32_t PlayoutChannels() { return mPlayoutChannels; }
+
+  FarEndAudioChunk *Pop();
+  uint32_t Size();
+
+private:
+  uint32_t mPlayoutFreq;
+  uint32_t mPlayoutChannels;
+
+  nsAutoPtr<webrtc::SingleRwFifo> mPlayoutFifo;
+  uint32_t mChunkSize;
+
+  // chunking to 10ms support
+  nsAutoPtr<FarEndAudioChunk> mSaved;
+  uint32_t mSamplesSaved;
+};
+
+// XXX until there's a registration API in MSG
+extern StaticAutoPtr<AudioOutputObserver> gFarendObserver;
+
+}
+
+#endif

content/media/webrtc/MediaEngineWebRTC.cpp

@@ -60,6 +60,8 @@ MediaEngineWebRTC::MediaEngineWebRTC(MediaEnginePrefs &aPrefs)
 #else
   AsyncLatencyLogger::Get()->AddRef();
 #endif
+  // XXX
+  gFarendObserver = new AudioOutputObserver();
 }
 
 void

content/media/webrtc/MediaEngineWebRTC.h

@@ -56,6 +56,7 @@
 #endif
 
 #include "NullTransport.h"
+#include "AudioOutputObserver.h"
 
 namespace mozilla {
 
@@ -258,6 +259,7 @@ public:
     , mCapIndex(aIndex)
     , mChannel(-1)
     , mInitDone(false)
+    , mStarted(false)
     , mEchoOn(false), mAgcOn(false), mNoiseOn(false)
     , mEchoCancel(webrtc::kEcDefault)
     , mAGC(webrtc::kAgcDefault)
@@ -323,6 +325,7 @@ private:
   int mChannel;
   TrackID mTrackID;
   bool mInitDone;
+  bool mStarted;
 
   nsString mDeviceName;
   nsString mDeviceUUID;
@@ -344,6 +347,8 @@ public:
 #ifdef MOZ_B2G_CAMERA
     AsyncLatencyLogger::Get()->Release();
 #endif
+    // XXX
+    gFarendObserver = nullptr;
   }
 
   // Clients should ensure to clean-up sources video/audio sources

content/media/webrtc/MediaEngineWebRTCAudio.cpp

@@ -3,6 +3,15 @@
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "MediaEngineWebRTC.h"
+#include <stdio.h>
+#include <algorithm>
+#include "mozilla/Assertions.h"
+
+// scoped_ptr.h uses FF
+#ifdef FF
+#undef FF
+#endif
+#include "webrtc/modules/audio_device/opensl/single_rw_fifo.h"
 
 #define CHANNELS 1
 #define ENCODING "L16"
@@ -12,6 +21,13 @@
 #define SAMPLE_FREQUENCY 16000
 #define SAMPLE_LENGTH ((SAMPLE_FREQUENCY*10)/1000)
 
+// These are restrictions from the webrtc.org code
+#define MAX_CHANNELS 2
+#define MAX_SAMPLING_FREQ 48000 // Hz - multiple of 100
+
+#define MAX_AEC_FIFO_DEPTH 200 // ms - multiple of 10
+static_assert(!(MAX_AEC_FIFO_DEPTH % 10), "Invalid MAX_AEC_FIFO_DEPTH");
+
 namespace mozilla {
 
 #ifdef LOG
@@ -30,6 +46,117 @@ extern PRLogModuleInfo* GetMediaManagerLog();
  */
 NS_IMPL_ISUPPORTS0(MediaEngineWebRTCAudioSource)
 
+// XXX temp until MSG supports registration
+StaticAutoPtr<AudioOutputObserver> gFarendObserver;
+
+AudioOutputObserver::AudioOutputObserver()
+  : mPlayoutFreq(0)
+  , mPlayoutChannels(0)
+  , mChunkSize(0)
+  , mSamplesSaved(0)
+{
+  // Buffers of 10ms chunks
+  mPlayoutFifo = new webrtc::SingleRwFifo(MAX_AEC_FIFO_DEPTH/10);
+}
+
+AudioOutputObserver::~AudioOutputObserver()
+{
+}
+
+void
+AudioOutputObserver::Clear()
+{
+  while (mPlayoutFifo->size() > 0) {
+    (void) mPlayoutFifo->Pop();
+  }
+}
+
+FarEndAudioChunk *
+AudioOutputObserver::Pop()
+{
+  return (FarEndAudioChunk *) mPlayoutFifo->Pop();
+}
+
+uint32_t
+AudioOutputObserver::Size()
+{
+  return mPlayoutFifo->size();
+}
+
+// static
+void
+AudioOutputObserver::InsertFarEnd(const AudioDataValue *aBuffer, uint32_t aSamples, bool aOverran,
+                                  int aFreq, int aChannels, AudioSampleFormat aFormat)
+{
+  if (mPlayoutChannels != 0) {
+    if (mPlayoutChannels != static_cast<uint32_t>(aChannels)) {
+      MOZ_CRASH();
+    }
+  } else {
+    MOZ_ASSERT(aChannels <= MAX_CHANNELS);
+    mPlayoutChannels = static_cast<uint32_t>(aChannels);
+  }
+  if (mPlayoutFreq != 0) {
+    if (mPlayoutFreq != static_cast<uint32_t>(aFreq)) {
+      MOZ_CRASH();
+    }
+  } else {
+    MOZ_ASSERT(aFreq <= MAX_SAMPLING_FREQ);
+    MOZ_ASSERT(!(aFreq % 100), "Sampling rate for far end data should be multiple of 100.");
+    mPlayoutFreq = aFreq;
+    mChunkSize = aFreq/100; // 10ms
+  }
+
+#ifdef LOG_FAREND_INSERTION
+  static FILE *fp = fopen("insertfarend.pcm","wb");
+#endif
+
+  if (mSaved) {
+    // flag overrun as soon as possible, and only once
+    mSaved->mOverrun = aOverran;
+    aOverran = false;
+  }
+  // Rechunk to 10ms.
+  // The AnalyzeReverseStream() and WebRtcAec_BufferFarend() functions insist on 10ms
+  // samples per call.  Annoying...
+  while (aSamples) {
+    if (!mSaved) {
+      mSaved = (FarEndAudioChunk *) moz_xmalloc(sizeof(FarEndAudioChunk) +
+                                                (mChunkSize * aChannels - 1)*sizeof(int16_t));
+      mSaved->mSamples = mChunkSize;
+      mSaved->mOverrun = aOverran;
+      aOverran = false;
+    }
+    uint32_t to_copy = mChunkSize - mSamplesSaved;
+    if (to_copy > aSamples) {
+      to_copy = aSamples;
+    }
+
+    int16_t *dest = &(mSaved->mData[mSamplesSaved * aChannels]);
+    ConvertAudioSamples(aBuffer, dest, to_copy * aChannels);
+
+#ifdef LOG_FAREND_INSERTION
+    if (fp) {
+      fwrite(&(mSaved->mData[mSamplesSaved * aChannels]), to_copy * aChannels, sizeof(int16_t), fp);
+    }
+#endif
+    aSamples -= to_copy;
+    mSamplesSaved += to_copy;
+
+    if (mSamplesSaved >= mChunkSize) {
+      int free_slots = mPlayoutFifo->capacity() - mPlayoutFifo->size();
+      if (free_slots <= 0) {
+        // XXX We should flag an overrun for the reader.  We can't drop data from it due to
+        // thread safety issues.
+        break;
+      } else {
+        mPlayoutFifo->Push((int8_t *) mSaved.forget()); // takes ownership
+        mSamplesSaved = 0;
+      }
+    }
+  }
+}
+
 void
 MediaEngineWebRTCAudioSource::GetName(nsAString& aName)
 {
@@ -172,10 +299,16 @@ MediaEngineWebRTCAudioSource::Start(SourceMediaStream* aStream, TrackID aID)
   // Make sure logger starts before capture
   AsyncLatencyLogger::Get(true);
 
+  // Register output observer
+  // XXX
+  MOZ_ASSERT(gFarendObserver);
+  gFarendObserver->Clear();
+
   // Configure audio processing in webrtc code
   Config(mEchoOn, webrtc::kEcUnchanged,
          mAgcOn, webrtc::kAgcUnchanged,
-         mNoiseOn, webrtc::kNsUnchanged);
+         mNoiseOn, webrtc::kNsUnchanged,
+         mPlayoutDelay);
 
   if (mVoEBase->StartReceive(mChannel)) {
     return NS_ERROR_FAILURE;
@@ -364,6 +497,32 @@ MediaEngineWebRTCAudioSource::Process(int channel,
   webrtc::ProcessingTypes type, sample* audio10ms,
   int length, int samplingFreq, bool isStereo)
 {
+  // On initial capture, throw away all far-end data except the most recent sample
+  // since it's already irrelevant and we want to keep avoid confusing the AEC far-end
+  // input code with "old" audio.
+  if (!mStarted) {
+    mStarted  = true;
+    while (gFarendObserver->Size() > 1) {
+      FarEndAudioChunk *buffer = gFarendObserver->Pop(); // only call if size() > 0
+      free(buffer);
+    }
+  }
+
+  while (gFarendObserver->Size() > 0) {
+    FarEndAudioChunk *buffer = gFarendObserver->Pop(); // only call if size() > 0
+    if (buffer) {
+      int length = buffer->mSamples;
+      if (mVoERender->ExternalPlayoutData(buffer->mData,
+                                          gFarendObserver->PlayoutFrequency(),
+                                          gFarendObserver->PlayoutChannels(),
+                                          mPlayoutDelay,
+                                          length) == -1) {
+        return;
+      }
+    }
+    free(buffer);
+  }
+
   MonitorAutoLock lock(mMonitor);
   if (mState != kStarted)
     return;

content/media/webrtc/moz.build

@@ -12,7 +12,8 @@ EXPORTS += [
 ]
 
 if CONFIG['MOZ_WEBRTC']:
-    EXPORTS += ['LoadManager.h',
+    EXPORTS += ['AudioOutputObserver.h',
+                'LoadManager.h',
                 'LoadManagerFactory.h',
                 'LoadMonitor.h',
                 'MediaEngineWebRTC.h']