Bug 1156472 - Part 3 - Implement AudioCaptureStream. r=roc

It is a ProcessMediaStream that simply mixes its inputs into a mono stream,
up/down mixing appropriately.

dom/media/AudioCaptureStream.cpp

@@ -0,0 +1,133 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaStreamGraphImpl.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/unused.h"
+
+#include "AudioSegment.h"
+#include "mozilla/Logging.h"
+#include "mozilla/Attributes.h"
+#include "AudioCaptureStream.h"
+#include "ImageContainer.h"
+#include "AudioNodeEngine.h"
+#include "AudioNodeStream.h"
+#include "AudioNodeExternalInputStream.h"
+#include "webaudio/MediaStreamAudioDestinationNode.h"
+#include <algorithm>
+#include "DOMMediaStream.h"
+
+using namespace mozilla::layers;
+using namespace mozilla::dom;
+using namespace mozilla::gfx;
+
+namespace mozilla
+{
+
+// We are mixing to mono until PeerConnection can accept stereo
+static const uint32_t MONO = 1;
+
+AudioCaptureStream::AudioCaptureStream(DOMMediaStream* aWrapper)
+  : ProcessedMediaStream(aWrapper), mTrackCreated(false)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_COUNT_CTOR(AudioCaptureStream);
+  mMixer.AddCallback(this);
+}
+
+AudioCaptureStream::~AudioCaptureStream()
+{
+  MOZ_COUNT_DTOR(AudioCaptureStream);
+  mMixer.RemoveCallback(this);
+}
+
+void
+AudioCaptureStream::ProcessInput(GraphTime aFrom, GraphTime aTo,
+                                 uint32_t aFlags)
+{
+  uint32_t inputCount = mInputs.Length();
+  StreamBuffer::Track* track = EnsureTrack(AUDIO_TRACK);
+  // Notify the DOM everything is in order.
+  if (!mTrackCreated) {
+    for (uint32_t i = 0; i < mListeners.Length(); i++) {
+      MediaStreamListener* l = mListeners[i];
+      AudioSegment tmp;
+      l->NotifyQueuedTrackChanges(
+        Graph(), AUDIO_TRACK, 0, MediaStreamListener::TRACK_EVENT_CREATED, tmp);
+      l->NotifyFinishedTrackCreation(Graph());
+    }
+    mTrackCreated = true;
+  }
+
+  // If the captured stream is connected back to a object on the page (be it an
+  // HTMLMediaElement with a stream as source, or an AudioContext), a cycle
+  // situation occur. This can work if it's an AudioContext with at least one
+  // DelayNode, but the MSG will mute the whole cycle otherwise.
+  bool blocked = mFinished || mBlocked.GetAt(aFrom);
+  if (blocked || InMutedCycle() || inputCount == 0) {
+    track->Get<AudioSegment>()->AppendNullData(aTo - aFrom);
+  } else {
+    // We mix down all the tracks of all inputs, to a stereo track. Everything
+    // is {up,down}-mixed to stereo.
+    mMixer.StartMixing();
+    AudioSegment output;
+    for (uint32_t i = 0; i < inputCount; i++) {
+      MediaStream* s = mInputs[i]->GetSource();
+      StreamBuffer::TrackIter tracks(s->GetStreamBuffer(), MediaSegment::AUDIO);
+      while (!tracks.IsEnded()) {
+        AudioSegment* inputSegment = tracks->Get<AudioSegment>();
+        StreamTime inputStart = s->GraphTimeToStreamTime(aFrom);
+        StreamTime inputEnd = s->GraphTimeToStreamTime(aTo);
+        AudioSegment toMix;
+        toMix.AppendSlice(*inputSegment, inputStart, inputEnd);
+        // Care for streams blocked in the [aTo, aFrom] range.
+        if (inputEnd - inputStart < aTo - aFrom) {
+          toMix.AppendNullData((aTo - aFrom) - (inputEnd - inputStart));
+        }
+        toMix.Mix(mMixer, MONO, Graph()->GraphRate());
+        tracks.Next();
+      }
+    }
+    // This calls MixerCallback below
+    mMixer.FinishMixing();
+  }
+
+  // Regardless of the status of the input tracks, we go foward.
+  mBuffer.AdvanceKnownTracksTime(GraphTimeToStreamTime((aTo)));
+}
+
+void
+AudioCaptureStream::MixerCallback(AudioDataValue* aMixedBuffer,
+                                  AudioSampleFormat aFormat, uint32_t aChannels,
+                                  uint32_t aFrames, uint32_t aSampleRate)
+{
+  nsAutoTArray<nsTArray<AudioDataValue>, MONO> output;
+  nsAutoTArray<const AudioDataValue*, MONO> bufferPtrs;
+  output.SetLength(MONO);
+  bufferPtrs.SetLength(MONO);
+
+  uint32_t written = 0;
+  // We need to copy here, because the mixer will reuse the storage, we should
+  // not hold onto it. Buffers are in planar format.
+  for (uint32_t channel = 0; channel < aChannels; channel++) {
+    AudioDataValue* out = output[channel].AppendElements(aFrames);
+    PodCopy(out, aMixedBuffer + written, aFrames);
+    bufferPtrs[channel] = out;
+    written += aFrames;
+  }
+  AudioChunk chunk;
+  chunk.mBuffer = new mozilla::SharedChannelArrayBuffer<AudioDataValue>(&output);
+  chunk.mDuration = aFrames;
+  chunk.mBufferFormat = aFormat;
+  chunk.mVolume = 1.0f;
+  chunk.mChannelData.SetLength(MONO);
+  for (uint32_t channel = 0; channel < aChannels; channel++) {
+    chunk.mChannelData[channel] = bufferPtrs[channel];
+  }
+
+  // Now we have mixed data, simply append it to out track.
+  EnsureTrack(AUDIO_TRACK)->Get<AudioSegment>()->AppendAndConsumeChunk(&chunk);
+}
+}

dom/media/AudioCaptureStream.h

@@ -0,0 +1,40 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MOZILLA_AUDIOCAPTURESTREAM_H_
+#define MOZILLA_AUDIOCAPTURESTREAM_H_
+
+#include "MediaStreamGraph.h"
+#include "AudioMixer.h"
+#include <algorithm>
+
+namespace mozilla
+{
+
+class DOMMediaStream;
+
+/**
+ * See MediaStreamGraph::CreateAudioCaptureStream.
+ */
+class AudioCaptureStream : public ProcessedMediaStream,
+                           public MixerCallbackReceiver
+{
+public:
+  explicit AudioCaptureStream(DOMMediaStream* aWrapper);
+  virtual ~AudioCaptureStream();
+
+  void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
+
+protected:
+  enum { AUDIO_TRACK = 1 };
+  void MixerCallback(AudioDataValue* aMixedBuffer, AudioSampleFormat aFormat,
+                     uint32_t aChannels, uint32_t aFrames,
+                     uint32_t aSampleRate) override;
+  AudioMixer mMixer;
+  bool mTrackCreated;
+};
+}
+
+#endif /* MOZILLA_AUDIOCAPTURESTREAM_H_ */

dom/media/AudioMixer.h

@@ -26,7 +26,9 @@ struct MixerCallbackReceiver {
  * stream.
  *
  * AudioMixer::Mix is to be called repeatedly with buffers that have the same
- * length, sample rate, sample format and channel count.
+ * length, sample rate, sample format and channel count. This class works with
+ * interleaved and plannar buffers, but the buffer mixed must be of the same
+ * type during a mixing cycle.
  *
  * When all the tracks have been mixed, calling FinishMixing will call back with
  * a buffer containing the mixed audio data.
@@ -71,7 +73,7 @@ public:
     mSampleRate = mChannels = mFrames = 0;
   }
 
-  /* Add a buffer to the mix. aSamples is interleaved. */
+  /* Add a buffer to the mix. */
   void Mix(AudioDataValue* aSamples,
            uint32_t aChannels,
            uint32_t aFrames,

dom/media/AudioSegment.cpp

@@ -146,6 +146,103 @@ void AudioSegment::ResampleChunks(SpeexResamplerState* aResampler, uint32_t aInR
   }
 }
 
+// This helps to to safely get a pointer to the position we want to start
+// writing a planar audio buffer, depending on the channel and the offset in the
+// buffer.
+static AudioDataValue*
+PointerForOffsetInChannel(AudioDataValue* aData, size_t aLengthSamples,
+                          uint32_t aChannelCount, uint32_t aChannel,
+                          uint32_t aOffsetSamples)
+{
+  size_t samplesPerChannel = aLengthSamples / aChannelCount;
+  size_t beginningOfChannel = samplesPerChannel * aChannel;
+  MOZ_ASSERT(aChannel * samplesPerChannel + aOffsetSamples < aLengthSamples,
+             "Offset request out of bounds.");
+  return aData + beginningOfChannel + aOffsetSamples;
+}
+
+void
+AudioSegment::Mix(AudioMixer& aMixer, uint32_t aOutputChannels,
+                  uint32_t aSampleRate)
+{
+  nsAutoTArray<AudioDataValue, AUDIO_PROCESSING_FRAMES* GUESS_AUDIO_CHANNELS>
+  buf;
+  nsAutoTArray<const void*, GUESS_AUDIO_CHANNELS> channelData;
+  uint32_t offsetSamples = 0;
+  uint32_t duration = GetDuration();
+
+  if (duration <= 0) {
+    MOZ_ASSERT(duration == 0);
+    return;
+  }
+
+  uint32_t outBufferLength = duration * aOutputChannels;
+  buf.SetLength(outBufferLength);
+
+  for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) {
+    AudioChunk& c = *ci;
+    uint32_t frames = c.mDuration;
+
+    // If the chunk is silent, simply write the right number of silence in the
+    // buffers.
+    if (c.mBufferFormat == AUDIO_FORMAT_SILENCE) {
+      for (uint32_t channel = 0; channel < aOutputChannels; channel++) {
+        AudioDataValue* ptr =
+          PointerForOffsetInChannel(buf.Elements(), outBufferLength,
+                                    aOutputChannels, channel, offsetSamples);
+        PodZero(ptr, frames);
+      }
+    } else {
+      // Othewise, we need to upmix or downmix appropriately, depending on the
+      // desired input and output channels.
+      channelData.SetLength(c.mChannelData.Length());
+      for (uint32_t i = 0; i < channelData.Length(); ++i) {
+        channelData[i] = c.mChannelData[i];
+      }
+      if (channelData.Length() < aOutputChannels) {
+        // Up-mix.
+        AudioChannelsUpMix(&channelData, aOutputChannels, gZeroChannel);
+        for (uint32_t channel = 0; channel < aOutputChannels; channel++) {
+          AudioDataValue* ptr =
+            PointerForOffsetInChannel(buf.Elements(), outBufferLength,
+                                      aOutputChannels, channel, offsetSamples);
+          PodCopy(ptr, reinterpret_cast<const float*>(channelData[channel]),
+                  frames);
+        }
+        MOZ_ASSERT(channelData.Length() == aOutputChannels);
+      } else if (channelData.Length() > aOutputChannels) {
+        // Down mix.
+        nsAutoTArray<float*, GUESS_AUDIO_CHANNELS> outChannelPtrs;
+        outChannelPtrs.SetLength(aOutputChannels);
+        uint32_t offsetSamples = 0;
+        for (uint32_t channel = 0; channel < aOutputChannels; channel++) {
+          outChannelPtrs[channel] =
+            PointerForOffsetInChannel(buf.Elements(), outBufferLength,
+                                      aOutputChannels, channel, offsetSamples);
+        }
+        AudioChannelsDownMix(channelData, outChannelPtrs.Elements(),
+                             aOutputChannels, frames);
+      } else {
+        // The channel count is already what we want, just copy it over.
+        for (uint32_t channel = 0; channel < aOutputChannels; channel++) {
+          AudioDataValue* ptr =
+            PointerForOffsetInChannel(buf.Elements(), outBufferLength,
+                                      aOutputChannels, channel, offsetSamples);
+          PodCopy(ptr, reinterpret_cast<const float*>(channelData[channel]),
+                  frames);
+        }
+      }
+    }
+    offsetSamples += frames;
+  }
+
+  if (offsetSamples) {
+    MOZ_ASSERT(offsetSamples == outBufferLength / aOutputChannels,
+               "We forgot to write some samples?");
+    aMixer.Mix(buf.Elements(), aOutputChannels, offsetSamples, aSampleRate);
+  }
+}
+
 void
 AudioSegment::WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aOutputChannels, uint32_t aSampleRate)
 {

dom/media/AudioSegment.h

@@ -299,7 +299,14 @@ public:
     return chunk;
   }
   void ApplyVolume(float aVolume);
-  void WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aChannelCount, uint32_t aSampleRate);
+  // Mix the segment into a mixer, interleaved. This is useful to output a
+  // segment to a system audio callback. It up or down mixes to aChannelCount
+  // channels.
+  void WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aChannelCount,
+               uint32_t aSampleRate);
+  // Mix the segment into a mixer, keeping it planar, up or down mixing to
+  // aChannelCount channels.
+  void Mix(AudioMixer& aMixer, uint32_t aChannelCount, uint32_t aSampleRate);
 
   int ChannelCount() {
     NS_WARN_IF_FALSE(!mChunks.IsEmpty(),

dom/media/DOMMediaStream.cpp

@@ -301,6 +301,18 @@ DOMMediaStream::InitTrackUnionStream(nsIDOMWindow* aWindow,
   InitStreamCommon(aGraph->CreateTrackUnionStream(this));
 }
 
+void
+DOMMediaStream::InitAudioCaptureStream(nsIDOMWindow* aWindow,
+                                       MediaStreamGraph* aGraph)
+{
+  mWindow = aWindow;
+
+  if (!aGraph) {
+    aGraph = MediaStreamGraph::GetInstance();
+  }
+  InitStreamCommon(aGraph->CreateAudioCaptureStream(this));
+}
+
 void
 DOMMediaStream::InitStreamCommon(MediaStream* aStream)
 {
@@ -329,6 +341,15 @@ DOMMediaStream::CreateTrackUnionStream(nsIDOMWindow* aWindow,
   return stream.forget();
 }
 
+already_AddRefed<DOMMediaStream>
+DOMMediaStream::CreateAudioCaptureStream(nsIDOMWindow* aWindow,
+                                         MediaStreamGraph* aGraph)
+{
+  nsRefPtr<DOMMediaStream> stream = new DOMMediaStream();
+  stream->InitAudioCaptureStream(aWindow, aGraph);
+  return stream.forget();
+}
+
 void
 DOMMediaStream::SetTrackEnabled(TrackID aTrackID, bool aEnabled)
 {
@@ -653,6 +674,15 @@ DOMLocalMediaStream::CreateTrackUnionStream(nsIDOMWindow* aWindow,
   return stream.forget();
 }
 
+already_AddRefed<DOMLocalMediaStream>
+DOMLocalMediaStream::CreateAudioCaptureStream(nsIDOMWindow* aWindow,
+                                              MediaStreamGraph* aGraph)
+{
+  nsRefPtr<DOMLocalMediaStream> stream = new DOMLocalMediaStream();
+  stream->InitAudioCaptureStream(aWindow, aGraph);
+  return stream.forget();
+}
+
 DOMAudioNodeMediaStream::DOMAudioNodeMediaStream(AudioNode* aNode)
 : mStreamNode(aNode)
 {

dom/media/DOMMediaStream.h

@@ -198,6 +198,13 @@ public:
   static already_AddRefed<DOMMediaStream> CreateTrackUnionStream(nsIDOMWindow* aWindow,
                                                                  MediaStreamGraph* aGraph = nullptr);
 
+  /**
+   * Create an nsDOMMediaStream whose underlying stream is an
+   * AudioCaptureStream
+   */
+  static already_AddRefed<DOMMediaStream> CreateAudioCaptureStream(
+    nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr);
+
   void SetLogicalStreamStartTime(StreamTime aTime)
   {
     mLogicalStreamStartTime = aTime;
@@ -261,6 +268,8 @@ protected:
                         MediaStreamGraph* aGraph = nullptr);
   void InitTrackUnionStream(nsIDOMWindow* aWindow,
                             MediaStreamGraph* aGraph = nullptr);
+  void InitAudioCaptureStream(nsIDOMWindow* aWindow,
+                              MediaStreamGraph* aGraph = nullptr);
   void InitStreamCommon(MediaStream* aStream);
   already_AddRefed<AudioTrack> CreateAudioTrack(AudioStreamTrack* aStreamTrack);
   already_AddRefed<VideoTrack> CreateVideoTrack(VideoStreamTrack* aStreamTrack);
@@ -351,6 +360,12 @@ public:
   CreateTrackUnionStream(nsIDOMWindow* aWindow,
                          MediaStreamGraph* aGraph = nullptr);
 
+  /**
+   * Create an nsDOMLocalMediaStream whose underlying stream is an
+   * AudioCaptureStream. */
+  static already_AddRefed<DOMLocalMediaStream> CreateAudioCaptureStream(
+    nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr);
+
 protected:
   virtual ~DOMLocalMediaStream();
 };

dom/media/moz.build

@@ -196,6 +196,7 @@ EXPORTS.mozilla.dom += [
 
 UNIFIED_SOURCES += [
     'AbstractThread.cpp',
+    'AudioCaptureStream.cpp',
     'AudioChannelFormat.cpp',
     'AudioCompactor.cpp',
     'AudioSegment.cpp',