From 6010fdc4cd9e2c17246106c86ff4c3dfe0b8c0cd Mon Sep 17 00:00:00 2001 From: Paul Adenot Date: Mon, 25 Aug 2014 14:13:08 +0200 Subject: [PATCH] Bug 989921 - Allow the MediaStreamGraph mixer to send data back to multiple consumers. r=jesup --- content/media/AudioMixer.h | 69 +++++++++++++++---- content/media/MediaStreamGraph.cpp | 29 +++----- content/media/MediaStreamGraph.h | 1 - content/media/compiledtest/TestAudioMixer.cpp | 45 +++++++----- content/media/webrtc/AudioOutputObserver.h | 8 ++- .../media/webrtc/MediaEngineWebRTCAudio.cpp | 11 +++ 6 files changed, 110 insertions(+), 53 deletions(-) diff --git a/content/media/AudioMixer.h b/content/media/AudioMixer.h index 5d07f245cf47..95ed69beb503 100644 --- a/content/media/AudioMixer.h +++ b/content/media/AudioMixer.h @@ -9,13 +9,17 @@ #include "AudioSampleFormat.h" #include "nsTArray.h" #include "mozilla/PodOperations.h" +#include "mozilla/LinkedList.h" namespace mozilla { -typedef void(*MixerFunc)(AudioDataValue* aMixedBuffer, - AudioSampleFormat aFormat, - uint32_t aChannels, - uint32_t aFrames, - uint32_t aSampleRate); + +struct MixerCallbackReceiver { + virtual void MixerCallback(AudioDataValue* aMixedBuffer, + AudioSampleFormat aFormat, + uint32_t aChannels, + uint32_t aFrames, + uint32_t aSampleRate) = 0; +}; /** * This class mixes multiple streams of audio together to output a single audio @@ -32,21 +36,34 @@ typedef void(*MixerFunc)(AudioDataValue* aMixedBuffer, class AudioMixer { public: - AudioMixer(MixerFunc aCallback) - : mCallback(aCallback), - mFrames(0), + AudioMixer() + : mFrames(0), mChannels(0), mSampleRate(0) { } + ~AudioMixer() + { + mCallbacks.clear(); + } + + void StartMixing() + { + mSampleRate = mChannels = mFrames = 0; + } + /* Get the data from the mixer. This is supposed to be called when all the * tracks have been mixed in. The caller should not hold onto the data. */ void FinishMixing() { - mCallback(mMixedAudio.Elements(), - AudioSampleTypeToFormat::Format, - mChannels, - mFrames, - mSampleRate); + MOZ_ASSERT(mChannels && mFrames && mSampleRate, "Mix not called for this cycle?"); + for (MixerCallback* cb = mCallbacks.getFirst(); + cb != nullptr; cb = cb->getNext()) { + cb->mReceiver->MixerCallback(mMixedAudio.Elements(), + AudioSampleTypeToFormat::Format, + mChannels, + mFrames, + mSampleRate); + } PodZero(mMixedAudio.Elements(), mMixedAudio.Length()); mSampleRate = mChannels = mFrames = 0; } @@ -71,6 +88,21 @@ public: mMixedAudio[i] += aSamples[i]; } } + + void AddCallback(MixerCallbackReceiver* aReceiver) { + mCallbacks.insertBack(new MixerCallback(aReceiver)); + } + + bool RemoveCallback(MixerCallbackReceiver* aReceiver) { + for (MixerCallback* cb = mCallbacks.getFirst(); + cb != nullptr; cb = cb->getNext()) { + if (cb->mReceiver == aReceiver) { + cb->remove(); + return true; + } + } + return false; + } private: void EnsureCapacityAndSilence() { if (mFrames * mChannels > mMixedAudio.Length()) { @@ -79,8 +111,17 @@ private: PodZero(mMixedAudio.Elements(), mMixedAudio.Length()); } + class MixerCallback : public LinkedListElement + { + public: + MixerCallback(MixerCallbackReceiver* aReceiver) + : mReceiver(aReceiver) + { } + MixerCallbackReceiver* mReceiver; + }; + /* Function that is called when the mixing is done. */ - MixerFunc mCallback; + LinkedList mCallbacks; /* Number of frames for this mixing block. */ uint32_t mFrames; /* Number of channels for this mixing block. */ diff --git a/content/media/MediaStreamGraph.cpp b/content/media/MediaStreamGraph.cpp index 0e6d65e422b3..e76e7ed958f7 100644 --- a/content/media/MediaStreamGraph.cpp +++ b/content/media/MediaStreamGraph.cpp @@ -529,24 +529,6 @@ MediaStreamGraphImpl::MarkConsumed(MediaStream* aStream) } } -static void AudioMixerCallback(AudioDataValue* aMixedBuffer, - AudioSampleFormat aFormat, - uint32_t aChannels, - uint32_t aFrames, - uint32_t aSampleRate) -{ - // Need an api to register mixer callbacks, bug 989921 -#ifdef MOZ_WEBRTC - if (aFrames > 0 && aChannels > 0) { - // XXX need Observer base class and registration API - if (gFarendObserver) { - gFarendObserver->InsertFarEnd(aMixedBuffer, aFrames, false, - aSampleRate, aChannels, aFormat); - } - } -#endif -} - void MediaStreamGraphImpl::UpdateStreamOrder() { @@ -573,7 +555,11 @@ MediaStreamGraphImpl::UpdateStreamOrder() mStreams[i]->mAudioOutputStreams[j].mStream->SetMicrophoneActive(true); } } + if (gFarendObserver) { + mMixer->AddCallback(gFarendObserver); + } } else if (mMixer && !shouldMix) { + mMixer->RemoveCallback(gFarendObserver); mMixer = nullptr; for (uint32_t i = 0; i < mStreams.Length(); ++i) { for (uint32_t j = 0; j < mStreams[i]->mAudioOutputStreams.Length(); ++j) { @@ -1425,6 +1411,11 @@ MediaStreamGraphImpl::RunThread() // This is the number of frame that are written to the AudioStreams, for // this cycle. TrackTicks ticksPlayed = 0; + + if (mMixer) { + mMixer->StartMixing(); + } + // Figure out what each stream wants to do for (uint32_t i = 0; i < mStreams.Length(); ++i) { MediaStream* stream = mStreams[i]; @@ -1480,7 +1471,7 @@ MediaStreamGraphImpl::RunThread() } } - if (mMixer) { + if (mMixer && ticksPlayed) { mMixer->FinishMixing(); } diff --git a/content/media/MediaStreamGraph.h b/content/media/MediaStreamGraph.h index d468617410b9..47c6417c3142 100644 --- a/content/media/MediaStreamGraph.h +++ b/content/media/MediaStreamGraph.h @@ -18,7 +18,6 @@ #include "MainThreadUtils.h" #include "nsAutoRef.h" #include -#include "AudioMixer.h" #include "mozilla/dom/AudioChannelBinding.h" class nsIRunnable; diff --git a/content/media/compiledtest/TestAudioMixer.cpp b/content/media/compiledtest/TestAudioMixer.cpp index a4e20f5d1d6f..e6aaa33a56c0 100644 --- a/content/media/compiledtest/TestAudioMixer.cpp +++ b/content/media/compiledtest/TestAudioMixer.cpp @@ -9,25 +9,28 @@ using mozilla::AudioDataValue; using mozilla::AudioSampleFormat; +struct MixerConsumer : public mozilla::MixerCallbackReceiver +{ /* In this test, the different audio stream and channels are always created to * cancel each other. */ -void MixingDone(AudioDataValue* aData, AudioSampleFormat aFormat, uint32_t aChannels, uint32_t aFrames, uint32_t aSampleRate) -{ - bool silent = true; - for (uint32_t i = 0; i < aChannels * aFrames; i++) { - if (aData[i] != 0.0) { - if (aFormat == mozilla::AUDIO_FORMAT_S16) { - fprintf(stderr, "Sample at %d is not silent: %d\n", i, (short)aData[i]); - } else { - fprintf(stderr, "Sample at %d is not silent: %f\n", i, (float)aData[i]); + void MixerCallback(AudioDataValue* aData, AudioSampleFormat aFormat, uint32_t aChannels, uint32_t aFrames, uint32_t aSampleRate) + { + bool silent = true; + for (uint32_t i = 0; i < aChannels * aFrames; i++) { + if (aData[i] != 0.0) { + if (aFormat == mozilla::AUDIO_FORMAT_S16) { + fprintf(stderr, "Sample at %d is not silent: %d\n", i, (short)aData[i]); + } else { + fprintf(stderr, "Sample at %d is not silent: %f\n", i, (float)aData[i]); + } + silent = false; } - silent = false; + } + if (!silent) { + MOZ_CRASH(); } } - if (!silent) { - MOZ_CRASH(); - } -} +}; /* Helper function to give us the maximum and minimum value that don't clip, * for a given sample format (integer or floating-point). */ @@ -68,6 +71,7 @@ void FillBuffer(AudioDataValue* aBuffer, uint32_t aLength, AudioDataValue aValue int main(int argc, char* argv[]) { const uint32_t CHANNEL_LENGTH = 256; const uint32_t AUDIO_RATE = 44100; + MixerConsumer consumer; AudioDataValue a[CHANNEL_LENGTH * 2]; AudioDataValue b[CHANNEL_LENGTH * 2]; FillBuffer(a, CHANNEL_LENGTH, GetLowValue()); @@ -77,7 +81,8 @@ int main(int argc, char* argv[]) { { int iterations = 2; - mozilla::AudioMixer mixer(MixingDone); + mozilla::AudioMixer mixer; + mixer.AddCallback(&consumer); fprintf(stderr, "Test AudioMixer constant buffer length.\n"); @@ -89,7 +94,8 @@ int main(int argc, char* argv[]) { } { - mozilla::AudioMixer mixer(MixingDone); + mozilla::AudioMixer mixer; + mixer.AddCallback(&consumer); fprintf(stderr, "Test AudioMixer variable buffer length.\n"); @@ -120,7 +126,9 @@ int main(int argc, char* argv[]) { FillBuffer(b, CHANNEL_LENGTH, GetHighValue()); { - mozilla::AudioMixer mixer(MixingDone); + mozilla::AudioMixer mixer; + mixer.AddCallback(&consumer); + fprintf(stderr, "Test AudioMixer variable channel count.\n"); mixer.Mix(a, 1, CHANNEL_LENGTH, AUDIO_RATE); @@ -135,7 +143,8 @@ int main(int argc, char* argv[]) { } { - mozilla::AudioMixer mixer(MixingDone); + mozilla::AudioMixer mixer; + mixer.AddCallback(&consumer); fprintf(stderr, "Test AudioMixer variable stream count.\n"); mixer.Mix(a, 2, CHANNEL_LENGTH, AUDIO_RATE); diff --git a/content/media/webrtc/AudioOutputObserver.h b/content/media/webrtc/AudioOutputObserver.h index b8116df26966..17ac1d752f7c 100644 --- a/content/media/webrtc/AudioOutputObserver.h +++ b/content/media/webrtc/AudioOutputObserver.h @@ -20,12 +20,18 @@ typedef struct FarEndAudioChunk_ { } FarEndAudioChunk; // XXX Really a singleton currently -class AudioOutputObserver // : public MSGOutputObserver +class AudioOutputObserver : public MixerCallbackReceiver { public: AudioOutputObserver(); virtual ~AudioOutputObserver(); + void MixerCallback(AudioDataValue* aMixedBuffer, + AudioSampleFormat aFormat, + uint32_t aChannels, + uint32_t aFrames, + uint32_t aSampleRate) MOZ_OVERRIDE; + void Clear(); void InsertFarEnd(const AudioDataValue *aBuffer, uint32_t aFrames, bool aOverran, int aFreq, int aChannels, AudioSampleFormat aFormat); diff --git a/content/media/webrtc/MediaEngineWebRTCAudio.cpp b/content/media/webrtc/MediaEngineWebRTCAudio.cpp index 691b31f14d2a..42b45b79106c 100644 --- a/content/media/webrtc/MediaEngineWebRTCAudio.cpp +++ b/content/media/webrtc/MediaEngineWebRTCAudio.cpp @@ -90,6 +90,17 @@ AudioOutputObserver::Size() return mPlayoutFifo->size(); } +void +AudioOutputObserver::MixerCallback(AudioDataValue* aMixedBuffer, + AudioSampleFormat aFormat, + uint32_t aChannels, + uint32_t aFrames, + uint32_t aSampleRate) +{ + gFarendObserver->InsertFarEnd(aMixedBuffer, aFrames, false, + aSampleRate, aChannels, aFormat); +} + // static void AudioOutputObserver::InsertFarEnd(const AudioDataValue *aBuffer, uint32_t aFrames, bool aOverran,