Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-16 23:05:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1271585 - Part 2 - Synchronously insert audio frames from the microphone in the MSG if possible. r=pehrsons,jesup

Browse Source 
MozReview-Commit-ID: Fm2woel600v
This commit is contained in:
Paul Adenot 2016-05-30 15:24:19 +02:00
parent 209bdf8893
commit 28b7a4423b
3 changed files with 119 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dom/media/webrtc/MediaEngine.h
View File

@ -58,6 +58,10 @@ public:
#else
static const int DEFAULT_SAMPLE_RATE = 16000;
#endif
// This allows using whatever rate the graph is using for the
// MediaStreamTrack. This is useful for microphone data, we know it's already
// at the correct rate for insertion in the MSG.
static const int USE_GRAPH_RATE = -1;
/* Populate an array of video sources in the nsTArray. Also include devices
* that are currently unavailable. */

21
dom/media/webrtc/MediaEngineWebRTC.h
View File

@ -434,6 +434,7 @@ public:
, mSampleFrequency(MediaEngine::DEFAULT_SAMPLE_RATE)
, mPlayoutDelay(0)
, mNullTransport(nullptr)
, mSkipProcessing(false)
{
MOZ_ASSERT(aVoiceEnginePtr);
MOZ_ASSERT(aAudioInput);
@ -515,6 +516,22 @@ private:
bool InitEngine();
void DeInitEngine();
// This is true when all processing is disabled, we can skip
// packetization, resampling and other processing passes.
bool PassThrough() {
return mSkipProcessing;
}
template<typename T>
void InsertInGraph(const T* aBuffer,
size_t aFrames,
uint32_t aChannels);
void PacketizeAndProcess(MediaStreamGraph* aGraph,
const AudioDataValue* aBuffer,
size_t aFrames,
TrackRate aRate,
uint32_t aChannels);
webrtc::VoiceEngine* mVoiceEngine;
RefPtr<mozilla::AudioInput> mAudioInput;
RefPtr<WebRTCAudioDataListener> mListener;
@ -555,6 +572,10 @@ private:
NullTransport *mNullTransport;
nsTArray<int16_t> mInputBuffer;
// mSkipProcessing is true if none of the processing passes are enabled,
// because of prefs or constraints. This allows simply copying the audio into
// the MSG, skipping resampling and the whole webrtc.org code.
bool mSkipProcessing;
};
class MediaEngineWebRTC : public MediaEngine

109
dom/media/webrtc/MediaEngineWebRTCAudio.cpp
View File

@ -298,6 +298,12 @@ MediaEngineWebRTCMicrophoneSource::Restart(const dom::MediaTrackConstraints& aCo
LOG(("%s Error setting NoiseSuppression Status: %d ",__FUNCTION__, error));
}
}
mSkipProcessing = !(aec_on || agc_on || noise_on);
if (mSkipProcessing) {
mSampleFrequency = MediaEngine::USE_GRAPH_RATE;
}
return NS_OK;
}
@ -344,6 +350,9 @@ MediaEngineWebRTCMicrophoneSource::Start(SourceMediaStream *aStream,
}
AudioSegment* segment = new AudioSegment();
if (mSampleFrequency == MediaEngine::USE_GRAPH_RATE) {
mSampleFrequency = aStream->GraphRate();
}
aStream->AddAudioTrack(aID, mSampleFrequency, 0, segment, SourceMediaStream::ADDTRACK_QUEUED);
// XXX Make this based on the pref.
@ -457,10 +466,8 @@ MediaEngineWebRTCMicrophoneSource::NotifyOutputData(MediaStreamGraph* aGraph,
{
}
// Called back on GraphDriver thread!
// Note this can be called back after ::Shutdown()
void
MediaEngineWebRTCMicrophoneSource::NotifyInputData(MediaStreamGraph* aGraph,
MediaEngineWebRTCMicrophoneSource::PacketizeAndProcess(MediaStreamGraph* aGraph,
const AudioDataValue* aBuffer,
size_t aFrames,
TrackRate aRate,
@ -471,7 +478,8 @@ MediaEngineWebRTCMicrophoneSource::NotifyInputData(MediaStreamGraph* aGraph,
mPacketizer->PacketSize() != aRate/100u ||
mPacketizer->Channels() != aChannels) {
// It's ok to drop the audio still in the packetizer here.
mPacketizer = new AudioPacketizer<AudioDataValue, int16_t>(aRate/100, aChannels);
mPacketizer =
new AudioPacketizer<AudioDataValue, int16_t>(aRate/100, aChannels);
}
mPacketizer->Input(aBuffer, static_cast<uint32_t>(aFrames));
@ -489,6 +497,69 @@ MediaEngineWebRTCMicrophoneSource::NotifyInputData(MediaStreamGraph* aGraph,
}
}
template<typename T>
void
MediaEngineWebRTCMicrophoneSource::InsertInGraph(const T* aBuffer,
size_t aFrames,
uint32_t aChannels)
{
if (mState != kStarted) {
return;
}
size_t len = mSources.Length();
for (size_t i = 0; i < len; i++) {
if (!mSources[i]) {
continue;
}
RefPtr<SharedBuffer> buffer =
SharedBuffer::Create(aFrames * aChannels * sizeof(T));
PodCopy(static_cast<T*>(buffer->Data()),
aBuffer, aFrames * aChannels);
TimeStamp insertTime;
// Make sure we include the stream and the track.
// The 0:1 is a flag to note when we've done the final insert for a given input block.
LogTime(AsyncLatencyLogger::AudioTrackInsertion,
LATENCY_STREAM_ID(mSources[i].get(), mTrackID),
(i+1 < len) ? 0 : 1, insertTime);
nsAutoPtr<AudioSegment> segment(new AudioSegment());
AutoTArray<const T*, 1> channels;
// XXX Bug 971528 - Support stereo capture in gUM
MOZ_ASSERT(aChannels == 1,
"GraphDriver only supports us stereo audio for now");
channels.AppendElement(static_cast<T*>(buffer->Data()));
segment->AppendFrames(buffer.forget(), channels, aFrames,
mPrincipalHandles[i]);
segment->GetStartTime(insertTime);
RUN_ON_THREAD(mThread,
WrapRunnable(mSources[i], &SourceMediaStream::AppendToTrack,
mTrackID, segment,
static_cast<AudioSegment*>(nullptr)),
NS_DISPATCH_NORMAL);
}
}
// Called back on GraphDriver thread!
// Note this can be called back after ::Shutdown()
void
MediaEngineWebRTCMicrophoneSource::NotifyInputData(MediaStreamGraph* aGraph,
const AudioDataValue* aBuffer,
size_t aFrames,
TrackRate aRate,
uint32_t aChannels)
{
// If some processing is necessary, packetize and insert in the WebRTC.org
// code. Otherwise, directly insert the mic data in the MSG, bypassing all processing.
if (PassThrough()) {
InsertInGraph<AudioDataValue>(aBuffer, aFrames, aChannels);
} else {
PacketizeAndProcess(aGraph, aBuffer, aFrames, aRate, aChannels);
}
}
#define ResetProcessingIfNeeded(_processing) \
do { \
webrtc::_processing##Modes mode; \
@ -678,6 +749,7 @@ MediaEngineWebRTCMicrophoneSource::Process(int channel,
sample *audio10ms, int length,
int samplingFreq, bool isStereo)
{
MOZ_ASSERT(!PassThrough(), "This should be bypassed when in PassThrough mode.");
// On initial capture, throw away all far-end data except the most recent sample
// since it's already irrelevant and we want to keep avoid confusing the AEC far-end
// input code with "old" audio.
@ -710,33 +782,8 @@ MediaEngineWebRTCMicrophoneSource::Process(int channel,
uint32_t len = mSources.Length();
for (uint32_t i = 0; i < len; i++) {
RefPtr<SharedBuffer> buffer = SharedBuffer::Create(length * sizeof(sample));
sample* dest = static_cast<sample*>(buffer->Data());
memcpy(dest, audio10ms, length * sizeof(sample));
nsAutoPtr<AudioSegment> segment(new AudioSegment());
AutoTArray<const sample*,1> channels;
channels.AppendElement(dest);
segment->AppendFrames(buffer.forget(), channels, length,
mPrincipalHandles[i]);
TimeStamp insertTime;
segment->GetStartTime(insertTime);
if (mSources[i]) {
// Make sure we include the stream and the track.
// The 0:1 is a flag to note when we've done the final insert for a given input block.
LogTime(AsyncLatencyLogger::AudioTrackInsertion, LATENCY_STREAM_ID(mSources[i].get(), mTrackID),
(i+1 < len) ? 0 : 1, insertTime);
// This is safe from any thread, and is safe if the track is Finished
// or Destroyed.
// Note: due to evil magic, the nsAutoPtr<AudioSegment>'s ownership transfers to
// the Runnable (AutoPtr<> = AutoPtr<>)
RUN_ON_THREAD(mThread, WrapRunnable(mSources[i], &SourceMediaStream::AppendToTrack,
mTrackID, segment, (AudioSegment *) nullptr),
NS_DISPATCH_NORMAL);
}
MOZ_ASSERT(!isStereo);
InsertInGraph<int16_t>(audio10ms, length, 1);
}
return;