mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-16 23:05:42 +00:00
Bug 1271585 - Part 2 - Synchronously insert audio frames from the microphone in the MSG if possible. r=pehrsons,jesup
MozReview-Commit-ID: Fm2woel600v
This commit is contained in:
parent
209bdf8893
commit
28b7a4423b
@ -58,6 +58,10 @@ public:
|
||||
#else
|
||||
static const int DEFAULT_SAMPLE_RATE = 16000;
|
||||
#endif
|
||||
// This allows using whatever rate the graph is using for the
|
||||
// MediaStreamTrack. This is useful for microphone data, we know it's already
|
||||
// at the correct rate for insertion in the MSG.
|
||||
static const int USE_GRAPH_RATE = -1;
|
||||
|
||||
/* Populate an array of video sources in the nsTArray. Also include devices
|
||||
* that are currently unavailable. */
|
||||
|
@ -434,6 +434,7 @@ public:
|
||||
, mSampleFrequency(MediaEngine::DEFAULT_SAMPLE_RATE)
|
||||
, mPlayoutDelay(0)
|
||||
, mNullTransport(nullptr)
|
||||
, mSkipProcessing(false)
|
||||
{
|
||||
MOZ_ASSERT(aVoiceEnginePtr);
|
||||
MOZ_ASSERT(aAudioInput);
|
||||
@ -515,6 +516,22 @@ private:
|
||||
bool InitEngine();
|
||||
void DeInitEngine();
|
||||
|
||||
// This is true when all processing is disabled, we can skip
|
||||
// packetization, resampling and other processing passes.
|
||||
bool PassThrough() {
|
||||
return mSkipProcessing;
|
||||
}
|
||||
template<typename T>
|
||||
void InsertInGraph(const T* aBuffer,
|
||||
size_t aFrames,
|
||||
uint32_t aChannels);
|
||||
|
||||
void PacketizeAndProcess(MediaStreamGraph* aGraph,
|
||||
const AudioDataValue* aBuffer,
|
||||
size_t aFrames,
|
||||
TrackRate aRate,
|
||||
uint32_t aChannels);
|
||||
|
||||
webrtc::VoiceEngine* mVoiceEngine;
|
||||
RefPtr<mozilla::AudioInput> mAudioInput;
|
||||
RefPtr<WebRTCAudioDataListener> mListener;
|
||||
@ -555,6 +572,10 @@ private:
|
||||
NullTransport *mNullTransport;
|
||||
|
||||
nsTArray<int16_t> mInputBuffer;
|
||||
// mSkipProcessing is true if none of the processing passes are enabled,
|
||||
// because of prefs or constraints. This allows simply copying the audio into
|
||||
// the MSG, skipping resampling and the whole webrtc.org code.
|
||||
bool mSkipProcessing;
|
||||
};
|
||||
|
||||
class MediaEngineWebRTC : public MediaEngine
|
||||
|
@ -298,6 +298,12 @@ MediaEngineWebRTCMicrophoneSource::Restart(const dom::MediaTrackConstraints& aCo
|
||||
LOG(("%s Error setting NoiseSuppression Status: %d ",__FUNCTION__, error));
|
||||
}
|
||||
}
|
||||
|
||||
mSkipProcessing = !(aec_on || agc_on || noise_on);
|
||||
if (mSkipProcessing) {
|
||||
mSampleFrequency = MediaEngine::USE_GRAPH_RATE;
|
||||
}
|
||||
|
||||
return NS_OK;
|
||||
}
|
||||
|
||||
@ -344,6 +350,9 @@ MediaEngineWebRTCMicrophoneSource::Start(SourceMediaStream *aStream,
|
||||
}
|
||||
|
||||
AudioSegment* segment = new AudioSegment();
|
||||
if (mSampleFrequency == MediaEngine::USE_GRAPH_RATE) {
|
||||
mSampleFrequency = aStream->GraphRate();
|
||||
}
|
||||
aStream->AddAudioTrack(aID, mSampleFrequency, 0, segment, SourceMediaStream::ADDTRACK_QUEUED);
|
||||
|
||||
// XXX Make this based on the pref.
|
||||
@ -457,6 +466,82 @@ MediaEngineWebRTCMicrophoneSource::NotifyOutputData(MediaStreamGraph* aGraph,
|
||||
{
|
||||
}
|
||||
|
||||
void
|
||||
MediaEngineWebRTCMicrophoneSource::PacketizeAndProcess(MediaStreamGraph* aGraph,
|
||||
const AudioDataValue* aBuffer,
|
||||
size_t aFrames,
|
||||
TrackRate aRate,
|
||||
uint32_t aChannels)
|
||||
{
|
||||
// This will call Process() with data coming out of the AEC/NS/AGC/etc chain
|
||||
if (!mPacketizer ||
|
||||
mPacketizer->PacketSize() != aRate/100u ||
|
||||
mPacketizer->Channels() != aChannels) {
|
||||
// It's ok to drop the audio still in the packetizer here.
|
||||
mPacketizer =
|
||||
new AudioPacketizer<AudioDataValue, int16_t>(aRate/100, aChannels);
|
||||
}
|
||||
|
||||
mPacketizer->Input(aBuffer, static_cast<uint32_t>(aFrames));
|
||||
|
||||
while (mPacketizer->PacketsAvailable()) {
|
||||
uint32_t samplesPerPacket = mPacketizer->PacketSize() *
|
||||
mPacketizer->Channels();
|
||||
if (mInputBuffer.Length() < samplesPerPacket) {
|
||||
mInputBuffer.SetLength(samplesPerPacket);
|
||||
}
|
||||
int16_t* packet = mInputBuffer.Elements();
|
||||
mPacketizer->Output(packet);
|
||||
|
||||
mVoERender->ExternalRecordingInsertData(packet, samplesPerPacket, aRate, 0);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void
|
||||
MediaEngineWebRTCMicrophoneSource::InsertInGraph(const T* aBuffer,
|
||||
size_t aFrames,
|
||||
uint32_t aChannels)
|
||||
{
|
||||
if (mState != kStarted) {
|
||||
return;
|
||||
}
|
||||
|
||||
size_t len = mSources.Length();
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
if (!mSources[i]) {
|
||||
continue;
|
||||
}
|
||||
RefPtr<SharedBuffer> buffer =
|
||||
SharedBuffer::Create(aFrames * aChannels * sizeof(T));
|
||||
PodCopy(static_cast<T*>(buffer->Data()),
|
||||
aBuffer, aFrames * aChannels);
|
||||
|
||||
TimeStamp insertTime;
|
||||
// Make sure we include the stream and the track.
|
||||
// The 0:1 is a flag to note when we've done the final insert for a given input block.
|
||||
LogTime(AsyncLatencyLogger::AudioTrackInsertion,
|
||||
LATENCY_STREAM_ID(mSources[i].get(), mTrackID),
|
||||
(i+1 < len) ? 0 : 1, insertTime);
|
||||
|
||||
nsAutoPtr<AudioSegment> segment(new AudioSegment());
|
||||
AutoTArray<const T*, 1> channels;
|
||||
// XXX Bug 971528 - Support stereo capture in gUM
|
||||
MOZ_ASSERT(aChannels == 1,
|
||||
"GraphDriver only supports us stereo audio for now");
|
||||
channels.AppendElement(static_cast<T*>(buffer->Data()));
|
||||
segment->AppendFrames(buffer.forget(), channels, aFrames,
|
||||
mPrincipalHandles[i]);
|
||||
segment->GetStartTime(insertTime);
|
||||
|
||||
RUN_ON_THREAD(mThread,
|
||||
WrapRunnable(mSources[i], &SourceMediaStream::AppendToTrack,
|
||||
mTrackID, segment,
|
||||
static_cast<AudioSegment*>(nullptr)),
|
||||
NS_DISPATCH_NORMAL);
|
||||
}
|
||||
}
|
||||
|
||||
// Called back on GraphDriver thread!
|
||||
// Note this can be called back after ::Shutdown()
|
||||
void
|
||||
@ -466,26 +551,12 @@ MediaEngineWebRTCMicrophoneSource::NotifyInputData(MediaStreamGraph* aGraph,
|
||||
TrackRate aRate,
|
||||
uint32_t aChannels)
|
||||
{
|
||||
// This will call Process() with data coming out of the AEC/NS/AGC/etc chain
|
||||
if (!mPacketizer ||
|
||||
mPacketizer->PacketSize() != aRate/100u ||
|
||||
mPacketizer->Channels() != aChannels) {
|
||||
// It's ok to drop the audio still in the packetizer here.
|
||||
mPacketizer = new AudioPacketizer<AudioDataValue, int16_t>(aRate/100, aChannels);
|
||||
}
|
||||
|
||||
mPacketizer->Input(aBuffer, static_cast<uint32_t>(aFrames));
|
||||
|
||||
while (mPacketizer->PacketsAvailable()) {
|
||||
uint32_t samplesPerPacket = mPacketizer->PacketSize() *
|
||||
mPacketizer->Channels();
|
||||
if (mInputBuffer.Length() < samplesPerPacket) {
|
||||
mInputBuffer.SetLength(samplesPerPacket);
|
||||
}
|
||||
int16_t* packet = mInputBuffer.Elements();
|
||||
mPacketizer->Output(packet);
|
||||
|
||||
mVoERender->ExternalRecordingInsertData(packet, samplesPerPacket, aRate, 0);
|
||||
// If some processing is necessary, packetize and insert in the WebRTC.org
|
||||
// code. Otherwise, directly insert the mic data in the MSG, bypassing all processing.
|
||||
if (PassThrough()) {
|
||||
InsertInGraph<AudioDataValue>(aBuffer, aFrames, aChannels);
|
||||
} else {
|
||||
PacketizeAndProcess(aGraph, aBuffer, aFrames, aRate, aChannels);
|
||||
}
|
||||
}
|
||||
|
||||
@ -678,6 +749,7 @@ MediaEngineWebRTCMicrophoneSource::Process(int channel,
|
||||
sample *audio10ms, int length,
|
||||
int samplingFreq, bool isStereo)
|
||||
{
|
||||
MOZ_ASSERT(!PassThrough(), "This should be bypassed when in PassThrough mode.");
|
||||
// On initial capture, throw away all far-end data except the most recent sample
|
||||
// since it's already irrelevant and we want to keep avoid confusing the AEC far-end
|
||||
// input code with "old" audio.
|
||||
@ -710,33 +782,8 @@ MediaEngineWebRTCMicrophoneSource::Process(int channel,
|
||||
|
||||
uint32_t len = mSources.Length();
|
||||
for (uint32_t i = 0; i < len; i++) {
|
||||
RefPtr<SharedBuffer> buffer = SharedBuffer::Create(length * sizeof(sample));
|
||||
|
||||
sample* dest = static_cast<sample*>(buffer->Data());
|
||||
memcpy(dest, audio10ms, length * sizeof(sample));
|
||||
|
||||
nsAutoPtr<AudioSegment> segment(new AudioSegment());
|
||||
AutoTArray<const sample*,1> channels;
|
||||
channels.AppendElement(dest);
|
||||
segment->AppendFrames(buffer.forget(), channels, length,
|
||||
mPrincipalHandles[i]);
|
||||
TimeStamp insertTime;
|
||||
segment->GetStartTime(insertTime);
|
||||
|
||||
if (mSources[i]) {
|
||||
// Make sure we include the stream and the track.
|
||||
// The 0:1 is a flag to note when we've done the final insert for a given input block.
|
||||
LogTime(AsyncLatencyLogger::AudioTrackInsertion, LATENCY_STREAM_ID(mSources[i].get(), mTrackID),
|
||||
(i+1 < len) ? 0 : 1, insertTime);
|
||||
|
||||
// This is safe from any thread, and is safe if the track is Finished
|
||||
// or Destroyed.
|
||||
// Note: due to evil magic, the nsAutoPtr<AudioSegment>'s ownership transfers to
|
||||
// the Runnable (AutoPtr<> = AutoPtr<>)
|
||||
RUN_ON_THREAD(mThread, WrapRunnable(mSources[i], &SourceMediaStream::AppendToTrack,
|
||||
mTrackID, segment, (AudioSegment *) nullptr),
|
||||
NS_DISPATCH_NORMAL);
|
||||
}
|
||||
MOZ_ASSERT(!isStereo);
|
||||
InsertInGraph<int16_t>(audio10ms, length, 1);
|
||||
}
|
||||
|
||||
return;
|
||||
|
Loading…
Reference in New Issue
Block a user