From 92bcfe62821a54cc658c76be494786af2b89dca3 Mon Sep 17 00:00:00 2001 From: Vasanthakumar Pandurangan Date: Sat, 22 Mar 2014 18:40:05 +0530 Subject: [PATCH] Bug 976172 - Part 2 Enable audio offloading in gecko r=padenot Implements a new class AudioOffloadPlayer to make use of Android::AudioTrack class's offloading capability --- configure.in | 3 + content/media/omx/AudioOffloadPlayer.cpp | 669 +++++++++++++++++++++++ content/media/omx/AudioOffloadPlayer.h | 251 +++++++++ content/media/omx/AudioOutput.cpp | 244 +++++++++ content/media/omx/AudioOutput.h | 114 ++++ content/media/omx/AudioSink.h | 89 +++ 6 files changed, 1370 insertions(+) create mode 100755 content/media/omx/AudioOffloadPlayer.cpp create mode 100755 content/media/omx/AudioOffloadPlayer.h create mode 100644 content/media/omx/AudioOutput.cpp create mode 100644 content/media/omx/AudioOutput.h create mode 100644 content/media/omx/AudioSink.h diff --git a/configure.in b/configure.in index ac38f4014ad1..c7b3eab26993 100644 --- a/configure.in +++ b/configure.in @@ -271,6 +271,9 @@ if test -n "$gonkdir" ; then MOZ_OMX_DECODER=1 MOZ_OMX_ENCODER=1 AC_DEFINE(MOZ_OMX_ENCODER) + MOZ_AUDIO_OFFLOAD=1 + AC_SUBST(MOZ_AUDIO_OFFLOAD) + AC_DEFINE(MOZ_AUDIO_OFFLOAD) ;; *) diff --git a/content/media/omx/AudioOffloadPlayer.cpp b/content/media/omx/AudioOffloadPlayer.cpp new file mode 100755 index 000000000000..48ed60a24968 --- /dev/null +++ b/content/media/omx/AudioOffloadPlayer.cpp @@ -0,0 +1,669 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* + * Copyright (c) 2014 The Linux Foundation. All rights reserved. + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "AudioOffloadPlayer.h" +#include "nsComponentManagerUtils.h" +#include "nsITimer.h" +#include "mozilla/dom/HTMLMediaElement.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace android; + +namespace mozilla { + +#ifdef PR_LOGGING +PRLogModuleInfo* gAudioOffloadPlayerLog; +#define AUDIO_OFFLOAD_LOG(type, msg) \ + PR_LOG(gAudioOffloadPlayerLog, type, msg) +#else +#define AUDIO_OFFLOAD_LOG(type, msg) +#endif + +AudioOffloadPlayer::AudioOffloadPlayer(MediaOmxDecoder* aObserver) : + mObserver(aObserver), + mInputBuffer(nullptr), + mSampleRate(0), + mSeeking(false), + mSeekDuringPause(false), + mReachedEOS(false), + mSeekTimeUs(0), + mStartPosUs(0), + mPositionTimeMediaUs(-1), + mStarted(false), + mPlaying(false), + mIsElementVisible(true) +{ + MOZ_ASSERT(NS_IsMainThread()); + +#ifdef PR_LOGGING + if (!gAudioOffloadPlayerLog) { + gAudioOffloadPlayerLog = PR_NewLogModule("AudioOffloadPlayer"); + } +#endif + + CHECK(aObserver); + mSessionId = AudioSystem::newAudioSessionId(); + AudioSystem::acquireAudioSessionId(mSessionId); + mAudioSink = new AudioOutput(mSessionId, + IPCThreadState::self()->getCallingUid()); +} + +AudioOffloadPlayer::~AudioOffloadPlayer() +{ + if (mStarted) { + Reset(); + } + AudioSystem::releaseAudioSessionId(mSessionId); +} + +void AudioOffloadPlayer::SetSource(const sp &aSource) +{ + MOZ_ASSERT(NS_IsMainThread()); + CHECK(!mSource.get()); + + mSource = aSource; +} + +status_t AudioOffloadPlayer::Start(bool aSourceAlreadyStarted) +{ + MOZ_ASSERT(NS_IsMainThread()); + CHECK(!mStarted); + CHECK(mSource.get()); + + status_t err; + CHECK(mAudioSink.get()); + + if (!aSourceAlreadyStarted) { + err = mSource->start(); + + if (err != OK) { + return err; + } + } + + MediaSource::ReadOptions options; + if (mSeeking) { + options.setSeekTo(mSeekTimeUs); + mSeeking = false; + } + + sp format = mSource->getFormat(); + const char* mime; + int avgBitRate = -1; + int32_t channelMask; + int32_t numChannels; + int64_t durationUs = -1; + audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT; + uint32_t flags = AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; + audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER; + + CHECK(format->findCString(kKeyMIMEType, &mime)); + CHECK(format->findInt32(kKeySampleRate, &mSampleRate)); + CHECK(format->findInt32(kKeyChannelCount, &numChannels)); + format->findInt32(kKeyBitRate, &avgBitRate); + format->findInt64(kKeyDuration, &durationUs); + + if(!format->findInt32(kKeyChannelMask, &channelMask)) { + channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER; + } + + if (mapMimeToAudioFormat(audioFormat, mime) != OK) { + AUDIO_OFFLOAD_LOG(PR_LOG_ERROR, ("Couldn't map mime type \"%s\" to a valid " + "AudioSystem::audio_format", mime)); + audioFormat = AUDIO_FORMAT_INVALID; + } + + offloadInfo.duration_us = durationUs; + offloadInfo.sample_rate = mSampleRate; + offloadInfo.channel_mask = channelMask; + offloadInfo.format = audioFormat; + offloadInfo.stream_type = AUDIO_STREAM_MUSIC; + offloadInfo.bit_rate = avgBitRate; + offloadInfo.has_video = false; + offloadInfo.is_streaming = false; + + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("isOffloadSupported: SR=%u, CM=0x%x, " + "Format=0x%x, StreamType=%d, BitRate=%u, duration=%lld us, has_video=%d", + offloadInfo.sample_rate, offloadInfo.channel_mask, offloadInfo.format, + offloadInfo.stream_type, offloadInfo.bit_rate, offloadInfo.duration_us, + offloadInfo.has_video)); + + err = mAudioSink->Open(mSampleRate, + numChannels, + channelMask, + audioFormat, + &AudioOffloadPlayer::AudioSinkCallback, + this, + (audio_output_flags_t) flags, + &offloadInfo); + if (err == OK) { + // If the playback is offloaded to h/w we pass the + // HAL some metadata information + // We don't want to do this for PCM because it will be going + // through the AudioFlinger mixer before reaching the hardware + SendMetaDataToHal(mAudioSink, format); + } + mStarted = true; + mPlaying = false; + + return err; +} + +void AudioOffloadPlayer::ChangeState(MediaDecoder::PlayState aState) +{ + MOZ_ASSERT(NS_IsMainThread()); + mPlayState = aState; + + switch (mPlayState) { + case MediaDecoder::PLAY_STATE_PLAYING: + Play(); + StartTimeUpdate(); + break; + + case MediaDecoder::PLAY_STATE_SEEKING: { + int64_t seekTimeUs + = mObserver->GetSeekTime(); + SeekTo(seekTimeUs); + mObserver->ResetSeekTime(); + } break; + + case MediaDecoder::PLAY_STATE_PAUSED: + case MediaDecoder::PLAY_STATE_SHUTDOWN: + // Just pause here during play state shutdown as well to stop playing + // offload track immediately. Resources will be freed by MediaOmxDecoder + Pause(); + break; + + case MediaDecoder::PLAY_STATE_ENDED: + Pause(true); + break; + + default: + break; + } +} + +void AudioOffloadPlayer::Pause(bool aPlayPendingSamples) +{ + MOZ_ASSERT(NS_IsMainThread()); + CHECK(mStarted); + CHECK(mAudioSink.get()); + + if (aPlayPendingSamples) { + mAudioSink->Stop(); + } else { + mAudioSink->Pause(); + } + mPlaying = false; +} + +status_t AudioOffloadPlayer::Play() +{ + MOZ_ASSERT(NS_IsMainThread()); + CHECK(mStarted); + CHECK(mAudioSink.get()); + + status_t err = OK; + err = mAudioSink->Start(); + + if (err == OK) { + mPlaying = true; + } + + return err; +} + +void AudioOffloadPlayer::Reset() +{ + CHECK(mStarted); + CHECK(mAudioSink.get()); + + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("reset: mPlaying=%d mReachedEOS=%d", + mPlaying, mReachedEOS)); + + mAudioSink->Stop(); + // If we're closing and have reached EOS, we don't want to flush + // the track because if it is offloaded there could be a small + // amount of residual data in the hardware buffer which we must + // play to give gapless playback. + // But if we're resetting when paused or before we've reached EOS + // we can't be doing a gapless playback and there could be a large + // amount of data queued in the hardware if the track is offloaded, + // so we must flush to prevent a track switch being delayed playing + // the buffered data that we don't want now + if (!mPlaying || !mReachedEOS) { + mAudioSink->Flush(); + } + + mAudioSink->Close(); + // Make sure to release any buffer we hold onto so that the + // source is able to stop(). + + if (mInputBuffer) { + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Releasing input buffer")); + + mInputBuffer->release(); + mInputBuffer = nullptr; + } + + IPCThreadState::self()->flushCommands(); + StopTimeUpdate(); + + mSeeking = false; + mSeekTimeUs = 0; + mReachedEOS = false; + mStarted = false; + mPlaying = false; + mStartPosUs = 0; +} + +status_t AudioOffloadPlayer::SeekTo(int64_t aTimeUs) +{ + MOZ_ASSERT(NS_IsMainThread()); + CHECK(mAudioSink.get()); + + android::Mutex::Autolock autoLock(mLock); + + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("SeekTo ( %lld )", aTimeUs)); + + mSeeking = true; + mReachedEOS = false; + mPositionTimeMediaUs = -1; + mSeekTimeUs = aTimeUs; + mStartPosUs = aTimeUs; + + nsCOMPtr nsEvent = NS_NewRunnableMethod(mObserver, + &MediaDecoder::SeekingStarted); + NS_DispatchToMainThread(nsEvent, NS_DISPATCH_NORMAL); + + if (mPlaying) { + mAudioSink->Pause(); + } + + mAudioSink->Flush(); + + if (mPlaying) { + mAudioSink->Start(); + } else { + mSeekDuringPause = true; + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Fake seek complete during pause")); + + nsCOMPtr nsEvent = NS_NewRunnableMethod(mObserver, + &MediaDecoder::SeekingStopped); + NS_DispatchToMainThread(nsEvent, NS_DISPATCH_NORMAL); + } + + return OK; +} + +double AudioOffloadPlayer::GetMediaTimeSecs() +{ + MOZ_ASSERT(NS_IsMainThread()); + return (static_cast(GetMediaTimeUs()) / + static_cast(USECS_PER_S)); +} + +int64_t AudioOffloadPlayer::GetMediaTimeUs() +{ + android::Mutex::Autolock autoLock(mLock); + + int64_t playPosition = 0; + if (mSeeking) { + return mSeekTimeUs; + } + + playPosition = GetOutputPlayPositionUs_l(); + if (!mReachedEOS) { + mPositionTimeMediaUs = playPosition; + } + + return mPositionTimeMediaUs; +} + +int64_t AudioOffloadPlayer::GetOutputPlayPositionUs_l() const +{ + CHECK(mAudioSink.get()); + uint32_t playedSamples = 0; + + mAudioSink->GetPosition(&playedSamples); + + const int64_t playedUs = (static_cast(playedSamples) * 1000000 ) / + mSampleRate; + + // HAL position is relative to the first buffer we sent at mStartPosUs + const int64_t renderedDuration = mStartPosUs + playedUs; + return renderedDuration; +} + +void AudioOffloadPlayer::NotifyAudioEOS() +{ + nsCOMPtr nsEvent = NS_NewRunnableMethod(mObserver, + &MediaDecoder::PlaybackEnded); + NS_DispatchToMainThread(nsEvent, NS_DISPATCH_NORMAL); +} + +void AudioOffloadPlayer::NotifyPositionChanged() +{ + nsCOMPtr nsEvent = NS_NewRunnableMethod(mObserver, + &MediaOmxDecoder::PlaybackPositionChanged); + NS_DispatchToMainThread(nsEvent, NS_DISPATCH_NORMAL); +} + +void AudioOffloadPlayer::NotifyAudioTearDown() +{ + nsCOMPtr nsEvent = NS_NewRunnableMethod(mObserver, + &MediaOmxDecoder::AudioOffloadTearDown); + NS_DispatchToMainThread(nsEvent, NS_DISPATCH_NORMAL); +} + +// static +size_t AudioOffloadPlayer::AudioSinkCallback(AudioSink* aAudioSink, + void* aBuffer, + size_t aSize, + void* aCookie, + AudioSink::cb_event_t aEvent) +{ + AudioOffloadPlayer* me = (AudioOffloadPlayer*) aCookie; + + switch (aEvent) { + + case AudioSink::CB_EVENT_FILL_BUFFER: + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Notify Audio position changed")); + me->NotifyPositionChanged(); + return me->FillBuffer(aBuffer, aSize); + + case AudioSink::CB_EVENT_STREAM_END: + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Notify Audio EOS")); + me->mReachedEOS = true; + me->NotifyAudioEOS(); + break; + + case AudioSink::CB_EVENT_TEAR_DOWN: + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Notify Tear down event")); + me->NotifyAudioTearDown(); + break; + + default: + AUDIO_OFFLOAD_LOG(PR_LOG_ERROR, ("Unknown event %d from audio sink", + aEvent)); + break; + } + return 0; +} + +size_t AudioOffloadPlayer::FillBuffer(void* aData, size_t aSize) +{ + CHECK(mAudioSink.get()); + + if (mReachedEOS) { + return 0; + } + + bool postSeekComplete = false; + + size_t sizeDone = 0; + size_t sizeRemaining = aSize; + while (sizeRemaining > 0) { + MediaSource::ReadOptions options; + bool refreshSeekTime = false; + + { + android::Mutex::Autolock autoLock(mLock); + + if (mSeeking) { + options.setSeekTo(mSeekTimeUs); + refreshSeekTime = true; + + if (mInputBuffer) { + mInputBuffer->release(); + mInputBuffer = nullptr; + } + + mSeeking = false; + postSeekComplete = true; + } + } + + if (!mInputBuffer) { + + status_t err; + err = mSource->read(&mInputBuffer, &options); + + CHECK((!err && mInputBuffer) || (err && !mInputBuffer)); + + android::Mutex::Autolock autoLock(mLock); + + if (err != OK) { + AUDIO_OFFLOAD_LOG(PR_LOG_ERROR, ("Error while reading media source %d " + "Ok to receive EOS error at end", err)); + if (!mReachedEOS) { + // After seek there is a possible race condition if + // OffloadThread is observing state_stopping_1 before + // framesReady() > 0. Ensure sink stop is called + // after last buffer is released. This ensures the + // partial buffer is written to the driver before + // stopping one is observed.The drawback is that + // there will be an unnecessary call to the parser + // after parser signalled EOS. + if (sizeDone > 0) { + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("send Partial buffer down")); + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("skip calling stop till next" + " fillBuffer")); + break; + } + // no more buffers to push - stop() and wait for STREAM_END + // don't set mReachedEOS until stream end received + mAudioSink->Stop(); + } + break; + } + + if(mInputBuffer->range_length() != 0) { + CHECK(mInputBuffer->meta_data()->findInt64( + kKeyTime, &mPositionTimeMediaUs)); + } + + // need to adjust the mStartPosUs for offload decoding since parser + // might not be able to get the exact seek time requested. + if (refreshSeekTime) { + if (postSeekComplete) { + + if (!mSeekDuringPause) { + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("FillBuffer posting SEEK_COMPLETE")); + nsCOMPtr nsEvent = NS_NewRunnableMethod(mObserver, + &MediaDecoder::SeekingStopped); + NS_DispatchToMainThread(nsEvent, NS_DISPATCH_NORMAL); + } else { + // Callback is already called for seek during pause. Just reset the + // flag + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Not posting seek complete as its" + " already faked")); + mSeekDuringPause = false; + } + + NotifyPositionChanged(); + postSeekComplete = false; + } + + mStartPosUs = mPositionTimeMediaUs; + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Adjust seek time to: %.2f", + mStartPosUs / 1E6)); + // clear seek time with mLock locked and once we have valid + // mPositionTimeMediaUs + // before clearing mSeekTimeUs check if a new seek request has been + // received while we were reading from the source with mLock released. + if (!mSeeking) { + mSeekTimeUs = 0; + } + } + } + + if (mInputBuffer->range_length() == 0) { + mInputBuffer->release(); + mInputBuffer = nullptr; + continue; + } + + size_t copy = sizeRemaining; + if (copy > mInputBuffer->range_length()) { + copy = mInputBuffer->range_length(); + } + + memcpy((char *)aData + sizeDone, + (const char *)mInputBuffer->data() + mInputBuffer->range_offset(), + copy); + + mInputBuffer->set_range(mInputBuffer->range_offset() + copy, + mInputBuffer->range_length() - copy); + + sizeDone += copy; + sizeRemaining -= copy; + } + return sizeDone; +} + +void AudioOffloadPlayer::SetElementVisibility(bool aIsVisible) +{ + MOZ_ASSERT(NS_IsMainThread()); + mIsElementVisible = aIsVisible; + if (mIsElementVisible) { + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Element is visible. Start time update")); + StartTimeUpdate(); + } +} + +static void TimeUpdateCallback(nsITimer* aTimer, void* aClosure) +{ + AudioOffloadPlayer* player = static_cast(aClosure); + player->TimeUpdate(); +} + +void AudioOffloadPlayer::TimeUpdate() +{ + MOZ_ASSERT(NS_IsMainThread()); + TimeStamp now = TimeStamp::Now(); + + // If TIMEUPDATE_MS has passed since the last fire update event fired, fire + // another timeupdate event. + if ((mLastFireUpdateTime.IsNull() || + now - mLastFireUpdateTime >= + TimeDuration::FromMilliseconds(TIMEUPDATE_MS))) { + mLastFireUpdateTime = now; + NotifyPositionChanged(); + } + + if (mPlayState != MediaDecoder::PLAY_STATE_PLAYING || !mIsElementVisible) { + StopTimeUpdate(); + } +} + +nsresult AudioOffloadPlayer::StartTimeUpdate() +{ + MOZ_ASSERT(NS_IsMainThread()); + if (mTimeUpdateTimer) { + return NS_OK; + } + + mTimeUpdateTimer = do_CreateInstance("@mozilla.org/timer;1"); + return mTimeUpdateTimer->InitWithFuncCallback(TimeUpdateCallback, + this, + TIMEUPDATE_MS, + nsITimer::TYPE_REPEATING_SLACK); +} + +nsresult AudioOffloadPlayer::StopTimeUpdate() +{ + MOZ_ASSERT(NS_IsMainThread()); + if (!mTimeUpdateTimer) { + return NS_OK; + } + + nsresult rv = mTimeUpdateTimer->Cancel(); + mTimeUpdateTimer = nullptr; + return rv; +} + +MediaDecoderOwner::NextFrameStatus AudioOffloadPlayer::GetNextFrameStatus() +{ + MOZ_ASSERT(NS_IsMainThread()); + if (mPlayState == MediaDecoder::PLAY_STATE_SEEKING) { + return MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING; + } else if (mPlayState == MediaDecoder::PLAY_STATE_ENDED) { + return MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE; + } else { + return MediaDecoderOwner::NEXT_FRAME_AVAILABLE; + } +} + +void AudioOffloadPlayer::SendMetaDataToHal(sp& aSink, + const sp& aMeta) +{ + int32_t sampleRate = 0; + int32_t bitRate = 0; + int32_t channelMask = 0; + int32_t delaySamples = 0; + int32_t paddingSamples = 0; + CHECK(aSink.get()); + + AudioParameter param = AudioParameter(); + + if (aMeta->findInt32(kKeySampleRate, &sampleRate)) { + param.addInt(String8(AUDIO_OFFLOAD_CODEC_SAMPLE_RATE), sampleRate); + } + if (aMeta->findInt32(kKeyChannelMask, &channelMask)) { + param.addInt(String8(AUDIO_OFFLOAD_CODEC_NUM_CHANNEL), channelMask); + } + if (aMeta->findInt32(kKeyBitRate, &bitRate)) { + param.addInt(String8(AUDIO_OFFLOAD_CODEC_AVG_BIT_RATE), bitRate); + } + if (aMeta->findInt32(kKeyEncoderDelay, &delaySamples)) { + param.addInt(String8(AUDIO_OFFLOAD_CODEC_DELAY_SAMPLES), delaySamples); + } + if (aMeta->findInt32(kKeyEncoderPadding, &paddingSamples)) { + param.addInt(String8(AUDIO_OFFLOAD_CODEC_PADDING_SAMPLES), paddingSamples); + } + + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("SendMetaDataToHal: bitRate %d," + " sampleRate %d, chanMask %d, delaySample %d, paddingSample %d", bitRate, + sampleRate, channelMask, delaySamples, paddingSamples)); + + aSink->SetParameters(param.toString()); + return; +} + +void AudioOffloadPlayer::SetVolume(double aVolume) +{ + MOZ_ASSERT(NS_IsMainThread()); + CHECK(mAudioSink.get()); + mAudioSink->SetVolume((float) aVolume); +} + +} // namespace mozilla diff --git a/content/media/omx/AudioOffloadPlayer.h b/content/media/omx/AudioOffloadPlayer.h new file mode 100755 index 000000000000..3b0d5f73d4a8 --- /dev/null +++ b/content/media/omx/AudioOffloadPlayer.h @@ -0,0 +1,251 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* + * Copyright (c) 2014 The Linux Foundation. All rights reserved. + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AUDIO_OFFLOAD_PLAYER_H_ +#define AUDIO_OFFLOAD_PLAYER_H_ + +#include +#include +#include +#include +#include + +#include "AudioOutput.h" + +#include "MediaDecoderOwner.h" +#include "MediaOmxDecoder.h" + +namespace mozilla { + +class MediaOmxDecoder; + +/** + * AudioOffloadPlayer adds support for audio tunneling to a digital signal + * processor (DSP) in the device chipset. With tunneling, audio decoding is + * off-loaded to the DSP, waking the application processor less often and using + * less battery + * + * This depends on offloading capability provided by Android KK AudioTrack class + * + * Audio playback is based on pull mechanism, whenever audio sink needs + * data, FillBuffer() will read data from compressed audio source and provide + * it to the sink + * + * Also this class passes state changes (play/pause/seek) from MediaOmxDecoder + * to AudioSink as well as provide AudioSink status (position changed, + * playback ended, seek complete, audio tear down) back to MediaOmxDecoder + * + * It acts as a bridge between MediaOmxDecoder and AudioSink during + * offload playback + */ + +class AudioOffloadPlayer : public AudioOffloadPlayerBase +{ + typedef android::Mutex Mutex; + typedef android::MetaData MetaData; + typedef android::status_t status_t; + typedef android::AudioTrack AudioTrack; + typedef android::MediaBuffer MediaBuffer; + typedef android::MediaSource MediaSource; + +public: + enum { + REACHED_EOS, + SEEK_COMPLETE + }; + + AudioOffloadPlayer(MediaOmxDecoder* aDecoder = nullptr); + + ~AudioOffloadPlayer(); + + // Caller retains ownership of "aSource". + void SetSource(const android::sp &aSource); + + // Start the source if it's not already started and open the AudioSink to + // create an offloaded audio track + status_t Start(bool aSourceAlreadyStarted = false); + + double GetMediaTimeSecs(); + + // To update progress bar when the element is visible + void SetElementVisibility(bool aIsVisible); + + void ChangeState(MediaDecoder::PlayState aState); + + void SetVolume(double aVolume); + + // Update ready state based on current play state. Not checking data + // availability since offloading is currently done only when whole compressed + // data is available + MediaDecoderOwner::NextFrameStatus GetNextFrameStatus(); + + void TimeUpdate(); + +private: + // Set when audio source is started and audioSink is initialized + // Used only in main thread + bool mStarted; + + // Set when audio sink is started. i.e. playback started + // Used only in main thread + bool mPlaying; + + // Set when playstate is seeking and reset when FillBUffer() acknowledged + // seeking by seeking audio source. Used in main thread and offload + // callback thread, protected by Mutex mLock + bool mSeeking; + + // Once playback reached end of stream (last ~100ms), position provided by DSP + // may be reset/corrupted. This bool is used to avoid that. + // Used in main thread and offload callback thread, protected by Mutex + // mLock + bool mReachedEOS; + + // Set when there is a seek request during pause. + // Used in main thread and offload callback thread, protected by Mutex + // mLock + bool mSeekDuringPause; + + // Set when the HTML Audio Element is visible to the user. + // Used only in main thread + bool mIsElementVisible; + + // Session id given by Android::AudioSystem and used while creating audio sink + // Used only in main thread + int mSessionId; + + // Sample rate of current audio track. Used only in main thread + int mSampleRate; + + // After seeking, positions returned by offlaoded tracks (DSP) will be + // relative to the seeked position. And seeked position may be slightly + // different than given mSeekTimeUs, if audio source cannot find a frame at + // that position. Store seeked position in mStartPosUs and provide + // mStartPosUs + GetPosition() (i.e. absolute position) to MediaOmxDecoder + // Used in main thread and offload callback thread, protected by Mutex + // mLock + int64_t mStartPosUs; + + // Given seek time when there is a request to seek + // Used in main thread and offload callback thread, protected by Mutex + // mLock + int64_t mSeekTimeUs; + + // Positions obtained from offlaoded tracks (DSP) + // Used in main thread and offload callback thread, protected by Mutex + // mLock + int64_t mPositionTimeMediaUs; + + // State obtained from MediaOmxDecoder. Used only in main thread + MediaDecoder::PlayState mPlayState; + + // Protect accessing audio position related variables between main thread and + // offload callback thread + Mutex mLock; + + // Compressed audio source. + // Used in main thread first and later in offload callback thread + android::sp mSource; + + // Audio sink wrapper to access offloaded audio tracks + // Used in main thread and offload callback thread, access is protected by + // Race conditions are protected in underlying Android::AudioTrack class + android::sp mAudioSink; + + // Buffer used to get date from audio source. Used in offload callback thread + MediaBuffer* mInputBuffer; + + // MediaOmxDecoder object used mainly to notify the audio sink status + MediaOmxDecoder* mObserver; + + TimeStamp mLastFireUpdateTime; + // Timer to trigger position changed events + nsCOMPtr mTimeUpdateTimer; + + int64_t GetMediaTimeUs(); + // Provide the playback position in microseconds from total number of + // frames played by audio track + int64_t GetOutputPlayPositionUs_l() const; + + // Fill the buffer given by audio sink with data from compressed audio + // source. Also handles the seek by seeking audio source and stop the sink in + // case of error + size_t FillBuffer(void *aData, size_t aSize); + + // Called by AudioSink when it needs data, to notify EOS or tear down event + static size_t AudioSinkCallback(AudioSink *aAudioSink, + void *aData, + size_t aSize, + void *aMe, + AudioSink::cb_event_t aEvent); + + bool IsSeeking(); + + // Set mSeekTime to the given position and restart the sink. Actual seek + // happens in FillBuffer(). To MediaDecoder, send SeekingStarted event always + // and SeekingStopped event when the play state is paused. + // When decoding and playing happens separately, if there is a seek during + // pause, we can decode and keep data ready. + // In case of offload player, no way to seek during pause. So just fake that + // seek is done. + status_t SeekTo(int64_t aTimeUs); + + // Close the audio sink, stop time updates, frees the input buffers + void Reset(); + + // Start/Resume the audio sink so that callback will start being called to get + // compressed data + status_t Play(); + + // Stop the audio sink if we need to play till we drain the current buffer. + // or Pause the sink in case we should stop playing immediately + void Pause(bool aPlayPendingSamples = false); + + // When audio is offloaded, application processor wakes up less frequently + // (>1sec) But when Player UI is visible we need to update progress bar + // atleast once in 250ms. Start a timer when player UI becomes visible or + // audio starts playing to send PlaybackPositionChanged events once in 250ms. + // Stop the timer when UI goes invisible or play state is not playing. + // Also make sure timer functions are always called from main thread + nsresult StartTimeUpdate(); + nsresult StopTimeUpdate(); + + // Notify end of stream by sending PlaybackEnded event to observer + // (i.e.MediaDecoder) + void NotifyAudioEOS(); + + // Notify position changed event by sending PlaybackPositionChanged event to + // observer + void NotifyPositionChanged(); + + // Offloaded audio track is invalidated due to usecase change. Notify + // MediaDecoder to re-evaluate offloading options + void NotifyAudioTearDown(); + + // Send information from MetaData to the HAL via AudioSink + void SendMetaDataToHal(android::sp& aSink, + const android::sp& aMeta); + + AudioOffloadPlayer(const AudioOffloadPlayer &); + AudioOffloadPlayer &operator=(const AudioOffloadPlayer &); +}; + +} // namespace mozilla + +#endif // AUDIO_OFFLOAD_PLAYER_H_ diff --git a/content/media/omx/AudioOutput.cpp b/content/media/omx/AudioOutput.cpp new file mode 100644 index 000000000000..7a96ddbab478 --- /dev/null +++ b/content/media/omx/AudioOutput.cpp @@ -0,0 +1,244 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* + * Copyright (c) 2014 The Linux Foundation. All rights reserved. + * Copyright (C) 2008 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "AudioOutput.h" + +namespace mozilla { + +#ifdef PR_LOGGING +extern PRLogModuleInfo* gAudioOffloadPlayerLog; +#define AUDIO_OFFLOAD_LOG(type, msg) \ + PR_LOG(gAudioOffloadPlayerLog, type, msg) +#else +#define AUDIO_OFFLOAD_LOG(type, msg) +#endif + +using namespace android; + +AudioOutput::AudioOutput(int aSessionId, int aUid) : + mCallback(nullptr), + mCallbackCookie(nullptr), + mCallbackData(nullptr), + mSessionId(aSessionId), + mUid(aUid) +{ +#ifdef PR_LOGGING + if (!gAudioOffloadPlayerLog) { + gAudioOffloadPlayerLog = PR_NewLogModule("AudioOffloadPlayer"); + } +#endif +} + +AudioOutput::~AudioOutput() +{ + Close(); +} + +ssize_t AudioOutput::FrameSize() const +{ + if (!mTrack.get()) { + return NO_INIT; + } + return mTrack->frameSize(); +} + +status_t AudioOutput::GetPosition(uint32_t *aPosition) const +{ + if (!mTrack.get()) { + return NO_INIT; + } + return mTrack->getPosition(aPosition); +} + +status_t AudioOutput::SetVolume(float aVolume) const +{ + if (!mTrack.get()) { + return NO_INIT; + } + return mTrack->setVolume(aVolume); +} + +status_t AudioOutput::SetParameters(const String8& aKeyValuePairs) +{ + if (!mTrack.get()) { + return NO_INIT; + } + return mTrack->setParameters(aKeyValuePairs); +} + +status_t AudioOutput::Open(uint32_t aSampleRate, + int aChannelCount, + audio_channel_mask_t aChannelMask, + audio_format_t aFormat, + AudioCallback aCb, + void* aCookie, + audio_output_flags_t aFlags, + const audio_offload_info_t *aOffloadInfo) +{ + mCallback = aCb; + mCallbackCookie = aCookie; + + if (((aFlags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) == 0) || !aCb || + !aOffloadInfo) { + return BAD_VALUE; + } + + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("open(%u, %d, 0x%x, 0x%x, %d 0x%x)", + aSampleRate, aChannelCount, aChannelMask, aFormat, mSessionId, aFlags)); + + if (aChannelMask == CHANNEL_MASK_USE_CHANNEL_ORDER) { + aChannelMask = audio_channel_out_mask_from_count(aChannelCount); + if (0 == aChannelMask) { + AUDIO_OFFLOAD_LOG(PR_LOG_ERROR, ("open() error, can\'t derive mask for" + " %d audio channels", aChannelCount)); + return NO_INIT; + } + } + + sp t; + CallbackData* newcbd = new CallbackData(this); + + t = new AudioTrack( + AUDIO_STREAM_MUSIC, + aSampleRate, + aFormat, + aChannelMask, + 0, // Offloaded tracks will get frame count from AudioFlinger + aFlags, + CallbackWrapper, + newcbd, + 0, // notification frames + mSessionId, + AudioTrack::TRANSFER_CALLBACK, + aOffloadInfo, + mUid); + + if ((!t.get()) || (t->initCheck() != NO_ERROR)) { + AUDIO_OFFLOAD_LOG(PR_LOG_ERROR, ("Unable to create audio track")); + delete newcbd; + return NO_INIT; + } + + mCallbackData = newcbd; + t->setVolume(1.0); + + mTrack = t; + return NO_ERROR; +} + +status_t AudioOutput::Start() +{ + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("%s", __PRETTY_FUNCTION__)); + if (!mTrack.get()) { + return NO_INIT; + } + mTrack->setVolume(1.0); + return mTrack->start(); +} + +void AudioOutput::Stop() +{ + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("%s", __PRETTY_FUNCTION__)); + if (mTrack.get()) { + mTrack->stop(); + } +} + +void AudioOutput::Flush() +{ + if (mTrack.get()) { + mTrack->flush(); + } +} + +void AudioOutput::Pause() +{ + if (mTrack.get()) { + mTrack->pause(); + } +} + +void AudioOutput::Close() +{ + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("%s", __PRETTY_FUNCTION__)); + mTrack.clear(); + + if (mCallbackData) { + delete mCallbackData; + mCallbackData = NULL; + } +} + +// static +void AudioOutput::CallbackWrapper(int aEvent, void* aCookie, void* aInfo) +{ + CallbackData* data = (CallbackData*) aCookie; + data->Lock(); + AudioOutput* me = data->GetOutput(); + AudioTrack::Buffer* buffer = (AudioTrack::Buffer*) aInfo; + if (!me) { + // no output set, likely because the track was scheduled to be reused + // by another player, but the format turned out to be incompatible. + data->Unlock(); + if (buffer) { + buffer->size = 0; + } + return; + } + + switch(aEvent) { + + case AudioTrack::EVENT_MORE_DATA: { + + size_t actualSize = (*me->mCallback)(me, buffer->raw, buffer->size, + me->mCallbackCookie, CB_EVENT_FILL_BUFFER); + + if (actualSize == 0 && buffer->size > 0) { + // We've reached EOS but the audio track is not stopped yet, + // keep playing silence. + memset(buffer->raw, 0, buffer->size); + actualSize = buffer->size; + } + + buffer->size = actualSize; + } break; + + case AudioTrack::EVENT_STREAM_END: + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Callback wrapper: EVENT_STREAM_END")); + (*me->mCallback)(me, nullptr /* buffer */, 0 /* size */, + me->mCallbackCookie, CB_EVENT_STREAM_END); + break; + + case AudioTrack::EVENT_NEW_IAUDIOTRACK : + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("Callback wrapper: EVENT_TEAR_DOWN")); + (*me->mCallback)(me, nullptr /* buffer */, 0 /* size */, + me->mCallbackCookie, CB_EVENT_TEAR_DOWN); + break; + + default: + AUDIO_OFFLOAD_LOG(PR_LOG_DEBUG, ("received unknown event type: %d in" + " Callback wrapper!", aEvent)); + break; + } + + data->Unlock(); +} + +} // namespace mozilla diff --git a/content/media/omx/AudioOutput.h b/content/media/omx/AudioOutput.h new file mode 100644 index 000000000000..5b837aed83d2 --- /dev/null +++ b/content/media/omx/AudioOutput.h @@ -0,0 +1,114 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* + * Copyright (c) 2014 The Linux Foundation. All rights reserved. + * Copyright (C) 2008 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AUDIOOUTPUT_H_ +#define AUDIOOUTPUT_H_ + +#include +#include +#include + +#include "AudioSink.h" + +#define LOG_TAG "AudioOffloadPlayer" + +namespace mozilla { + +/** + * Stripped version of Android KK MediaPlayerService::AudioOutput class + * Android MediaPlayer uses AudioOutput as a wrapper to handle + * Android::AudioTrack + * Similarly to ease handling offloaded tracks, part of AudioOutput is used here + */ +class AudioOutput : public AudioSink +{ + typedef android::Mutex Mutex; + typedef android::String8 String8; + typedef android::status_t status_t; + typedef android::AudioTrack AudioTrack; + + class CallbackData; + +public: + AudioOutput(int aSessionId, int aUid); + virtual ~AudioOutput(); + + virtual ssize_t FrameSize() const; + virtual status_t GetPosition(uint32_t* aPosition) const; + virtual status_t SetVolume(float aVolume) const; + virtual status_t SetParameters(const String8& aKeyValuePairs); + + // Creates an offloaded audio track with the given parameters + // TODO: Try to recycle audio tracks instead of creating new audio tracks + // every time + virtual status_t Open(uint32_t aSampleRate, + int aChannelCount, + audio_channel_mask_t aChannelMask, + audio_format_t aFormat, + AudioCallback aCb, + void* aCookie, + audio_output_flags_t aFlags = AUDIO_OUTPUT_FLAG_NONE, + const audio_offload_info_t* aOffloadInfo = nullptr); + + virtual status_t Start(); + virtual void Stop(); + virtual void Flush(); + virtual void Pause(); + virtual void Close(); + +private: + static void CallbackWrapper(int aEvent, void* aMe, void* aInfo); + + android::sp mTrack; + void* mCallbackCookie; + AudioCallback mCallback; + CallbackData* mCallbackData; + + // Uid of the current process, need to create audio track + int mUid; + + // Session id given by Android::AudioSystem and used to create audio track + int mSessionId; + + // CallbackData is what is passed to the AudioTrack as the "user" data. + // We need to be able to target this to a different Output on the fly, + // so we can't use the Output itself for this. + class CallbackData + { + public: + CallbackData(AudioOutput* aCookie) + { + mData = aCookie; + } + AudioOutput* GetOutput() { return mData;} + void SetOutput(AudioOutput* aNewcookie) { mData = aNewcookie; } + // Lock/Unlock are used by the callback before accessing the payload of + // this object + void Lock() { mLock.lock(); } + void Unlock() { mLock.unlock(); } + private: + AudioOutput* mData; + mutable Mutex mLock; + DISALLOW_EVIL_CONSTRUCTORS(CallbackData); + }; +}; // AudioOutput + +} // namespace mozilla + +#endif /* AUDIOOUTPUT_H_ */ diff --git a/content/media/omx/AudioSink.h b/content/media/omx/AudioSink.h new file mode 100644 index 000000000000..69e720a4e721 --- /dev/null +++ b/content/media/omx/AudioSink.h @@ -0,0 +1,89 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* + * Copyright (c) 2014 The Linux Foundation. All rights reserved. + * Copyright (C) 2007 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AUDIO_SINK_H_ +#define AUDIO_SINK_H_ + +#include +#include +#include + +#define DEFAULT_AUDIOSINK_BUFFERCOUNT 4 +#define DEFAULT_AUDIOSINK_BUFFERSIZE 1200 +#define DEFAULT_AUDIOSINK_SAMPLERATE 44100 + +// when the channel mask isn't known, use the channel count to derive a mask in +// AudioSink::open() +#define CHANNEL_MASK_USE_CHANNEL_ORDER 0 + +namespace mozilla { + +/** + * AudioSink: abstraction layer for audio output + * Stripped version of Android KK MediaPlayerBase::AudioSink class + */ + +class AudioSink : public android::RefBase +{ + typedef android::String8 String8; + typedef android::status_t status_t; + +public: + enum cb_event_t { + CB_EVENT_FILL_BUFFER, // Request to write more data to buffer. + CB_EVENT_STREAM_END, // Sent after all the buffers queued in AF and HW + // are played back (after stop is called) + CB_EVENT_TEAR_DOWN // The AudioTrack was invalidated due to usecase + // change. Need to re-evaluate offloading options + }; + + // Callback returns the number of bytes actually written to the buffer. + typedef size_t (*AudioCallback)(AudioSink* aAudioSink, + void* aBuffer, + size_t aSize, + void* aCookie, + cb_event_t aEvent); + virtual ~AudioSink() {} + virtual ssize_t FrameSize() const = 0; + virtual status_t GetPosition(uint32_t* aPosition) const = 0; + virtual status_t SetVolume(float aVolume) const = 0; + virtual status_t SetParameters(const String8& aKeyValuePairs) + { + return android::NO_ERROR; + } + + virtual status_t Open(uint32_t aSampleRate, + int aChannelCount, + audio_channel_mask_t aChannelMask, + audio_format_t aFormat=AUDIO_FORMAT_PCM_16_BIT, + AudioCallback aCb = nullptr, + void* aCookie = nullptr, + audio_output_flags_t aFlags = AUDIO_OUTPUT_FLAG_NONE, + const audio_offload_info_t* aOffloadInfo = nullptr) = 0; + + virtual status_t Start() = 0; + virtual void Stop() = 0; + virtual void Flush() = 0; + virtual void Pause() = 0; + virtual void Close() = 0; +}; + +} // namespace mozilla + +#endif // AUDIO_SINK_H_