2015-10-19 09:32:16 +00:00
|
|
|
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
|
|
|
|
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
|
|
|
|
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
|
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
|
|
|
2015-12-18 07:12:45 +00:00
|
|
|
#include "MediaQueue.h"
|
2015-10-19 09:32:16 +00:00
|
|
|
#include "VideoSink.h"
|
2016-08-19 09:34:42 +00:00
|
|
|
#include "MediaPrefs.h"
|
2015-10-19 09:32:16 +00:00
|
|
|
|
|
|
|
namespace mozilla {
|
|
|
|
|
2015-11-15 13:49:01 +00:00
|
|
|
extern LazyLogModule gMediaDecoderLog;
|
2016-08-23 23:45:58 +00:00
|
|
|
|
|
|
|
#undef FMT
|
|
|
|
#undef DUMP_LOG
|
|
|
|
|
|
|
|
#define FMT(x, ...) "VideoSink=%p " x, this, ##__VA_ARGS__
|
2016-10-11 05:28:56 +00:00
|
|
|
#define VSINK_LOG(x, ...) MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (FMT(x, ##__VA_ARGS__)))
|
|
|
|
#define VSINK_LOG_V(x, ...) MOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, (FMT(x, ##__VA_ARGS__)))
|
|
|
|
#define DUMP_LOG(x, ...) NS_DebugBreak(NS_DEBUG_WARNING, nsPrintfCString(FMT(x, ##__VA_ARGS__)).get(), nullptr, nullptr, -1)
|
2015-10-19 09:32:16 +00:00
|
|
|
|
|
|
|
using namespace mozilla::layers;
|
|
|
|
|
|
|
|
namespace media {
|
|
|
|
|
2016-11-07 03:21:58 +00:00
|
|
|
// Minimum update frequency is 1/120th of a second, i.e. half the
|
|
|
|
// duration of a 60-fps frame.
|
|
|
|
static const int64_t MIN_UPDATE_INTERVAL_US = 1000000 / (60 * 2);
|
|
|
|
|
2015-10-19 10:08:11 +00:00
|
|
|
VideoSink::VideoSink(AbstractThread* aThread,
|
|
|
|
MediaSink* aAudioSink,
|
|
|
|
MediaQueue<MediaData>& aVideoQueue,
|
|
|
|
VideoFrameContainer* aContainer,
|
|
|
|
FrameStatistics& aFrameStats,
|
|
|
|
uint32_t aVQueueSentToCompositerSize)
|
|
|
|
: mOwnerThread(aThread)
|
|
|
|
, mAudioSink(aAudioSink)
|
|
|
|
, mVideoQueue(aVideoQueue)
|
|
|
|
, mContainer(aContainer)
|
|
|
|
, mProducerID(ImageContainer::AllocateProducerID())
|
|
|
|
, mFrameStats(aFrameStats)
|
|
|
|
, mVideoFrameEndTime(-1)
|
|
|
|
, mHasVideo(false)
|
|
|
|
, mUpdateScheduler(aThread)
|
|
|
|
, mVideoQueueSendToCompositorSize(aVQueueSentToCompositerSize)
|
2016-08-19 09:34:42 +00:00
|
|
|
, mMinVideoQueueSize(MediaPrefs::RuinAvSync() ? 1 : 0)
|
2015-10-19 10:08:11 +00:00
|
|
|
{
|
|
|
|
MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
|
|
|
|
}
|
|
|
|
|
2015-10-19 09:32:16 +00:00
|
|
|
VideoSink::~VideoSink()
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
const MediaSink::PlaybackParams&
|
|
|
|
VideoSink::GetPlaybackParams() const
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
2015-10-19 10:08:11 +00:00
|
|
|
|
2015-10-19 09:32:16 +00:00
|
|
|
return mAudioSink->GetPlaybackParams();
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
VideoSink::SetPlaybackParams(const PlaybackParams& aParams)
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
2015-10-19 10:08:11 +00:00
|
|
|
|
2015-10-19 09:32:16 +00:00
|
|
|
mAudioSink->SetPlaybackParams(aParams);
|
|
|
|
}
|
|
|
|
|
|
|
|
RefPtr<GenericPromise>
|
|
|
|
VideoSink::OnEnded(TrackType aType)
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
|
|
|
|
|
|
|
|
if (aType == TrackInfo::kAudioTrack) {
|
|
|
|
return mAudioSink->OnEnded(aType);
|
|
|
|
} else if (aType == TrackInfo::kVideoTrack) {
|
|
|
|
return mEndPromise;
|
|
|
|
}
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
int64_t
|
|
|
|
VideoSink::GetEndTime(TrackType aType) const
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
|
|
|
|
|
|
|
|
if (aType == TrackInfo::kVideoTrack) {
|
|
|
|
return mVideoFrameEndTime;
|
|
|
|
} else if (aType == TrackInfo::kAudioTrack) {
|
|
|
|
return mAudioSink->GetEndTime(aType);
|
|
|
|
}
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
int64_t
|
|
|
|
VideoSink::GetPosition(TimeStamp* aTimeStamp) const
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
|
|
|
|
return mAudioSink->GetPosition(aTimeStamp);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
VideoSink::HasUnplayedFrames(TrackType aType) const
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
MOZ_ASSERT(aType == TrackInfo::kAudioTrack, "Not implemented for non audio tracks.");
|
|
|
|
|
|
|
|
return mAudioSink->HasUnplayedFrames(aType);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
VideoSink::SetPlaybackRate(double aPlaybackRate)
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
|
|
|
|
mAudioSink->SetPlaybackRate(aPlaybackRate);
|
|
|
|
}
|
|
|
|
|
2015-10-19 10:14:31 +00:00
|
|
|
void
|
|
|
|
VideoSink::SetVolume(double aVolume)
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
|
|
|
|
mAudioSink->SetVolume(aVolume);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
VideoSink::SetPreservesPitch(bool aPreservesPitch)
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
|
|
|
|
mAudioSink->SetPreservesPitch(aPreservesPitch);
|
|
|
|
}
|
|
|
|
|
2015-10-19 09:32:16 +00:00
|
|
|
void
|
|
|
|
VideoSink::SetPlaying(bool aPlaying)
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
VSINK_LOG_V(" playing (%d) -> (%d)", mAudioSink->IsPlaying(), aPlaying);
|
|
|
|
|
2015-10-19 10:08:11 +00:00
|
|
|
if (!aPlaying) {
|
|
|
|
// Reset any update timer if paused.
|
|
|
|
mUpdateScheduler.Reset();
|
|
|
|
// Since playback is paused, tell compositor to render only current frame.
|
|
|
|
RenderVideoFrames(1);
|
2016-09-05 06:38:43 +00:00
|
|
|
if (mContainer) {
|
|
|
|
mContainer->ClearCachedResources();
|
|
|
|
}
|
2015-10-19 10:08:11 +00:00
|
|
|
}
|
|
|
|
|
2015-10-19 09:32:16 +00:00
|
|
|
mAudioSink->SetPlaying(aPlaying);
|
2015-10-19 10:08:11 +00:00
|
|
|
|
|
|
|
if (mHasVideo && aPlaying) {
|
|
|
|
// There's no thread in VideoSink for pulling video frames, need to trigger
|
|
|
|
// rendering while becoming playing status. because the VideoQueue may be
|
|
|
|
// full already.
|
|
|
|
TryUpdateRenderedVideoFrames();
|
|
|
|
}
|
2015-10-19 09:32:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
VideoSink::Start(int64_t aStartTime, const MediaInfo& aInfo)
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
VSINK_LOG("[%s]", __func__);
|
2015-10-19 10:08:11 +00:00
|
|
|
|
2015-10-19 09:32:16 +00:00
|
|
|
mAudioSink->Start(aStartTime, aInfo);
|
|
|
|
|
2015-10-19 10:08:11 +00:00
|
|
|
mHasVideo = aInfo.HasVideo();
|
|
|
|
|
|
|
|
if (mHasVideo) {
|
2015-10-19 09:32:16 +00:00
|
|
|
mEndPromise = mEndPromiseHolder.Ensure(__func__);
|
2015-12-22 00:42:38 +00:00
|
|
|
|
|
|
|
// If the underlying MediaSink has an end promise for the video track (which
|
|
|
|
// happens when mAudioSink refers to a DecodedStream), we must wait for it
|
|
|
|
// to complete before resolving our own end promise. Otherwise, MDSM might
|
|
|
|
// stop playback before DecodedStream plays to the end and cause
|
|
|
|
// test_streams_element_capture.html to time out.
|
|
|
|
RefPtr<GenericPromise> p = mAudioSink->OnEnded(TrackInfo::kVideoTrack);
|
|
|
|
if (p) {
|
|
|
|
RefPtr<VideoSink> self = this;
|
|
|
|
mVideoSinkEndRequest.Begin(p->Then(mOwnerThread, __func__,
|
|
|
|
[self] () {
|
|
|
|
self->mVideoSinkEndRequest.Complete();
|
|
|
|
self->TryUpdateRenderedVideoFrames();
|
2016-08-29 13:28:36 +00:00
|
|
|
// It is possible the video queue size is 0 and we have no frames to
|
|
|
|
// render. However, we need to call MaybeResolveEndPromise() to ensure
|
|
|
|
// mEndPromiseHolder is resolved.
|
|
|
|
self->MaybeResolveEndPromise();
|
2015-12-22 00:42:38 +00:00
|
|
|
}, [self] () {
|
|
|
|
self->mVideoSinkEndRequest.Complete();
|
|
|
|
self->TryUpdateRenderedVideoFrames();
|
2016-08-29 13:28:36 +00:00
|
|
|
self->MaybeResolveEndPromise();
|
2015-12-22 00:42:38 +00:00
|
|
|
}));
|
|
|
|
}
|
|
|
|
|
2015-10-19 10:08:11 +00:00
|
|
|
ConnectListener();
|
2015-12-22 00:42:38 +00:00
|
|
|
// Run the render loop at least once so we can resolve the end promise
|
|
|
|
// when video duration is 0.
|
|
|
|
UpdateRenderedVideoFrames();
|
2015-10-19 09:32:16 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
VideoSink::Stop()
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
MOZ_ASSERT(mAudioSink->IsStarted(), "playback not started.");
|
|
|
|
VSINK_LOG("[%s]", __func__);
|
|
|
|
|
|
|
|
mAudioSink->Stop();
|
|
|
|
|
2015-10-19 10:08:11 +00:00
|
|
|
mUpdateScheduler.Reset();
|
|
|
|
if (mHasVideo) {
|
|
|
|
DisconnectListener();
|
2015-12-22 00:42:38 +00:00
|
|
|
mVideoSinkEndRequest.DisconnectIfExists();
|
2015-12-22 00:42:38 +00:00
|
|
|
mEndPromiseHolder.ResolveIfExists(true, __func__);
|
2015-10-19 10:08:11 +00:00
|
|
|
mEndPromise = nullptr;
|
|
|
|
}
|
|
|
|
mVideoFrameEndTime = -1;
|
2015-10-19 09:32:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
VideoSink::IsStarted() const
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
|
|
|
|
return mAudioSink->IsStarted();
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
VideoSink::IsPlaying() const
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
|
|
|
|
return mAudioSink->IsPlaying();
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
VideoSink::Shutdown()
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
MOZ_ASSERT(!mAudioSink->IsStarted(), "must be called after playback stops.");
|
|
|
|
VSINK_LOG("[%s]", __func__);
|
|
|
|
|
|
|
|
mAudioSink->Shutdown();
|
|
|
|
}
|
|
|
|
|
2015-10-19 10:08:11 +00:00
|
|
|
void
|
2015-12-22 00:42:38 +00:00
|
|
|
VideoSink::OnVideoQueuePushed(RefPtr<MediaData>&& aSample)
|
2015-10-19 10:08:11 +00:00
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
// Listen to push event, VideoSink should try rendering ASAP if first frame
|
|
|
|
// arrives but update scheduler is not triggered yet.
|
2015-11-25 07:15:05 +00:00
|
|
|
VideoData* v = aSample->As<VideoData>();
|
2016-11-23 07:08:17 +00:00
|
|
|
if (!v->IsSentToCompositor()) {
|
2015-11-25 07:15:05 +00:00
|
|
|
// Since we push rendered frames back to the queue, we will receive
|
|
|
|
// push events for them. We only need to trigger render loop
|
|
|
|
// when this frame is not rendered yet.
|
|
|
|
TryUpdateRenderedVideoFrames();
|
|
|
|
}
|
2015-10-19 10:08:11 +00:00
|
|
|
}
|
|
|
|
|
2015-12-22 00:42:38 +00:00
|
|
|
void
|
|
|
|
VideoSink::OnVideoQueueFinished()
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
// Run render loop if the end promise is not resolved yet.
|
|
|
|
if (!mUpdateScheduler.IsScheduled() &&
|
|
|
|
mAudioSink->IsPlaying() &&
|
|
|
|
!mEndPromiseHolder.IsEmpty()) {
|
|
|
|
UpdateRenderedVideoFrames();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-10-19 10:08:11 +00:00
|
|
|
void
|
2016-05-17 02:33:32 +00:00
|
|
|
VideoSink::Redraw(const VideoInfo& aInfo)
|
2015-10-19 10:08:11 +00:00
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
2016-05-17 02:33:32 +00:00
|
|
|
|
|
|
|
// No video track, nothing to draw.
|
|
|
|
if (!aInfo.IsValid() || !mContainer) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2016-11-15 04:04:26 +00:00
|
|
|
RefPtr<MediaData> frame = VideoQueue().PeekFront();
|
|
|
|
if (frame) {
|
|
|
|
VideoData* video = frame->As<VideoData>();
|
2016-12-09 11:57:12 +00:00
|
|
|
video->MarkSentToCompositor();
|
2016-11-15 04:04:26 +00:00
|
|
|
mContainer->SetCurrentFrame(video->mDisplay, video->mImage, TimeStamp::Now());
|
2016-05-17 02:33:32 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// When we reach here, it means there are no frames in this video track.
|
|
|
|
// Draw a blank frame to ensure there is something in the image container
|
|
|
|
// to fire 'loadeddata'.
|
|
|
|
RefPtr<Image> blank =
|
|
|
|
mContainer->GetImageContainer()->CreatePlanarYCbCrImage();
|
|
|
|
mContainer->SetCurrentFrame(aInfo.mDisplay, blank, TimeStamp::Now());
|
2015-10-19 10:08:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
VideoSink::TryUpdateRenderedVideoFrames()
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
if (!mUpdateScheduler.IsScheduled() && VideoQueue().GetSize() >= 1 &&
|
|
|
|
mAudioSink->IsPlaying()) {
|
|
|
|
UpdateRenderedVideoFrames();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
VideoSink::UpdateRenderedVideoFramesByTimer()
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
mUpdateScheduler.CompleteRequest();
|
|
|
|
UpdateRenderedVideoFrames();
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
VideoSink::ConnectListener()
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
mPushListener = VideoQueue().PushEvent().Connect(
|
2015-12-22 00:42:38 +00:00
|
|
|
mOwnerThread, this, &VideoSink::OnVideoQueuePushed);
|
|
|
|
mFinishListener = VideoQueue().FinishEvent().Connect(
|
|
|
|
mOwnerThread, this, &VideoSink::OnVideoQueueFinished);
|
2015-10-19 10:08:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
VideoSink::DisconnectListener()
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
mPushListener.Disconnect();
|
2015-12-22 00:42:38 +00:00
|
|
|
mFinishListener.Disconnect();
|
2015-10-19 10:08:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
VideoSink::RenderVideoFrames(int32_t aMaxFrames,
|
|
|
|
int64_t aClockTime,
|
|
|
|
const TimeStamp& aClockTimeStamp)
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
|
2016-02-02 15:36:30 +00:00
|
|
|
AutoTArray<RefPtr<MediaData>,16> frames;
|
2015-10-19 10:08:11 +00:00
|
|
|
VideoQueue().GetFirstElements(aMaxFrames, &frames);
|
|
|
|
if (frames.IsEmpty() || !mContainer) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2016-02-02 15:36:30 +00:00
|
|
|
AutoTArray<ImageContainer::NonOwningImage,16> images;
|
2015-10-19 10:08:11 +00:00
|
|
|
TimeStamp lastFrameTime;
|
|
|
|
MediaSink::PlaybackParams params = mAudioSink->GetPlaybackParams();
|
|
|
|
for (uint32_t i = 0; i < frames.Length(); ++i) {
|
|
|
|
VideoData* frame = frames[i]->As<VideoData>();
|
|
|
|
|
2016-11-23 07:08:17 +00:00
|
|
|
frame->MarkSentToCompositor();
|
2015-10-19 10:08:11 +00:00
|
|
|
|
2016-09-19 13:41:36 +00:00
|
|
|
if (!frame->mImage || !frame->mImage->IsValid() ||
|
|
|
|
!frame->mImage->GetSize().width || !frame->mImage->GetSize().height) {
|
2015-10-19 10:08:11 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
int64_t frameTime = frame->mTime;
|
|
|
|
if (frameTime < 0) {
|
|
|
|
// Frame times before the start time are invalid; drop such frames
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
TimeStamp t;
|
|
|
|
if (aMaxFrames > 1) {
|
|
|
|
MOZ_ASSERT(!aClockTimeStamp.IsNull());
|
|
|
|
int64_t delta = frame->mTime - aClockTime;
|
|
|
|
t = aClockTimeStamp +
|
|
|
|
TimeDuration::FromMicroseconds(delta / params.mPlaybackRate);
|
|
|
|
if (!lastFrameTime.IsNull() && t <= lastFrameTime) {
|
|
|
|
// Timestamps out of order; drop the new frame. In theory we should
|
|
|
|
// probably replace the previous frame with the new frame if the
|
|
|
|
// timestamps are equal, but this is a corrupt video file already so
|
|
|
|
// never mind.
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
lastFrameTime = t;
|
|
|
|
}
|
|
|
|
|
|
|
|
ImageContainer::NonOwningImage* img = images.AppendElement();
|
|
|
|
img->mTimeStamp = t;
|
|
|
|
img->mImage = frame->mImage;
|
|
|
|
img->mFrameID = frame->mFrameID;
|
|
|
|
img->mProducerID = mProducerID;
|
|
|
|
|
|
|
|
VSINK_LOG_V("playing video frame %lld (id=%x) (vq-queued=%i)",
|
|
|
|
frame->mTime, frame->mFrameID, VideoQueue().GetSize());
|
|
|
|
}
|
|
|
|
mContainer->SetCurrentFrames(frames[0]->As<VideoData>()->mDisplay, images);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
VideoSink::UpdateRenderedVideoFrames()
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
MOZ_ASSERT(mAudioSink->IsPlaying(), "should be called while playing.");
|
|
|
|
|
Bug 1258870 - Don't push late video frames to the compositor, drop them. r=jwwang
We can get out of A/V sync if the decode is struggling to keep up. This is
because of the loop in VideoSink::UpdateRenderedVideoFrames(). If this function
runs while there's only one frame in the video queue, we won't drop that one
frame if it's late. If we're struggling to keep up, it's increasingly likely
that we'll end up running this function with only one frame in the video queue.
That results in us entering VideoSink::RenderVideoFrames() with only 1 late
frame, which the compositor dutifully draws. Resulting in a late frame being
drawn, and thus broken A/V sync.
This change makes VideoSink::UpdateRenderedVideoFrames() drop all late
frames, even the last one in the video queue. We now keep A/V sync when the
decode is struggling to keep up. However, if I do this, we end up dropping
(and reporting that we drop) a lot more frames, and thus rendering a lot fewer.
But since we when we drop the frames we report them as dropped, a well written
MSE player can detect that we've failing miserably to keep up, and and lower
their bitrate.
MozReview-Commit-ID: ybkq48mKk2
--HG--
extra : rebase_source : f03c186059a365a45de698b2a30e632daae47fb8
2016-08-15 01:35:52 +00:00
|
|
|
// Get the current playback position.
|
2015-10-19 10:08:11 +00:00
|
|
|
TimeStamp nowTime;
|
|
|
|
const int64_t clockTime = mAudioSink->GetPosition(&nowTime);
|
|
|
|
NS_ASSERTION(clockTime >= 0, "Should have positive clock time.");
|
|
|
|
|
Bug 1258870 - Don't push late video frames to the compositor, drop them. r=jwwang
We can get out of A/V sync if the decode is struggling to keep up. This is
because of the loop in VideoSink::UpdateRenderedVideoFrames(). If this function
runs while there's only one frame in the video queue, we won't drop that one
frame if it's late. If we're struggling to keep up, it's increasingly likely
that we'll end up running this function with only one frame in the video queue.
That results in us entering VideoSink::RenderVideoFrames() with only 1 late
frame, which the compositor dutifully draws. Resulting in a late frame being
drawn, and thus broken A/V sync.
This change makes VideoSink::UpdateRenderedVideoFrames() drop all late
frames, even the last one in the video queue. We now keep A/V sync when the
decode is struggling to keep up. However, if I do this, we end up dropping
(and reporting that we drop) a lot more frames, and thus rendering a lot fewer.
But since we when we drop the frames we report them as dropped, a well written
MSE player can detect that we've failing miserably to keep up, and and lower
their bitrate.
MozReview-Commit-ID: ybkq48mKk2
--HG--
extra : rebase_source : f03c186059a365a45de698b2a30e632daae47fb8
2016-08-15 01:35:52 +00:00
|
|
|
// Skip frames up to the playback position.
|
2016-11-10 09:12:01 +00:00
|
|
|
int64_t lastFrameEndTime = 0;
|
2016-08-19 09:34:42 +00:00
|
|
|
while (VideoQueue().GetSize() > mMinVideoQueueSize &&
|
2016-08-29 05:46:56 +00:00
|
|
|
clockTime >= VideoQueue().PeekFront()->GetEndTime()) {
|
Bug 1258870 - Don't push late video frames to the compositor, drop them. r=jwwang
We can get out of A/V sync if the decode is struggling to keep up. This is
because of the loop in VideoSink::UpdateRenderedVideoFrames(). If this function
runs while there's only one frame in the video queue, we won't drop that one
frame if it's late. If we're struggling to keep up, it's increasingly likely
that we'll end up running this function with only one frame in the video queue.
That results in us entering VideoSink::RenderVideoFrames() with only 1 late
frame, which the compositor dutifully draws. Resulting in a late frame being
drawn, and thus broken A/V sync.
This change makes VideoSink::UpdateRenderedVideoFrames() drop all late
frames, even the last one in the video queue. We now keep A/V sync when the
decode is struggling to keep up. However, if I do this, we end up dropping
(and reporting that we drop) a lot more frames, and thus rendering a lot fewer.
But since we when we drop the frames we report them as dropped, a well written
MSE player can detect that we've failing miserably to keep up, and and lower
their bitrate.
MozReview-Commit-ID: ybkq48mKk2
--HG--
extra : rebase_source : f03c186059a365a45de698b2a30e632daae47fb8
2016-08-15 01:35:52 +00:00
|
|
|
RefPtr<MediaData> frame = VideoQueue().PopFront();
|
2016-11-10 09:12:01 +00:00
|
|
|
lastFrameEndTime = frame->GetEndTime();
|
2016-11-23 07:08:17 +00:00
|
|
|
if (frame->As<VideoData>()->IsSentToCompositor()) {
|
2015-10-19 10:08:11 +00:00
|
|
|
mFrameStats.NotifyPresentedFrame();
|
Bug 1258870 - Don't push late video frames to the compositor, drop them. r=jwwang
We can get out of A/V sync if the decode is struggling to keep up. This is
because of the loop in VideoSink::UpdateRenderedVideoFrames(). If this function
runs while there's only one frame in the video queue, we won't drop that one
frame if it's late. If we're struggling to keep up, it's increasingly likely
that we'll end up running this function with only one frame in the video queue.
That results in us entering VideoSink::RenderVideoFrames() with only 1 late
frame, which the compositor dutifully draws. Resulting in a late frame being
drawn, and thus broken A/V sync.
This change makes VideoSink::UpdateRenderedVideoFrames() drop all late
frames, even the last one in the video queue. We now keep A/V sync when the
decode is struggling to keep up. However, if I do this, we end up dropping
(and reporting that we drop) a lot more frames, and thus rendering a lot fewer.
But since we when we drop the frames we report them as dropped, a well written
MSE player can detect that we've failing miserably to keep up, and and lower
their bitrate.
MozReview-Commit-ID: ybkq48mKk2
--HG--
extra : rebase_source : f03c186059a365a45de698b2a30e632daae47fb8
2016-08-15 01:35:52 +00:00
|
|
|
} else {
|
|
|
|
mFrameStats.NotifyDecodedFrames({ 0, 0, 1 });
|
|
|
|
VSINK_LOG_V("discarding video frame mTime=%lld clock_time=%lld",
|
|
|
|
frame->mTime, clockTime);
|
2015-10-19 10:08:11 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Bug 1258870 - Don't push late video frames to the compositor, drop them. r=jwwang
We can get out of A/V sync if the decode is struggling to keep up. This is
because of the loop in VideoSink::UpdateRenderedVideoFrames(). If this function
runs while there's only one frame in the video queue, we won't drop that one
frame if it's late. If we're struggling to keep up, it's increasingly likely
that we'll end up running this function with only one frame in the video queue.
That results in us entering VideoSink::RenderVideoFrames() with only 1 late
frame, which the compositor dutifully draws. Resulting in a late frame being
drawn, and thus broken A/V sync.
This change makes VideoSink::UpdateRenderedVideoFrames() drop all late
frames, even the last one in the video queue. We now keep A/V sync when the
decode is struggling to keep up. However, if I do this, we end up dropping
(and reporting that we drop) a lot more frames, and thus rendering a lot fewer.
But since we when we drop the frames we report them as dropped, a well written
MSE player can detect that we've failing miserably to keep up, and and lower
their bitrate.
MozReview-Commit-ID: ybkq48mKk2
--HG--
extra : rebase_source : f03c186059a365a45de698b2a30e632daae47fb8
2016-08-15 01:35:52 +00:00
|
|
|
// The presentation end time of the last video frame displayed is either
|
|
|
|
// the end time of the current frame, or if we dropped all frames in the
|
|
|
|
// queue, the end time of the last frame we removed from the queue.
|
|
|
|
RefPtr<MediaData> currentFrame = VideoQueue().PeekFront();
|
2016-08-29 12:46:26 +00:00
|
|
|
mVideoFrameEndTime = std::max(mVideoFrameEndTime,
|
2016-11-10 09:12:01 +00:00
|
|
|
currentFrame ? currentFrame->GetEndTime() : lastFrameEndTime);
|
Bug 1258870 - Don't push late video frames to the compositor, drop them. r=jwwang
We can get out of A/V sync if the decode is struggling to keep up. This is
because of the loop in VideoSink::UpdateRenderedVideoFrames(). If this function
runs while there's only one frame in the video queue, we won't drop that one
frame if it's late. If we're struggling to keep up, it's increasingly likely
that we'll end up running this function with only one frame in the video queue.
That results in us entering VideoSink::RenderVideoFrames() with only 1 late
frame, which the compositor dutifully draws. Resulting in a late frame being
drawn, and thus broken A/V sync.
This change makes VideoSink::UpdateRenderedVideoFrames() drop all late
frames, even the last one in the video queue. We now keep A/V sync when the
decode is struggling to keep up. However, if I do this, we end up dropping
(and reporting that we drop) a lot more frames, and thus rendering a lot fewer.
But since we when we drop the frames we report them as dropped, a well written
MSE player can detect that we've failing miserably to keep up, and and lower
their bitrate.
MozReview-Commit-ID: ybkq48mKk2
--HG--
extra : rebase_source : f03c186059a365a45de698b2a30e632daae47fb8
2016-08-15 01:35:52 +00:00
|
|
|
|
2016-08-29 12:56:00 +00:00
|
|
|
MaybeResolveEndPromise();
|
2015-12-22 00:42:38 +00:00
|
|
|
|
2015-10-19 10:08:11 +00:00
|
|
|
RenderVideoFrames(mVideoQueueSendToCompositorSize, clockTime, nowTime);
|
|
|
|
|
Bug 1258870 - Don't push late video frames to the compositor, drop them. r=jwwang
We can get out of A/V sync if the decode is struggling to keep up. This is
because of the loop in VideoSink::UpdateRenderedVideoFrames(). If this function
runs while there's only one frame in the video queue, we won't drop that one
frame if it's late. If we're struggling to keep up, it's increasingly likely
that we'll end up running this function with only one frame in the video queue.
That results in us entering VideoSink::RenderVideoFrames() with only 1 late
frame, which the compositor dutifully draws. Resulting in a late frame being
drawn, and thus broken A/V sync.
This change makes VideoSink::UpdateRenderedVideoFrames() drop all late
frames, even the last one in the video queue. We now keep A/V sync when the
decode is struggling to keep up. However, if I do this, we end up dropping
(and reporting that we drop) a lot more frames, and thus rendering a lot fewer.
But since we when we drop the frames we report them as dropped, a well written
MSE player can detect that we've failing miserably to keep up, and and lower
their bitrate.
MozReview-Commit-ID: ybkq48mKk2
--HG--
extra : rebase_source : f03c186059a365a45de698b2a30e632daae47fb8
2016-08-15 01:35:52 +00:00
|
|
|
// Get the timestamp of the next frame. Schedule the next update at
|
|
|
|
// the start time of the next frame. If we don't have a next frame,
|
|
|
|
// we will run render loops again upon incoming frames.
|
|
|
|
nsTArray<RefPtr<MediaData>> frames;
|
|
|
|
VideoQueue().GetFirstElements(2, &frames);
|
|
|
|
if (frames.Length() < 2) {
|
2015-11-25 07:15:05 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
Bug 1258870 - Don't push late video frames to the compositor, drop them. r=jwwang
We can get out of A/V sync if the decode is struggling to keep up. This is
because of the loop in VideoSink::UpdateRenderedVideoFrames(). If this function
runs while there's only one frame in the video queue, we won't drop that one
frame if it's late. If we're struggling to keep up, it's increasingly likely
that we'll end up running this function with only one frame in the video queue.
That results in us entering VideoSink::RenderVideoFrames() with only 1 late
frame, which the compositor dutifully draws. Resulting in a late frame being
drawn, and thus broken A/V sync.
This change makes VideoSink::UpdateRenderedVideoFrames() drop all late
frames, even the last one in the video queue. We now keep A/V sync when the
decode is struggling to keep up. However, if I do this, we end up dropping
(and reporting that we drop) a lot more frames, and thus rendering a lot fewer.
But since we when we drop the frames we report them as dropped, a well written
MSE player can detect that we've failing miserably to keep up, and and lower
their bitrate.
MozReview-Commit-ID: ybkq48mKk2
--HG--
extra : rebase_source : f03c186059a365a45de698b2a30e632daae47fb8
2016-08-15 01:35:52 +00:00
|
|
|
int64_t nextFrameTime = frames[1]->mTime;
|
2016-11-07 03:21:58 +00:00
|
|
|
int64_t delta = std::max<int64_t>((nextFrameTime - clockTime), MIN_UPDATE_INTERVAL_US);
|
2015-11-25 07:15:05 +00:00
|
|
|
TimeStamp target = nowTime + TimeDuration::FromMicroseconds(
|
2016-11-07 03:21:58 +00:00
|
|
|
delta / mAudioSink->GetPlaybackParams().mPlaybackRate);
|
2015-10-19 10:08:11 +00:00
|
|
|
|
|
|
|
RefPtr<VideoSink> self = this;
|
|
|
|
mUpdateScheduler.Ensure(target, [self] () {
|
|
|
|
self->UpdateRenderedVideoFramesByTimer();
|
|
|
|
}, [self] () {
|
|
|
|
self->UpdateRenderedVideoFramesByTimer();
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2016-08-29 12:56:00 +00:00
|
|
|
void
|
|
|
|
VideoSink::MaybeResolveEndPromise()
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
// All frames are rendered, Let's resolve the promise.
|
|
|
|
if (VideoQueue().IsFinished() &&
|
|
|
|
VideoQueue().GetSize() <= 1 &&
|
|
|
|
!mVideoSinkEndRequest.Exists()) {
|
|
|
|
mEndPromiseHolder.ResolveIfExists(true, __func__);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-23 23:45:58 +00:00
|
|
|
void
|
|
|
|
VideoSink::DumpDebugInfo()
|
|
|
|
{
|
|
|
|
AssertOwnerThread();
|
|
|
|
DUMP_LOG(
|
|
|
|
"IsStarted=%d IsPlaying=%d, VideoQueue: finished=%d size=%d, "
|
|
|
|
"mVideoFrameEndTime=%lld mHasVideo=%d mVideoSinkEndRequest.Exists()=%d "
|
|
|
|
"mEndPromiseHolder.IsEmpty()=%d",
|
|
|
|
IsStarted(), IsPlaying(), VideoQueue().IsFinished(), VideoQueue().GetSize(),
|
|
|
|
mVideoFrameEndTime, mHasVideo, mVideoSinkEndRequest.Exists(), mEndPromiseHolder.IsEmpty());
|
2016-11-01 06:41:09 +00:00
|
|
|
mAudioSink->DumpDebugInfo();
|
2016-08-23 23:45:58 +00:00
|
|
|
}
|
|
|
|
|
2015-10-19 09:32:16 +00:00
|
|
|
} // namespace media
|
|
|
|
} // namespace mozilla
|