gecko-dev/dom/media/DriftCompensation.h

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

#ifndef DriftCompensation_h_
#define DriftCompensation_h_

#include "MediaSegment.h"
#include "VideoUtils.h"
#include "mozilla/Atomics.h"
#include "mozilla/Unused.h"

namespace mozilla {

static LazyLogModule gDriftCompensatorLog("DriftCompensator");
#define LOG(type, ...) MOZ_LOG(gDriftCompensatorLog, type, (__VA_ARGS__))

/**
 * DriftCompensator can be used to handle drift between audio and video tracks
 * from the MediaTrackGraph.
 *
 * Drift can occur because audio is driven by a MediaTrackGraph running off an
 * audio callback, thus it's progressed by the clock of one the audio output
 * devices on the user's machine. Video on the other hand is always expressed in
 * wall-clock TimeStamps, i.e., it's progressed by the system clock. These
 * clocks will, over time, drift apart.
 *
 * Do not use the DriftCompensator across multiple audio tracks, as it will
 * automatically record the start time of the first audio samples, and all
 * samples for the same audio track on the same audio clock will have to be
 * processed to retain accuracy.
 *
 * DriftCompensator is designed to be used from two threads:
 * - The audio thread for notifications of audio samples.
 * - The video thread for compensating drift of video frames to match the audio
 *   clock.
 */
class DriftCompensator {
  const RefPtr<nsIEventTarget> mVideoThread;
  const TrackRate mAudioRate;

  // Number of audio samples produced. Any thread.
  Atomic<TrackTime> mAudioSamples{0};

  // Time the first audio samples were added. mVideoThread only.
  TimeStamp mAudioStartTime;

  void SetAudioStartTime(TimeStamp aTime) {
    MOZ_ASSERT(mVideoThread->IsOnCurrentThread());
    MOZ_ASSERT(mAudioStartTime.IsNull());
    mAudioStartTime = aTime;
  }

 protected:
  virtual ~DriftCompensator() = default;

 public:
  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DriftCompensator)

  DriftCompensator(RefPtr<nsIEventTarget> aVideoThread, TrackRate aAudioRate)
      : mVideoThread(std::move(aVideoThread)), mAudioRate(aAudioRate) {
    MOZ_ASSERT(mAudioRate > 0);
  }

  void NotifyAudioStart(TimeStamp aStart) {
    MOZ_ASSERT(mAudioSamples == 0);
    LOG(LogLevel::Info, "DriftCompensator %p at rate %d started", this,
        mAudioRate);
    nsresult rv = mVideoThread->Dispatch(NewRunnableMethod<TimeStamp>(
        "DriftCompensator::SetAudioStartTime", this,
        &DriftCompensator::SetAudioStartTime, aStart));
    MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
    Unused << rv;
  }

  /**
   * aSamples is the number of samples fed by an AudioStream.
   */
  void NotifyAudio(TrackTime aSamples) {
    MOZ_ASSERT(aSamples > 0);
    mAudioSamples += aSamples;

    LOG(LogLevel::Verbose,
        "DriftCompensator %p Processed another %" PRId64
        " samples; now %.3fs audio",
        this, aSamples, static_cast<double>(mAudioSamples) / mAudioRate);
  }

  /**
   * Drift compensates a video TimeStamp based on historical audio data.
   */
  virtual TimeStamp GetVideoTime(TimeStamp aNow, TimeStamp aTime) {
    MOZ_ASSERT(mVideoThread->IsOnCurrentThread());
    TrackTime samples = mAudioSamples;

    if (samples / mAudioRate < 10) {
      // We don't apply compensation for the first 10 seconds because of the
      // higher inaccuracy during this time.
      LOG(LogLevel::Debug, "DriftCompensator %p %" PRId64 "ms so far; ignoring",
          this, samples * 1000 / mAudioRate);
      return aTime;
    }

    if (aNow == mAudioStartTime) {
      LOG(LogLevel::Warning,
          "DriftCompensator %p video scale 0, assuming no drift", this);
      return aTime;
    }

    double videoScaleUs = (aNow - mAudioStartTime).ToMicroseconds();
    double audioScaleUs = FramesToUsecs(samples, mAudioRate).value();
    double videoDurationUs = (aTime - mAudioStartTime).ToMicroseconds();

    TimeStamp reclocked =
        mAudioStartTime + TimeDuration::FromMicroseconds(
                              videoDurationUs * audioScaleUs / videoScaleUs);

    LOG(LogLevel::Debug,
        "DriftCompensator %p GetVideoTime, v-now: %.3fs, a-now: %.3fs; %.3fs "
        "-> %.3fs (d %.3fms)",
        this, (aNow - mAudioStartTime).ToSeconds(),
        TimeDuration::FromMicroseconds(audioScaleUs).ToSeconds(),
        (aTime - mAudioStartTime).ToSeconds(),
        (reclocked - mAudioStartTime).ToSeconds(),
        (reclocked - aTime).ToMilliseconds());

    return reclocked;
  }
};

#undef LOG

}  // namespace mozilla

#endif /* DriftCompensation_h_ */