diff --git a/dom/media/MediaData.cpp b/dom/media/MediaData.cpp
index 4213911fd072..260f869a2524 100644
--- a/dom/media/MediaData.cpp
+++ b/dom/media/MediaData.cpp
@@ -25,9 +25,6 @@ using layers::ImageContainer;
 using layers::PlanarYCbCrImage;
 using layers::PlanarYCbCrData;
 
-const char* AudioData::sTypeName = "audio";
-const char* VideoData::sTypeName = "video";
-
 void
 AudioData::EnsureAudioBuffer()
 {
@@ -112,7 +109,7 @@ VideoData::VideoData(int64_t aOffset,
                      int64_t aTime,
                      int64_t aDuration,
                      int64_t aTimecode)
-  : MediaData(sType, aOffset, aTime, aDuration)
+  : MediaData(VIDEO_DATA, aOffset, aTime, aDuration)
   , mDuplicate(true)
 {
   NS_ASSERTION(mDuration >= 0, "Frame must have non-negative duration.");
@@ -125,7 +122,7 @@ VideoData::VideoData(int64_t aOffset,
                      bool aKeyframe,
                      int64_t aTimecode,
                      IntSize aDisplay)
-  : MediaData(sType, aOffset, aTime, aDuration)
+  : MediaData(VIDEO_DATA, aOffset, aTime, aDuration)
   , mDisplay(aDisplay)
   , mDuplicate(false)
 {
diff --git a/dom/media/MediaData.h b/dom/media/MediaData.h
index 964ccdc58e16..93e0526fd084 100644
--- a/dom/media/MediaData.h
+++ b/dom/media/MediaData.h
@@ -73,11 +73,6 @@ public:
 
   int64_t GetEndTime() const { return mTime + mDuration; }
 
-  bool AdjustForStartTime(int64_t aStartTime)
-  {
-    mTime = mTime - aStartTime;
-    return mTime >= 0;
-  }
 protected:
   explicit MediaData(Type aType)
     : mType(aType)
@@ -104,15 +99,12 @@ public:
             AudioDataValue* aData,
             uint32_t aChannels,
             uint32_t aRate)
-    : MediaData(sType, aOffset, aTime, aDuration)
+    : MediaData(AUDIO_DATA, aOffset, aTime, aDuration)
     , mFrames(aFrames)
     , mChannels(aChannels)
     , mRate(aRate)
     , mAudioData(aData) {}
 
-  static const Type sType = AUDIO_DATA;
-  static const char* sTypeName;
-
   // Creates a new VideoData identical to aOther, but with a different
   // specified timestamp and duration. All data from aOther is copied
   // into the new AudioData but the audio data which is transferred.
@@ -156,9 +148,6 @@ public:
   typedef layers::Image Image;
   typedef layers::PlanarYCbCrImage PlanarYCbCrImage;
 
-  static const Type sType = VIDEO_DATA;
-  static const char* sTypeName;
-
   // YCbCr data obtained from decoding the video. The index's are:
   //   0 = Y
   //   1 = Cb
diff --git a/dom/media/MediaDecoderReader.cpp b/dom/media/MediaDecoderReader.cpp
index 451efc65d9f1..bc43ade298e2 100644
--- a/dom/media/MediaDecoderReader.cpp
+++ b/dom/media/MediaDecoderReader.cpp
@@ -148,14 +148,12 @@ void
 MediaDecoderReader::SetStartTime(int64_t aStartTime)
 {
   mDecoder->GetReentrantMonitor().AssertCurrentThreadIn();
-  MOZ_ASSERT(mStartTime == -1);
   mStartTime = aStartTime;
 }
 
 media::TimeIntervals
 MediaDecoderReader::GetBuffered()
 {
-  NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals());
   AutoPinned<MediaResource> stream(mDecoder->GetResource());
   int64_t durationUs = 0;
   {
@@ -165,6 +163,20 @@ MediaDecoderReader::GetBuffered()
   return GetEstimatedBufferedTimeRanges(stream, durationUs);
 }
 
+int64_t
+MediaDecoderReader::ComputeStartTime(const VideoData* aVideo, const AudioData* aAudio)
+{
+  int64_t startTime = std::min<int64_t>(aAudio ? aAudio->mTime : INT64_MAX,
+                                        aVideo ? aVideo->mTime : INT64_MAX);
+  if (startTime == INT64_MAX) {
+    startTime = 0;
+  }
+  DECODER_LOG("ComputeStartTime first video frame start %lld", aVideo ? aVideo->mTime : -1);
+  DECODER_LOG("ComputeStartTime first audio frame start %lld", aAudio ? aAudio->mTime : -1);
+  NS_ASSERTION(startTime >= 0, "Start time is negative");
+  return startTime;
+}
+
 nsRefPtr<MediaDecoderReader::MetadataPromise>
 MediaDecoderReader::AsyncReadMetadata()
 {
diff --git a/dom/media/MediaDecoderReader.h b/dom/media/MediaDecoderReader.h
index 77eb28b3224d..bc5e689dafba 100644
--- a/dom/media/MediaDecoderReader.h
+++ b/dom/media/MediaDecoderReader.h
@@ -213,8 +213,7 @@ public:
   // called.
   virtual media::TimeIntervals GetBuffered();
 
-  // MediaSourceReader opts out of the start-time-guessing mechanism.
-  virtual bool ForceZeroStartTime() const { return false; }
+  virtual int64_t ComputeStartTime(const VideoData* aVideo, const AudioData* aAudio);
 
   // The MediaDecoderStateMachine uses various heuristics that assume that
   // raw media data is arriving sequentially from a network channel. This
@@ -326,13 +325,7 @@ protected:
   // The start time of the media, in microseconds. This is the presentation
   // time of the first frame decoded from the media. This is initialized to -1,
   // and then set to a value >= by MediaDecoderStateMachine::SetStartTime(),
-  // after which point it never changes (though SetStartTime may be called
-  // multiple times with the same value).
-  //
-  // This is an ugly breach of abstractions - it's currently necessary for the
-  // readers to return the correct value of GetBuffered. We should refactor
-  // things such that all GetBuffered calls go through the MDSM, which would
-  // offset the range accordingly.
+  // after which point it never changes.
   int64_t mStartTime;
 
   // This is a quick-and-dirty way for DecodeAudioData implementations to
diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp
index 62d990fce662..38fa3b34ca98 100644
--- a/dom/media/MediaDecoderStateMachine.cpp
+++ b/dom/media/MediaDecoderStateMachine.cpp
@@ -56,6 +56,8 @@ using namespace mozilla::media;
 #undef DECODER_LOG
 #undef VERBOSE_LOG
 
+extern PRLogModuleInfo* gMediaDecoderLog;
+extern PRLogModuleInfo* gMediaSampleLog;
 #define LOG(m, l, x, ...) \
   MOZ_LOG(m, l, ("Decoder=%p " x, mDecoder.get(), ##__VA_ARGS__))
 #define DECODER_LOG(x, ...) \
@@ -187,6 +189,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
   mDelayedScheduler(this),
   mState(DECODER_STATE_DECODING_NONE, "MediaDecoderStateMachine::mState"),
   mPlayDuration(0),
+  mStartTime(-1),
   mEndTime(-1),
   mDurationSet(false),
   mEstimatedDuration(mTaskQueue, NullableTimeUnit(),
@@ -205,8 +208,8 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
   mFragmentEndTime(-1),
   mReader(aReader),
   mCurrentPosition(mTaskQueue, 0, "MediaDecoderStateMachine::mCurrentPosition (Canonical)"),
-  mStreamStartTime(0),
-  mAudioStartTime(0),
+  mStreamStartTime(-1),
+  mAudioStartTime(-1),
   mAudioEndTime(-1),
   mDecodedAudioEndTime(-1),
   mVideoFrameEndTime(-1),
@@ -223,7 +226,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
   mAudioCaptured(false),
   mPositionChangeQueued(false),
   mAudioCompleted(false, "MediaDecoderStateMachine::mAudioCompleted"),
-  mNotifyMetadataBeforeFirstFrame(false),
+  mGotDurationFromMetaData(false),
   mDispatchedEventToDecode(false),
   mQuickBuffering(false),
   mMinimizePreroll(false),
@@ -762,7 +765,6 @@ MediaDecoderStateMachine::OnAudioDecoded(AudioData* aAudioSample)
   nsRefPtr<AudioData> audio(aAudioSample);
   MOZ_ASSERT(audio);
   mAudioDataRequest.Complete();
-  aAudioSample->AdjustForStartTime(StartTime());
   mDecodedAudioEndTime = audio->GetEndTime();
 
   SAMPLE_LOG("OnAudioDecoded [%lld,%lld] disc=%d",
@@ -1043,9 +1045,7 @@ MediaDecoderStateMachine::OnVideoDecoded(VideoData* aVideoSample)
   MOZ_ASSERT(OnTaskQueue());
   ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
   nsRefPtr<VideoData> video(aVideoSample);
-  MOZ_ASSERT(video);
   mVideoDataRequest.Complete();
-  aVideoSample->AdjustForStartTime(StartTime());
   mDecodedVideoEndTime = video ? video->GetEndTime() : mDecodedVideoEndTime;
 
   SAMPLE_LOG("OnVideoDecoded [%lld,%lld] disc=%d",
@@ -1256,7 +1256,7 @@ void MediaDecoderStateMachine::StopPlayback()
   mDecoder->NotifyPlaybackStopped();
 
   if (IsPlaying()) {
-    mPlayDuration = GetClock();
+    mPlayDuration = GetClock() - mStartTime;
     SetPlayStartTime(TimeStamp());
   }
   // Notify the audio sink, so that it notices that we've stopped playing,
@@ -1306,10 +1306,11 @@ void MediaDecoderStateMachine::MaybeStartPlayback()
 void MediaDecoderStateMachine::UpdatePlaybackPositionInternal(int64_t aTime)
 {
   MOZ_ASSERT(OnTaskQueue());
-  SAMPLE_LOG("UpdatePlaybackPositionInternal(%lld)", aTime);
+  SAMPLE_LOG("UpdatePlaybackPositionInternal(%lld) (mStartTime=%lld)", aTime, mStartTime);
   AssertCurrentThreadInMonitor();
 
-  mCurrentPosition = aTime;
+  NS_ASSERTION(mStartTime >= 0, "Should have positive mStartTime");
+  mCurrentPosition = aTime - mStartTime;
   NS_ASSERTION(mCurrentPosition >= 0, "CurrentTime should be positive!");
   mObservedDuration = std::max(mObservedDuration.Ref(),
                                TimeUnit::FromMicroseconds(mCurrentPosition.Ref()));
@@ -1385,9 +1386,9 @@ int64_t MediaDecoderStateMachine::GetDuration()
 {
   AssertCurrentThreadInMonitor();
 
-  if (mEndTime == -1)
+  if (mEndTime == -1 || mStartTime == -1)
     return -1;
-  return mEndTime;
+  return mEndTime - mStartTime;
 }
 
 int64_t MediaDecoderStateMachine::GetEndTime()
@@ -1424,6 +1425,8 @@ void MediaDecoderStateMachine::RecomputeDuration()
     fireDurationChanged = true;
   } else if (mInfo.mMetadataDuration.isSome()) {
     duration = mInfo.mMetadataDuration.ref();
+  } else if (mInfo.mMetadataEndTime.isSome() && mStartTime >= 0) {
+    duration = mInfo.mMetadataEndTime.ref() - TimeUnit::FromMicroseconds(mStartTime);
   } else {
     return;
   }
@@ -1432,11 +1435,9 @@ void MediaDecoderStateMachine::RecomputeDuration()
     duration = mObservedDuration;
     fireDurationChanged = true;
   }
-  fireDurationChanged = fireDurationChanged && duration.ToMicroseconds() != GetDuration();
 
-  MOZ_ASSERT(duration.ToMicroseconds() >= 0);
-  mEndTime = duration.IsInfinite() ? -1 : duration.ToMicroseconds();
-  mDurationSet = true;
+  fireDurationChanged = fireDurationChanged && duration.ToMicroseconds() != GetDuration();
+  SetDuration(duration);
 
   if (fireDurationChanged) {
     nsCOMPtr<nsIRunnable> event =
@@ -1445,11 +1446,30 @@ void MediaDecoderStateMachine::RecomputeDuration()
   }
 }
 
+void MediaDecoderStateMachine::SetDuration(TimeUnit aDuration)
+{
+  MOZ_ASSERT(OnTaskQueue());
+  ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
+  MOZ_ASSERT(aDuration.ToMicroseconds() >= 0);
+  mDurationSet = true;
+
+  if (mStartTime == -1) {
+    SetStartTime(0);
+  }
+
+  if (aDuration.IsInfinite()) {
+    mEndTime = -1;
+    return;
+  }
+
+  mEndTime = mStartTime + aDuration.ToMicroseconds();
+}
+
 void MediaDecoderStateMachine::SetFragmentEndTime(int64_t aEndTime)
 {
   AssertCurrentThreadInMonitor();
 
-  mFragmentEndTime = aEndTime < 0 ? aEndTime : aEndTime;
+  mFragmentEndTime = aEndTime < 0 ? aEndTime : aEndTime + mStartTime;
 }
 
 bool MediaDecoderStateMachine::IsDormantNeeded()
@@ -1548,11 +1568,6 @@ void MediaDecoderStateMachine::Shutdown()
 
   Reset();
 
-  // Shut down our start time rendezvous.
-  if (mStartTimeRendezvous) {
-    mStartTimeRendezvous->Destroy();
-  }
-
   // Put a task in the decode queue to shutdown the reader.
   // the queue to spin down.
   ProxyMediaCall(DecodeTaskQueue(), mReader.get(), __func__, &MediaDecoderReader::Shutdown)
@@ -1676,11 +1691,9 @@ void MediaDecoderStateMachine::NotifyDataArrived(const char* aBuffer,
   //
   // Make sure to only do this if we have a start time, otherwise the reader
   // doesn't know how to compute GetBuffered.
-  if (!mDecoder->IsInfinite() || !HaveStartTime())
-  {
+  if (!mDecoder->IsInfinite() || mStartTime == -1) {
     return;
   }
-
   media::TimeIntervals buffered{mDecoder->GetBuffered()};
   if (!buffered.IsInvalid()) {
     bool exists;
@@ -1836,11 +1849,12 @@ MediaDecoderStateMachine::InitiateSeek()
 
   // Bound the seek time to be inside the media range.
   int64_t end = GetEndTime();
+  NS_ASSERTION(mStartTime != -1, "Should know start time by now");
   NS_ASSERTION(end != -1, "Should know end time by now");
-  int64_t seekTime = mCurrentSeek.mTarget.mTime;
+  int64_t seekTime = mCurrentSeek.mTarget.mTime + mStartTime;
   seekTime = std::min(seekTime, end);
-  seekTime = std::max(int64_t(0), seekTime);
-  NS_ASSERTION(seekTime >= 0 && seekTime <= end,
+  seekTime = std::max(mStartTime, seekTime);
+  NS_ASSERTION(seekTime >= mStartTime && seekTime <= end,
                "Can only seek in range [0,duration]");
   mCurrentSeek.mTarget.mTime = seekTime;
 
@@ -2155,36 +2169,9 @@ MediaDecoderStateMachine::OnMetadataRead(MetadataHolder* aMetadata)
   mDecoder->SetMediaSeekable(mReader->IsMediaSeekable());
   mInfo = aMetadata->mInfo;
   mMetadataTags = aMetadata->mTags.forget();
-  nsRefPtr<MediaDecoderStateMachine> self = this;
 
-  // Set up the start time rendezvous if it doesn't already exist (which is
-  // generally the case, unless we're coming out of dormant mode).
-  if (!mStartTimeRendezvous) {
-    mStartTimeRendezvous = new StartTimeRendezvous(TaskQueue(), HasAudio(), HasVideo(),
-                                                   mReader->ForceZeroStartTime() || IsRealTime());
-
-    mStartTimeRendezvous->AwaitStartTime()->Then(TaskQueue(), __func__,
-      [self] () -> void {
-        NS_ENSURE_TRUE_VOID(!self->IsShutdown());
-        ReentrantMonitorAutoEnter mon(self->mDecoder->GetReentrantMonitor());
-        self->mReader->SetStartTime(self->StartTime());
-      },
-      [] () -> void { NS_WARNING("Setting start time on reader failed"); }
-    );
-  }
-
-  if (mInfo.mMetadataDuration.isSome()) {
+  if (mInfo.mMetadataDuration.isSome() || mInfo.mMetadataEndTime.isSome()) {
     RecomputeDuration();
-  } else if (mInfo.mUnadjustedMetadataEndTime.isSome()) {
-    mStartTimeRendezvous->AwaitStartTime()->Then(TaskQueue(), __func__,
-      [self] () -> void {
-        NS_ENSURE_TRUE_VOID(!self->IsShutdown());
-        TimeUnit unadjusted = self->mInfo.mUnadjustedMetadataEndTime.ref();
-        TimeUnit adjustment = TimeUnit::FromMicroseconds(self->StartTime());
-        self->mInfo.mMetadataDuration.emplace(unadjusted - adjustment);
-        self->RecomputeDuration();
-      }, [] () -> void { NS_WARNING("Adjusting metadata end time failed"); }
-    );
   }
 
   if (HasVideo()) {
@@ -2195,15 +2182,10 @@ MediaDecoderStateMachine::OnMetadataRead(MetadataHolder* aMetadata)
   }
 
   mDecoder->StartProgressUpdates();
-
-  // In general, we wait until we know the duration before notifying the decoder.
-  // However, we notify  unconditionally in this case without waiting for the start
-  // time, since the caller might be waiting on metadataloaded to be fired before
-  // feeding in the CDM, which we need to decode the first frame (and
-  // thus get the metadata). We could fix this if we could compute the start
-  // time by demuxing without necessaring decoding.
-  mNotifyMetadataBeforeFirstFrame = mDurationSet || mReader->IsWaitingOnCDMResource();
-  if (mNotifyMetadataBeforeFirstFrame) {
+  mGotDurationFromMetaData = (GetDuration() != -1) || mDurationSet;
+  if (mGotDurationFromMetaData) {
+    // We have all the information required: duration and size
+    // Inform the element that we've loaded the metadata.
     EnqueueLoadedMetadataEvent();
   }
 
@@ -2289,37 +2271,29 @@ MediaDecoderStateMachine::DecodeFirstFrame()
   DECODER_LOG("DecodeFirstFrame started");
 
   if (IsRealTime()) {
+    SetStartTime(0);
     nsresult res = FinishDecodeFirstFrame();
     NS_ENSURE_SUCCESS(res, res);
   } else if (mSentFirstFrameLoadedEvent) {
     // We're resuming from dormant state, so we don't need to request
     // the first samples in order to determine the media start time,
     // we have the start time from last time we loaded.
+    SetStartTime(mStartTime);
     nsresult res = FinishDecodeFirstFrame();
     NS_ENSURE_SUCCESS(res, res);
   } else {
     if (HasAudio()) {
-      mAudioDataRequest.Begin(
-        ProxyMediaCall(DecodeTaskQueue(), mReader.get(), __func__,
-                       &MediaDecoderReader::RequestAudioData)
-        ->Then(TaskQueue(), __func__, mStartTimeRendezvous.get(),
-               &StartTimeRendezvous::ProcessFirstSample<AudioDataPromise>,
-               &StartTimeRendezvous::FirstSampleRejected<AudioData>)
-        ->CompletionPromise()
+      mAudioDataRequest.Begin(ProxyMediaCall(DecodeTaskQueue(), mReader.get(),
+                                             __func__, &MediaDecoderReader::RequestAudioData)
         ->Then(TaskQueue(), __func__, this,
                &MediaDecoderStateMachine::OnAudioDecoded,
-               &MediaDecoderStateMachine::OnAudioNotDecoded)
-      );
+               &MediaDecoderStateMachine::OnAudioNotDecoded));
     }
     if (HasVideo()) {
       mVideoDecodeStartTime = TimeStamp::Now();
-      mVideoDataRequest.Begin(
-        ProxyMediaCall(DecodeTaskQueue(), mReader.get(), __func__,
-                       &MediaDecoderReader::RequestVideoData, false, int64_t(0))
-        ->Then(TaskQueue(), __func__, mStartTimeRendezvous.get(),
-               &StartTimeRendezvous::ProcessFirstSample<VideoDataPromise>,
-               &StartTimeRendezvous::FirstSampleRejected<VideoData>)
-        ->CompletionPromise()
+      mVideoDataRequest.Begin(ProxyMediaCall(DecodeTaskQueue(), mReader.get(),
+                                             __func__, &MediaDecoderReader::RequestVideoData, false,
+                                             int64_t(0))
         ->Then(TaskQueue(), __func__, this,
                &MediaDecoderStateMachine::OnVideoDecoded,
                &MediaDecoderStateMachine::OnVideoNotDecoded));
@@ -2341,18 +2315,22 @@ MediaDecoderStateMachine::FinishDecodeFirstFrame()
   }
 
   if (!IsRealTime() && !mSentFirstFrameLoadedEvent) {
+    const VideoData* v = VideoQueue().PeekFront();
+    const AudioData* a = AudioQueue().PeekFront();
+    SetStartTime(mReader->ComputeStartTime(v, a));
     if (VideoQueue().GetSize()) {
       ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor());
       RenderVideoFrame(VideoQueue().PeekFront(), TimeStamp::Now());
     }
   }
 
+  NS_ASSERTION(mStartTime != -1, "Must have start time");
   MOZ_ASSERT(!(mDecoder->IsMediaSeekable() && mDecoder->IsTransportSeekable()) ||
                (GetDuration() != -1) || mDurationSet,
              "Seekable media should have duration");
   DECODER_LOG("Media goes from %lld to %lld (duration %lld) "
               "transportSeekable=%d, mediaSeekable=%d",
-              0, mEndTime, GetDuration(),
+              mStartTime, mEndTime, GetDuration(),
               mDecoder->IsTransportSeekable(), mDecoder->IsMediaSeekable());
 
   if (HasAudio() && !HasVideo()) {
@@ -2372,11 +2350,15 @@ MediaDecoderStateMachine::FinishDecodeFirstFrame()
 
   nsAutoPtr<MediaInfo> info(new MediaInfo());
   *info = mInfo;
-  if (!mNotifyMetadataBeforeFirstFrame) {
-    // If we didn't have duration and/or start time before, we should now.
+  if (!mGotDurationFromMetaData) {
+    // We now have a duration, we can fire the LoadedMetadata and
+    // FirstFrame event.
     EnqueueLoadedMetadataEvent();
+    EnqueueFirstFrameLoadedEvent();
+  } else {
+    // Inform the element that we've loaded the first frame.
+    EnqueueFirstFrameLoadedEvent();
   }
-  EnqueueFirstFrameLoadedEvent();
 
   if (mState == DECODER_STATE_DECODING_FIRSTFRAME) {
     StartDecoding();
@@ -2426,7 +2408,7 @@ MediaDecoderStateMachine::SeekCompleted()
     newCurrentTime = video ? video->mTime : seekTime;
   }
   mStreamStartTime = newCurrentTime;
-  mPlayDuration = newCurrentTime;
+  mPlayDuration = newCurrentTime - mStartTime;
 
   mDecoder->StartProgressUpdates();
 
@@ -2743,8 +2725,8 @@ MediaDecoderStateMachine::Reset()
 
   mVideoFrameEndTime = -1;
   mDecodedVideoEndTime = -1;
-  mStreamStartTime = 0;
-  mAudioStartTime = 0;
+  mStreamStartTime = -1;
+  mAudioStartTime = -1;
   mAudioEndTime = -1;
   mDecodedAudioEndTime = -1;
   mAudioCompleted = false;
@@ -2813,7 +2795,7 @@ void MediaDecoderStateMachine::ResyncAudioClock()
   AssertCurrentThreadInMonitor();
   if (IsPlaying()) {
     SetPlayStartTime(TimeStamp::Now());
-    mPlayDuration = GetAudioClock();
+    mPlayDuration = GetAudioClock() - mStartTime;
   }
 }
 
@@ -2843,14 +2825,14 @@ int64_t MediaDecoderStateMachine::GetVideoStreamPosition() const
   AssertCurrentThreadInMonitor();
 
   if (!IsPlaying()) {
-    return mPlayDuration;
+    return mPlayDuration + mStartTime;
   }
 
   // Time elapsed since we started playing.
   int64_t delta = DurationToUsecs(TimeStamp::Now() - mPlayStartTime);
   // Take playback rate into account.
   delta *= mPlaybackRate;
-  return mPlayDuration + delta;
+  return mStartTime + mPlayDuration + delta;
 }
 
 int64_t MediaDecoderStateMachine::GetClock() const
@@ -2864,7 +2846,7 @@ int64_t MediaDecoderStateMachine::GetClock() const
   // fed to a MediaStream, use that stream as the source of the clock.
   int64_t clock_time = -1;
   if (!IsPlaying()) {
-    clock_time = mPlayDuration;
+    clock_time = mPlayDuration + mStartTime;
   } else {
     if (mAudioCaptured) {
       clock_time = GetStreamClock();
@@ -2907,7 +2889,7 @@ void MediaDecoderStateMachine::AdvanceFrame()
   // Skip frames up to the frame at the playback position, and figure out
   // the time remaining until it's time to display the next frame.
   int64_t remainingTime = AUDIO_DURATION_USECS;
-  NS_ASSERTION(clock_time >= 0, "Should have positive clock time.");
+  NS_ASSERTION(clock_time >= mStartTime, "Should have positive clock time.");
   nsRefPtr<VideoData> currentFrame;
   if (VideoQueue().GetSize() > 0) {
     VideoData* frame = VideoQueue().PeekFront();
@@ -2987,7 +2969,7 @@ void MediaDecoderStateMachine::AdvanceFrame()
     // Decode one frame and display it.
     int64_t delta = currentFrame->mTime - clock_time;
     TimeStamp presTime = nowTime + TimeDuration::FromMicroseconds(delta / mPlaybackRate);
-    NS_ASSERTION(currentFrame->mTime >= 0, "Should have positive frame time");
+    NS_ASSERTION(currentFrame->mTime >= mStartTime, "Should have positive frame time");
     {
       ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor());
       // If we have video, we want to increment the clock in steps of the frame
@@ -3148,6 +3130,37 @@ MediaDecoderStateMachine::DropAudioUpToSeekTarget(AudioData* aSample)
   return NS_OK;
 }
 
+void MediaDecoderStateMachine::SetStartTime(int64_t aStartTimeUsecs)
+{
+  AssertCurrentThreadInMonitor();
+  DECODER_LOG("SetStartTime(%lld)", aStartTimeUsecs);
+  mStartTime = 0;
+  if (aStartTimeUsecs != 0) {
+    mStartTime = aStartTimeUsecs;
+    if (mGotDurationFromMetaData && GetEndTime() != INT64_MAX) {
+      NS_ASSERTION(mEndTime != -1,
+                   "We should have mEndTime as supplied duration here");
+      // We were specified a duration from a Content-Duration HTTP header.
+      // Adjust mEndTime so that mEndTime-mStartTime matches the specified
+      // duration.
+      mEndTime = mStartTime + mEndTime;
+    }
+  }
+
+  // Pass along this immutable value to the reader so that it can make
+  // calculations independently of the state machine.
+  mReader->SetStartTime(mStartTime);
+
+  // Set the audio start time to be start of media. If this lies before the
+  // first actual audio frame we have, we'll inject silence during playback
+  // to ensure the audio starts at the correct time.
+  mAudioStartTime = mStartTime;
+  mStreamStartTime = mStartTime;
+  DECODER_LOG("Set media start time to %lld", mStartTime);
+
+  RecomputeDuration();
+}
+
 void MediaDecoderStateMachine::UpdateNextFrameStatus()
 {
   MOZ_ASSERT(OnTaskQueue());
@@ -3315,7 +3328,7 @@ MediaDecoderStateMachine::LogicalPlaybackRateChanged()
   if (!HasAudio() && IsPlaying()) {
     // Remember how much time we've spent in playing the media
     // for playback rate will change from now on.
-    mPlayDuration = GetVideoStreamPosition();
+    mPlayDuration = GetVideoStreamPosition() - mStartTime;
     SetPlayStartTime(TimeStamp::Now());
   }
 
@@ -3337,6 +3350,7 @@ void MediaDecoderStateMachine::PreservesPitchChanged()
 
 bool MediaDecoderStateMachine::IsShutdown()
 {
+  AssertCurrentThreadInMonitor();
   return mState == DECODER_STATE_ERROR ||
          mState == DECODER_STATE_SHUTDOWN;
 }
diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h
index de3ae2ff5ba0..dd04fd1c666e 100644
--- a/dom/media/MediaDecoderStateMachine.h
+++ b/dom/media/MediaDecoderStateMachine.h
@@ -100,9 +100,6 @@ class AudioSegment;
 class MediaTaskQueue;
 class AudioSink;
 
-extern PRLogModuleInfo* gMediaDecoderLog;
-extern PRLogModuleInfo* gMediaSampleLog;
-
 /*
   The state machine class. This manages the decoding and seeking in the
   MediaDecoderReader on the decode task queue, and A/V sync on the shared
@@ -120,8 +117,6 @@ class MediaDecoderStateMachine
   friend class AudioSink;
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDecoderStateMachine)
 public:
-  typedef MediaDecoderReader::AudioDataPromise AudioDataPromise;
-  typedef MediaDecoderReader::VideoDataPromise VideoDataPromise;
   typedef MediaDecoderOwner::NextFrameStatus NextFrameStatus;
   MediaDecoderStateMachine(MediaDecoder* aDecoder,
                            MediaDecoderReader* aReader,
@@ -196,6 +191,14 @@ public:
   // Access controlled by decoder monitor.
   int64_t GetEndTime();
 
+  // Called from the main thread to set the duration of the media resource
+  // if it is able to be obtained via HTTP headers. Called from the
+  // state machine thread to set the duration if it is obtained from the
+  // media metadata. The decoder monitor must be obtained before calling this.
+  // aDuration is in microseconds.
+  // A value of INT64_MAX will be treated as infinity.
+  void SetDuration(media::TimeUnit aDuration);
+
   // Functions used by assertions to ensure we're calling things
   // on the appropriate threads.
   bool OnDecodeTaskQueue() const;
@@ -263,7 +266,14 @@ public:
   }
 
   media::TimeIntervals GetBuffered() {
+    // It's possible for JS to query .buffered before we've determined the start
+    // time from metadata, in which case the reader isn't ready to be asked this
+    // question.
     ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
+    if (mStartTime < 0) {
+      return media::TimeIntervals();
+    }
+
     return mReader->GetBuffered();
   }
 
@@ -598,7 +608,7 @@ protected:
   // which is in the range [0,duration].
   int64_t GetMediaTime() const {
     AssertCurrentThreadInMonitor();
-    return mCurrentPosition;
+    return mStartTime + mCurrentPosition;
   }
 
   // Returns an upper bound on the number of microseconds of audio that is
@@ -779,133 +789,6 @@ public:
 
   } mDelayedScheduler;
 
-  // StartTimeRendezvous is a helper class that quarantines the first sample
-  // until it gets a sample from both channels, such that we can be guaranteed
-  // to know the start time by the time On{Audio,Video}Decoded is called.
-  class StartTimeRendezvous {
-  public:
-    typedef MediaDecoderReader::AudioDataPromise AudioDataPromise;
-    typedef MediaDecoderReader::VideoDataPromise VideoDataPromise;
-    typedef MediaPromise<bool, bool, /* isExclusive = */ false> HaveStartTimePromise;
-
-    NS_INLINE_DECL_THREADSAFE_REFCOUNTING(StartTimeRendezvous);
-    StartTimeRendezvous(AbstractThread* aOwnerThread, bool aHasAudio, bool aHasVideo,
-                        bool aForceZeroStartTime)
-      : mOwnerThread(aOwnerThread)
-    {
-      if (aForceZeroStartTime) {
-        mAudioStartTime.emplace(0);
-        mVideoStartTime.emplace(0);
-        return;
-      }
-
-      if (!aHasAudio) {
-        mAudioStartTime.emplace(INT64_MAX);
-      }
-
-      if (!aHasVideo) {
-        mVideoStartTime.emplace(INT64_MAX);
-      }
-    }
-
-    void Destroy()
-    {
-      mAudioStartTime = Some(mAudioStartTime.refOr(INT64_MAX));
-      mVideoStartTime = Some(mVideoStartTime.refOr(INT64_MAX));
-      mHaveStartTimePromise.RejectIfExists(false, __func__);
-    }
-
-    nsRefPtr<HaveStartTimePromise> AwaitStartTime()
-    {
-      if (HaveStartTime()) {
-        return HaveStartTimePromise::CreateAndResolve(true, __func__);
-      }
-      return mHaveStartTimePromise.Ensure(__func__);
-    }
-
-    template<typename PromiseType>
-    struct PromiseSampleType {
-      typedef typename PromiseType::ResolveValueType::element_type Type;
-    };
-
-    template<typename PromiseType>
-    nsRefPtr<PromiseType> ProcessFirstSample(typename PromiseSampleType<PromiseType>::Type* aData)
-    {
-      typedef typename PromiseSampleType<PromiseType>::Type DataType;
-      typedef typename PromiseType::Private PromisePrivate;
-      MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
-
-      MaybeSetChannelStartTime<DataType>(aData->mTime);
-
-      nsRefPtr<PromisePrivate> p = new PromisePrivate(__func__);
-      nsRefPtr<DataType> data = aData;
-      nsRefPtr<StartTimeRendezvous> self = this;
-      AwaitStartTime()->Then(mOwnerThread, __func__,
-                             [p, data, self] () -> void {
-                               MOZ_ASSERT(self->mOwnerThread->IsCurrentThreadIn());
-                               p->Resolve(data, __func__);
-                             },
-                             [p] () -> void { p->Reject(MediaDecoderReader::CANCELED, __func__); });
-
-      return p.forget();
-    }
-
-    template<typename SampleType>
-    void FirstSampleRejected(MediaDecoderReader::NotDecodedReason aReason)
-    {
-      MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
-      if (aReason == MediaDecoderReader::DECODE_ERROR) {
-        mHaveStartTimePromise.RejectIfExists(false, __func__);
-      } else if (aReason == MediaDecoderReader::END_OF_STREAM) {
-        MOZ_LOG(gMediaDecoderLog, LogLevel::Debug,
-                ("StartTimeRendezvous=%p %s Has no samples.", this, SampleType::sTypeName));
-        MaybeSetChannelStartTime<SampleType>(INT64_MAX);
-      }
-    }
-
-    bool HaveStartTime() { return mAudioStartTime.isSome() && mVideoStartTime.isSome(); }
-    int64_t StartTime()
-    {
-      int64_t time = std::min(mAudioStartTime.ref(), mVideoStartTime.ref());
-      return time == INT64_MAX ? 0 : time;
-    }
-  private:
-    virtual ~StartTimeRendezvous() {}
-
-    template<typename SampleType>
-    void MaybeSetChannelStartTime(int64_t aStartTime)
-    {
-      if (ChannelStartTime(SampleType::sType).isSome()) {
-        // If we're initialized with aForceZeroStartTime=true, the channel start
-        // times are already set.
-        return;
-      }
-
-      MOZ_LOG(gMediaDecoderLog, LogLevel::Debug,
-              ("StartTimeRendezvous=%p Setting %s start time to %lld",
-               this, SampleType::sTypeName, aStartTime));
-
-      ChannelStartTime(SampleType::sType).emplace(aStartTime);
-      if (HaveStartTime()) {
-        mHaveStartTimePromise.ResolveIfExists(true, __func__);
-      }
-    }
-
-    Maybe<int64_t>& ChannelStartTime(MediaData::Type aType)
-    {
-      return aType == MediaData::AUDIO_DATA ? mAudioStartTime : mVideoStartTime;
-    }
-
-    MediaPromiseHolder<HaveStartTimePromise> mHaveStartTimePromise;
-    nsRefPtr<AbstractThread> mOwnerThread;
-    Maybe<int64_t> mAudioStartTime;
-    Maybe<int64_t> mVideoStartTime;
-  };
-  nsRefPtr<StartTimeRendezvous> mStartTimeRendezvous;
-
-  bool HaveStartTime() { return mStartTimeRendezvous && mStartTimeRendezvous->HaveStartTime(); }
-  int64_t StartTime() { return mStartTimeRendezvous->StartTime(); }
-
   // Time at which the last video sample was requested. If it takes too long
   // before the sample arrives, we will increase the amount of audio we buffer.
   // This is necessary for legacy synchronous decoders to prevent underruns.
@@ -947,6 +830,12 @@ public:
   // buffering.
   TimeStamp mBufferingStart;
 
+  // Start time of the media, in microseconds. This is the presentation
+  // time of the first frame decoded from the media, and is used to calculate
+  // duration and as a bounds for seeking. Accessed on state machine, decode,
+  // and main threads. Access controlled by decoder monitor.
+  int64_t mStartTime;
+
   // Time of the last frame in the media, in microseconds. This is the
   // end time of the last frame in the media. Accessed on state
   // machine, decode, and main threads. Access controlled by decoder monitor.
@@ -1269,13 +1158,9 @@ protected:
   // been written to the MediaStream.
   Watchable<bool> mAudioCompleted;
 
-  // Flag whether we notify metadata before decoding the first frame or after.
-  //
-  // Note that the odd semantics here are designed to replicate the current
-  // behavior where we notify the decoder each time we come out of dormant, but
-  // send suppressed event visibility for those cases. This code can probably be
-  // simplified.
-  bool mNotifyMetadataBeforeFirstFrame;
+  // True if mDuration has a value obtained from an HTTP header, or from
+  // the media index/metadata. Accessed on the state machine thread.
+  bool mGotDurationFromMetaData;
 
   // True if we've dispatched an event to the decode task queue to call
   // DecodeThreadRun(). We use this flag to prevent us from dispatching
diff --git a/dom/media/MediaFormatReader.cpp b/dom/media/MediaFormatReader.cpp
index fedf47db1dc5..0982a6189934 100644
--- a/dom/media/MediaFormatReader.cpp
+++ b/dom/media/MediaFormatReader.cpp
@@ -1300,7 +1300,7 @@ MediaFormatReader::GetBuffered()
   int64_t startTime;
   {
     ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
-    NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals());
+    MOZ_ASSERT(mStartTime != -1, "Need to finish metadata decode first");
     startTime = mStartTime;
   }
   if (NS_IsMainThread()) {
@@ -1474,10 +1474,13 @@ MediaFormatReader::NotifyDataRemoved()
   TaskQueue()->Dispatch(task.forget());
 }
 
-bool
-MediaFormatReader::ForceZeroStartTime() const
+int64_t
+MediaFormatReader::ComputeStartTime(const VideoData* aVideo, const AudioData* aAudio)
 {
-  return !mDemuxer->ShouldComputeStartTime();
+  if (mDemuxer->ShouldComputeStartTime()) {
+    return MediaDecoderReader::ComputeStartTime(aVideo, aAudio);
+  }
+  return 0;
 }
 
 } // namespace mozilla
diff --git a/dom/media/MediaFormatReader.h b/dom/media/MediaFormatReader.h
index 6cabca0619f9..6abc3329d806 100644
--- a/dom/media/MediaFormatReader.h
+++ b/dom/media/MediaFormatReader.h
@@ -75,8 +75,6 @@ public:
 
   media::TimeIntervals GetBuffered() override;
 
-  virtual bool ForceZeroStartTime() const override;
-
   // For Media Resource Management
   void SetIdle() override;
   bool IsDormantNeeded() override;
@@ -99,6 +97,8 @@ public:
 
   bool IsWaitingOnCDMResource() override;
 
+  int64_t ComputeStartTime(const VideoData* aVideo, const AudioData* aAudio) override;
+
   bool UseBufferingHeuristics() override
   {
     return mTrackDemuxersMayBlock;
diff --git a/dom/media/MediaInfo.h b/dom/media/MediaInfo.h
index 261dde9d09f3..61e0a34b67bc 100644
--- a/dom/media/MediaInfo.h
+++ b/dom/media/MediaInfo.h
@@ -355,7 +355,7 @@ public:
   // The Ogg reader tries to kinda-sorta compute the duration by seeking to the
   // end and determining the timestamp of the last frame. This isn't useful as
   // a duration until we know the start time, so we need to track it separately.
-  media::NullableTimeUnit mUnadjustedMetadataEndTime;
+  media::NullableTimeUnit mMetadataEndTime;
 
   EncryptionInfo mCrypto;
 };
diff --git a/dom/media/fmp4/MP4Reader.cpp b/dom/media/fmp4/MP4Reader.cpp
index 3d7d91ad1eed..6fad89c47701 100644
--- a/dom/media/fmp4/MP4Reader.cpp
+++ b/dom/media/fmp4/MP4Reader.cpp
@@ -1081,7 +1081,7 @@ MP4Reader::GetBuffered()
     return buffered;
   }
   UpdateIndex();
-  NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals());
+  MOZ_ASSERT(mStartTime != -1, "Need to finish metadata decode first");
 
   AutoPinned<MediaResource> resource(mDecoder->GetResource());
   nsTArray<MediaByteRange> ranges;
diff --git a/dom/media/mediasource/MediaSourceReader.h b/dom/media/mediasource/MediaSourceReader.h
index f4fb66092f3c..2e68e2d5d3ee 100644
--- a/dom/media/mediasource/MediaSourceReader.h
+++ b/dom/media/mediasource/MediaSourceReader.h
@@ -94,7 +94,7 @@ public:
   // We can't compute a proper start time since we won't necessarily
   // have the first frame of the resource available. This does the same
   // as chrome/blink and assumes that we always start at t=0.
-  virtual bool ForceZeroStartTime() const override { return true; }
+  virtual int64_t ComputeStartTime(const VideoData* aVideo, const AudioData* aAudio) override { return 0; }
 
   // Buffering heuristics don't make sense for MSE, because the arrival of data
   // is at least partly controlled by javascript, and javascript does not expect
diff --git a/dom/media/ogg/OggReader.cpp b/dom/media/ogg/OggReader.cpp
index f02709dc67ff..8221a3e12291 100644
--- a/dom/media/ogg/OggReader.cpp
+++ b/dom/media/ogg/OggReader.cpp
@@ -489,7 +489,7 @@ nsresult OggReader::ReadMetadata(MediaInfo* aInfo,
         endTime = RangeEndTime(length);
       }
       if (endTime != -1) {
-        mInfo.mUnadjustedMetadataEndTime.emplace(TimeUnit::FromMicroseconds(endTime));
+        mInfo.mMetadataEndTime.emplace(TimeUnit::FromMicroseconds(endTime));
         LOG(LogLevel::Debug, ("Got Ogg duration from seeking to end %lld", endTime));
       }
     }
@@ -1852,7 +1852,7 @@ nsresult OggReader::SeekBisection(int64_t aTarget,
 
 media::TimeIntervals OggReader::GetBuffered()
 {
-  NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals());
+  MOZ_ASSERT(mStartTime != -1, "Need to finish metadata decode first");
   {
     mozilla::ReentrantMonitorAutoEnter mon(mMonitor);
     if (mIsChained) {
diff --git a/dom/media/test/test_buffered.html b/dom/media/test/test_buffered.html
index 407ae7404055..892e39ef2d8e 100644
--- a/dom/media/test/test_buffered.html
+++ b/dom/media/test/test_buffered.html
@@ -25,7 +25,6 @@ var manager = new MediaTestManager;
 
 function testBuffered(e) {
   var v = e.target;
-  v.removeEventListener('timeupdate', testBuffered);
   
   // The whole media should be buffered...
   var b = v.buffered;
@@ -95,12 +94,12 @@ function startTest(test, token) {
   // we have deterministic behaviour.
   var onfetched = function(uri) {
     var v = document.createElement('video');
-    v.autoplay = true;
+    v.preload = "metadata";
     v._token = token;
     v.src = uri;
     v._name = test.name;
     v._test = test;
-    v.addEventListener("timeupdate", testBuffered, false);
+    v.addEventListener("loadedmetadata", testBuffered, false);
     document.body.appendChild(v);
   };
 
diff --git a/dom/media/webm/WebMReader.cpp b/dom/media/webm/WebMReader.cpp
index 638e0f6a2f42..1ed5c8ac3562 100644
--- a/dom/media/webm/WebMReader.cpp
+++ b/dom/media/webm/WebMReader.cpp
@@ -1093,7 +1093,7 @@ nsresult WebMReader::SeekInternal(int64_t aTarget)
 
 media::TimeIntervals WebMReader::GetBuffered()
 {
-  NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals());
+  MOZ_ASSERT(mStartTime != -1, "Need to finish metadata decode first");
   AutoPinned<MediaResource> resource(mDecoder->GetResource());
 
   media::TimeIntervals buffered;