diff --git a/dom/media/encoder/TrackEncoder.cpp b/dom/media/encoder/TrackEncoder.cpp
index 189163609edd..f1961c985798 100644
--- a/dom/media/encoder/TrackEncoder.cpp
+++ b/dom/media/encoder/TrackEncoder.cpp
@@ -317,11 +317,13 @@ VideoTrackEncoder::AppendVideoSegment(const VideoSegment& aSegment)
       // Adapt to the time before the first frame. This extends the first frame
       // from [start, end] to [0, end], but it'll do for now.
       CheckedInt64 diff = FramesToUsecs(nullDuration, mTrackRate);
-      MOZ_ASSERT(diff.isValid());
-      if (diff.isValid()) {
-        mLastChunk.mTimeStamp -= TimeDuration::FromMicroseconds(diff.value());
-        mLastChunk.mDuration += nullDuration;
+      if (!diff.isValid()) {
+        NS_ERROR("null duration overflow");
+        return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
       }
+
+      mLastChunk.mTimeStamp -= TimeDuration::FromMicroseconds(diff.value());
+      mLastChunk.mDuration += nullDuration;
     }
 
     MOZ_ASSERT(!mLastChunk.IsNull());
@@ -357,25 +359,41 @@ VideoTrackEncoder::AppendVideoSegment(const VideoSegment& aSegment)
       }
     }
 
-    TimeDuration diff = chunk.mTimeStamp - mLastChunk.mTimeStamp;
-    if (diff <= TimeDuration::FromSeconds(0)) {
-      // The timestamp from mLastChunk is newer than from chunk.
-      // This means the durations reported from MediaStreamGraph for mLastChunk
-      // were larger than the timestamp diff - and durations were used to
-      // trigger the 1-second frame above. This could happen due to drift or
-      // underruns in the graph.
-      TRACK_LOG(LogLevel::Warning,
-                ("[VideoTrackEncoder]: Underrun detected. Diff=%.5fs",
-                 diff.ToSeconds()));
-      chunk.mTimeStamp = mLastChunk.mTimeStamp;
-    } else {
-      RefPtr<layers::Image> lastImage = mLastChunk.mFrame.GetImage();
-      TRACK_LOG(LogLevel::Verbose,
-                ("[VideoTrackEncoder]: Appending video frame %p, duration=%.5f",
-                 lastImage.get(), diff.ToSeconds()));
-      CheckedInt64 duration = UsecsToFrames(diff.ToMicroseconds(), mTrackRate);
-      MOZ_ASSERT(duration.isValid());
-      if (duration.isValid()) {
+    if (mStartOffset.IsNull()) {
+      mStartOffset = mLastChunk.mTimeStamp;
+    }
+
+    TimeDuration relativeTime = chunk.mTimeStamp - mStartOffset;
+    RefPtr<layers::Image> lastImage = mLastChunk.mFrame.GetImage();
+    TRACK_LOG(LogLevel::Verbose,
+              ("[VideoTrackEncoder]: Appending video frame %p, at pos %.5fs",
+               lastImage.get(), relativeTime.ToSeconds()));
+    CheckedInt64 totalDuration =
+      UsecsToFrames(relativeTime.ToMicroseconds(), mTrackRate);
+    if (!totalDuration.isValid()) {
+      NS_ERROR("Duration overflow");
+      return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
+    }
+
+    CheckedInt64 duration = totalDuration - mEncodedTicks;
+    if (!duration.isValid()) {
+      NS_ERROR("Duration overflow");
+      return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
+    }
+
+    if (duration.isValid()) {
+      if (duration.value() <= 0) {
+        // The timestamp for mLastChunk is newer than for chunk.
+        // This means the durations reported from MediaStreamGraph for
+        // mLastChunk were larger than the timestamp diff - and durations were
+        // used to trigger the 1-second frame above. This could happen due to
+        // drift or underruns in the graph.
+        TRACK_LOG(LogLevel::Warning,
+                  ("[VideoTrackEncoder]: Underrun detected. Diff=%lld",
+                   duration.value()));
+        chunk.mTimeStamp = mLastChunk.mTimeStamp;
+      } else {
+        mEncodedTicks += duration.value();
         mRawSegment.AppendFrame(lastImage.forget(),
                                 duration.value(),
                                 mLastChunk.mFrame.GetIntrinsicSize(),
diff --git a/dom/media/encoder/TrackEncoder.h b/dom/media/encoder/TrackEncoder.h
index 96fbf5df37d0..02d6c927e037 100644
--- a/dom/media/encoder/TrackEncoder.h
+++ b/dom/media/encoder/TrackEncoder.h
@@ -76,7 +76,7 @@ public:
   {
     ReentrantMonitorAutoEnter mon(mReentrantMonitor);
     mCanceled = true;
-    mReentrantMonitor.NotifyAll();
+    NotifyEndOfStream();
   }
 
   virtual void SetBitrate(const uint32_t aBitrate) {}
@@ -255,6 +255,7 @@ public:
     , mDisplayWidth(0)
     , mDisplayHeight(0)
     , mTrackRate(aTrackRate)
+    , mEncodedTicks(0)
     , mVideoBitrate(0)
   {
     mLastChunk.mDuration = 0;
@@ -349,6 +350,18 @@ protected:
    */
   VideoSegment mRawSegment;
 
+  /**
+   * The number of mTrackRate ticks we have passed to the encoder.
+   * Only accessed in AppendVideoSegment().
+   */
+  StreamTime mEncodedTicks;
+
+  /**
+   * The time of the first real video frame passed to the encoder.
+   * Only accessed in AppendVideoSegment().
+   */
+  TimeStamp mStartOffset;
+
   uint32_t mVideoBitrate;
 };