diff --git a/content/html/content/src/TimeRanges.cpp b/content/html/content/src/TimeRanges.cpp
index dfe1ed8cc42c..299a436e1a13 100644
--- a/content/html/content/src/TimeRanges.cpp
+++ b/content/html/content/src/TimeRanges.cpp
@@ -126,6 +126,17 @@ TimeRanges::Normalize()
   }
 }
 
+TimeRanges::index_type
+TimeRanges::Find(double aTime)
+{
+  for (index_type i = 0; i < mRanges.Length(); ++i) {
+    if (aTime >= mRanges[i].mStart && aTime < mRanges[i].mEnd) {
+      return i;
+    }
+  }
+  return NoIndex;
+}
+
 JSObject*
 TimeRanges::WrapObject(JSContext* aCx)
 {
diff --git a/content/html/content/src/TimeRanges.h b/content/html/content/src/TimeRanges.h
index cc40c7d290e1..174e8358c6b8 100644
--- a/content/html/content/src/TimeRanges.h
+++ b/content/html/content/src/TimeRanges.h
@@ -80,6 +80,12 @@ private:
   };
 
   nsAutoTArray<TimeRange,4> mRanges;
+
+public:
+  typedef nsTArray<TimeRange>::index_type index_type;
+  static const index_type NoIndex = index_type(-1);
+
+  index_type Find(double aTime);
 };
 
 } // namespace dom
diff --git a/content/media/mediasource/MediaSourceDecoder.cpp b/content/media/mediasource/MediaSourceDecoder.cpp
index e222fa961702..e61e4ba0ad55 100644
--- a/content/media/mediasource/MediaSourceDecoder.cpp
+++ b/content/media/mediasource/MediaSourceDecoder.cpp
@@ -44,12 +44,6 @@ extern PRLogModuleInfo* GetMediaSourceAPILog();
 
 namespace mozilla {
 
-namespace dom {
-
-class TimeRanges;
-
-} // namespace dom
-
 class MediaSourceReader : public MediaDecoderReader
 {
 public:
@@ -206,17 +200,25 @@ private:
 
     for (uint32_t i = mActiveVideoDecoder + 1; i < mDecoders.Length(); ++i) {
       SubBufferDecoder* decoder = mDecoders[i];
+
+      nsRefPtr<dom::TimeRanges> ranges = new dom::TimeRanges();
+      decoder->GetBuffered(ranges);
+
       MSE_DEBUGV("MediaDecoderReader(%p)::SwitchVideoReaders(%d) decoder=%u (%p) discarded=%d"
-                 " hasVideo=%d timeThreshold=%lld startTime=%lld",
+                 " hasVideo=%d timeThreshold=%f startTime=%f endTime=%f length=%u",
                  this, aType, i, decoder, decoder->IsDiscarded(),
                  decoder->GetReader()->GetMediaInfo().HasVideo(),
-                 mTimeThreshold, decoder->GetMediaStartTime());
+                 double(mTimeThreshold) / USECS_PER_S,
+                 ranges->GetStartTime(), ranges->GetEndTime(), ranges->Length());
 
-      if (decoder->IsDiscarded() || !decoder->GetReader()->GetMediaInfo().HasVideo()) {
+      if (decoder->IsDiscarded() ||
+          !decoder->GetReader()->GetMediaInfo().HasVideo() ||
+          ranges->Length() == 0) {
         continue;
       }
 
-      if (aType == SWITCH_FORCED || mTimeThreshold >= decoder->GetMediaStartTime()) {
+      if (aType == SWITCH_FORCED ||
+          ranges->Find(double(mTimeThreshold) / USECS_PER_S) != dom::TimeRanges::NoIndex) {
         GetVideoReader()->SetIdle();
 
         mActiveVideoDecoder = i;
@@ -423,27 +425,22 @@ MediaSourceReader::InitializePendingDecoders()
     MediaInfo mi;
     nsAutoPtr<MetadataTags> tags; // TODO: Handle metadata.
     nsresult rv;
-    int64_t startTime = INT64_MIN;
     {
       ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor());
       rv = reader->ReadMetadata(&mi, getter_Transfers(tags));
-      if (NS_SUCCEEDED(rv)) {
-        reader->FindStartTime(startTime);
-      }
     }
     reader->SetIdle();
-    if (NS_FAILED(rv) || startTime == INT64_MIN) {
+    if (NS_FAILED(rv)) {
       // XXX: Need to signal error back to owning SourceBuffer.
-      MSE_DEBUG("MediaSourceReader(%p): Reader %p failed to initialize rv=%x startTime=%lld",
-                this, reader, rv, startTime);
+      MSE_DEBUG("MediaSourceReader(%p): Reader %p failed to initialize rv=%x",
+                this, reader, rv);
       continue;
     }
-    decoder->SetMediaStartTime(startTime);
 
     bool active = false;
     if (mi.HasVideo() || mi.HasAudio()) {
-      MSE_DEBUG("MediaSourceReader(%p): Reader %p has video=%d audio=%d startTime=%lld",
-                this, reader, mi.HasVideo(), mi.HasAudio(), startTime);
+      MSE_DEBUG("MediaSourceReader(%p): Reader %p has video=%d audio=%d",
+                this, reader, mi.HasVideo(), mi.HasAudio());
       if (mi.HasVideo()) {
         MSE_DEBUG("MediaSourceReader(%p): Reader %p video resolution=%dx%d",
                   this, reader, mi.mVideo.mDisplay.width, mi.mVideo.mDisplay.height);
diff --git a/content/media/mediasource/SubBufferDecoder.h b/content/media/mediasource/SubBufferDecoder.h
index 070b8710d1d2..67fc33a724cb 100644
--- a/content/media/mediasource/SubBufferDecoder.h
+++ b/content/media/mediasource/SubBufferDecoder.h
@@ -29,7 +29,7 @@ public:
   // of the caller to manage the memory of the MediaResource object.
   SubBufferDecoder(MediaResource* aResource, MediaSourceDecoder* aParentDecoder)
     : BufferDecoder(aResource), mParentDecoder(aParentDecoder), mReader(nullptr)
-    , mMediaDuration(-1), mMediaStartTime(0), mDiscarded(false)
+    , mMediaDuration(-1), mDiscarded(false)
   {
   }
 
@@ -69,16 +69,6 @@ public:
     return mMediaDuration;
   }
 
-  int64_t GetMediaStartTime()
-  {
-    return mMediaStartTime;
-  }
-
-  void SetMediaStartTime(int64_t aMediaStartTime)
-  {
-    mMediaStartTime = aMediaStartTime;
-  }
-
   bool IsDiscarded()
   {
     return mDiscarded;
@@ -94,7 +84,6 @@ private:
   MediaSourceDecoder* mParentDecoder;
   nsRefPtr<MediaDecoderReader> mReader;
   int64_t mMediaDuration;
-  int64_t mMediaStartTime;
   bool mDiscarded;
 };