Bug 1803025 - part3 : shutdown task queue and engine stream earlier. r=jolin

This patch aims to solve the shutdown hang on the debug build on the try
server, which was caused by the media engine calling media source's
shutdown too late, so the task queue held by the engine stream and the
media source was still alive and blocked the shutdown process. [1]

Therefore, we need to shutdown the task queue earlier to order to avoid
this kind of race.

[1] https://bugzilla.mozilla.org/show_bug.cgi?id=1803025#c0

Differential Revision: https://phabricator.services.mozilla.com/D164386

dom/media/ipc/MFMediaEngineParent.cpp

@@ -90,7 +90,10 @@ void MFMediaEngineParent::DestroyEngineIfExists(
   ENGINE_MARKER("MFMediaEngineParent::DestroyEngineIfExists");
   mMediaEngineNotify = nullptr;
   mMediaEngineExtension = nullptr;
-  mMediaSource = nullptr;
+  if (mMediaSource) {
+    mMediaSource->ShutdownTaskQueue();
+    mMediaSource = nullptr;
+  }
   if (mMediaEngine) {
     mMediaEngine->Shutdown();
     mMediaEngine = nullptr;

dom/media/platforms/wmf/MFMediaEngineStream.cpp

@@ -112,6 +112,7 @@ HRESULT MFMediaEngineStream::RuntimeClassInitialize(
     uint64_t aStreamId, const TrackInfo& aInfo, MFMediaSource* aParentSource) {
   mParentSource = aParentSource;
   mTaskQueue = aParentSource->GetTaskQueue();
+  MOZ_ASSERT(mTaskQueue);
   mStreamId = aStreamId;
   RETURN_IF_FAILED(wmf::MFCreateEventQueue(&mMediaEventQueue));
 
@@ -142,8 +143,12 @@ HRESULT MFMediaEngineStream::Start(const PROPVARIANT* aPosition) {
     SLOG("No need to start non-selected stream");
     return S_OK;
   }
+  if (IsShutdown()) {
+    return MF_E_SHUTDOWN;
+  }
   SLOG("Start");
   RETURN_IF_FAILED(QueueEvent(MEStreamStarted, GUID_NULL, S_OK, aPosition));
+  MOZ_ASSERT(mTaskQueue);
   Unused << mTaskQueue->Dispatch(NS_NewRunnableFunction(
       "MFMediaEngineStream::Start", [self = RefPtr{this}, aPosition, this]() {
         if (const bool isFromCurrentPosition = aPosition->vt == VT_EMPTY;
@@ -203,12 +208,14 @@ void MFMediaEngineStream::Shutdown() {
   // MF_E_SHUTDOWN.
   RETURN_VOID_IF_FAILED(mMediaEventQueue->Shutdown());
   ComPtr<MFMediaEngineStream> self = this;
+  MOZ_ASSERT(mTaskQueue);
   Unused << mTaskQueue->Dispatch(
       NS_NewRunnableFunction("MFMediaEngineStream::Shutdown", [self]() {
         self->mParentSource = nullptr;
         self->mRawDataQueueForFeedingEngine.Reset();
         self->mRawDataQueueForGeneratingOutput.Reset();
         self->ShutdownCleanUpOnTaskQueue();
+        self->mTaskQueue = nullptr;
       }));
 }
 
@@ -247,6 +254,7 @@ IFACEMETHODIMP MFMediaEngineStream::RequestSample(IUnknown* aToken) {
 
   ComPtr<IUnknown> token = aToken;
   ComPtr<MFMediaEngineStream> self = this;
+  MOZ_ASSERT(mTaskQueue);
   Unused << mTaskQueue->Dispatch(NS_NewRunnableFunction(
       "MFMediaEngineStream::RequestSample", [token, self, this]() {
         AssertOnTaskQueue();
@@ -482,14 +490,16 @@ RefPtr<MediaDataDecoder::DecodePromise> MFMediaEngineStream::Drain() {
 }
 
 void MFMediaEngineStream::AssertOnTaskQueue() const {
-  MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
+  MOZ_ASSERT(mTaskQueue && mTaskQueue->IsCurrentThreadIn());
 }
 
 void MFMediaEngineStream::AssertOnMFThreadPool() const {
   // We can't really assert the thread id from thread pool, because it would
   // change any time. So we just assert this is not the task queue, and use the
   // explicit function name to indicate what thread we should run on.
-  MOZ_ASSERT(!mTaskQueue->IsCurrentThreadIn());
+  // TODO : this assertion is not precise, because the running thread could be
+  // the stream wrapper thread as well,
+  MOZ_ASSERT(!mTaskQueue || !mTaskQueue->IsCurrentThreadIn());
 }
 
 #undef WLOGV

dom/media/platforms/wmf/MFMediaSource.cpp

@@ -275,7 +275,7 @@ IFACEMETHODIMP MFMediaSource::Pause() {
 }
 
 IFACEMETHODIMP MFMediaSource::Shutdown() {
-  AssertOnManagerThread();
+  // Could be called on either manager thread or MF thread pool.
   MutexAutoLock lock(mMutex);
   if (mState == State::Shutdowned) {
     return MF_E_SHUTDOWN;
@@ -285,6 +285,15 @@ IFACEMETHODIMP MFMediaSource::Shutdown() {
   // After this method is called, all IMFMediaEventQueue methods return
   // MF_E_SHUTDOWN.
   RETURN_IF_FAILED(mMediaEventQueue->Shutdown());
+  mState = State::Shutdowned;
+  LOG("Shutdowned media source");
+  return S_OK;
+}
+
+void MFMediaSource::ShutdownTaskQueue() {
+  AssertOnManagerThread();
+  LOG("ShutdownTaskQueue");
+  MutexAutoLock lock(mMutex);
   if (mAudioStream) {
     mAudioStream->Shutdown();
     mAudioStream = nullptr;
@@ -295,11 +304,8 @@ IFACEMETHODIMP MFMediaSource::Shutdown() {
     mVideoStream = nullptr;
     mVideoStreamEndedListener.DisconnectIfExists();
   }
-
-  mState = State::Shutdowned;
   Unused << mTaskQueue->BeginShutdown();
-  LOG("Shutdowned media source");
-  return S_OK;
+  mTaskQueue = nullptr;
 }
 
 IFACEMETHODIMP MFMediaSource::GetEvent(DWORD aFlags, IMFMediaEvent** aEvent) {

dom/media/platforms/wmf/MFMediaSource.h

@@ -113,6 +113,8 @@ class MFMediaSource
 
   void SetDCompSurfaceHandle(HANDLE aDCompSurfaceHandle);
 
+  void ShutdownTaskQueue();
+
  private:
   void AssertOnManagerThread() const;
   void AssertOnMFThreadPool() const;