Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2025-02-17 14:25:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1758789 - part3 : create the media engine. r=jolin

Browse Source 
Create a media engine that would be responsible to play media. We would perform any commands sent from the content process on the media engine, and also notify the content process when the media engine updates its status, notifying some useful events, or encountering any error.

Differential Revision: https://phabricator.services.mozilla.com/D144634
This commit is contained in:
alwu 2022-06-30 22:34:16 +00:00
parent 5176904d9d
commit 9ff25b7c78
17 changed files with 665 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dom/media/ExternalEngineStateMachine.h
View File

@ -81,6 +81,12 @@ class ExternalEngineStateMachine final
"ExternalEngineStateMachine::NotifyEvent",
[self = RefPtr{this}, aEvent] { self->NotifyEventInternal(aEvent); }));
}
void NotifyError(const MediaResult& aError) {
// On the engine manager thread.
Unused << OwnerThread()->Dispatch(NS_NewRunnableFunction(
"ExternalEngineStateMachine::NotifyError",
[self = RefPtr{this}, aError] { self->DecodeError(aError); }));
}
const char* GetStateStr() const;

68
dom/media/ipc/MFMediaEngineChild.cpp
View File

@ -20,6 +20,11 @@ namespace mozilla {
("MFMediaEngineWrapper=%p, Id=%" PRId64 ", " msg, this, this->Id(), \
##__VA_ARGS__))
#define WLOGV(msg, ...) \
MOZ_LOG(gMFMediaEngineLog, LogLevel::Verbose, \
("MFMediaEngineWrapper=%p, Id=%" PRId64 ", " msg, this, this->Id(), \
##__VA_ARGS__))
using media::TimeUnit;
MFMediaEngineChild::MFMediaEngineChild(MFMediaEngineWrapper* aOwner)
@ -88,15 +93,18 @@ RefPtr<GenericNonExclusivePromise> MFMediaEngineChild::Init(
return p;
}
mozilla::ipc::IPCResult MFMediaEngineChild::RecvRequestSample(TrackType aType) {
mozilla::ipc::IPCResult MFMediaEngineChild::RecvRequestSample(TrackType aType,
bool aIsEnough) {
AssertOnManagerThread();
if (!mOwner) {
return IPC_OK();
}
if (aType == TrackType::kVideoTrack) {
mOwner->NotifyEvent(ExternalEngineEvent::RequestForVideo);
mOwner->NotifyEvent(aIsEnough ? ExternalEngineEvent::VideoEnough
: ExternalEngineEvent::RequestForVideo);
} else if (aType == TrackType::kAudioTrack) {
mOwner->NotifyEvent(ExternalEngineEvent::RequestForAudio);
mOwner->NotifyEvent(aIsEnough ? ExternalEngineEvent::AudioEnough
: ExternalEngineEvent::RequestForAudio);
}
return IPC_OK();
}
@ -110,6 +118,50 @@ mozilla::ipc::IPCResult MFMediaEngineChild::RecvUpdateCurrentTime(
return IPC_OK();
}
mozilla::ipc::IPCResult MFMediaEngineChild::RecvNotifyEvent(
MFMediaEngineEvent aEvent) {
AssertOnManagerThread();
switch (aEvent) {
case MF_MEDIA_ENGINE_EVENT_FIRSTFRAMEREADY:
mOwner->NotifyEvent(ExternalEngineEvent::LoadedFirstFrame);
break;
case MF_MEDIA_ENGINE_EVENT_LOADEDDATA:
mOwner->NotifyEvent(ExternalEngineEvent::LoadedData);
break;
case MF_MEDIA_ENGINE_EVENT_WAITING:
mOwner->NotifyEvent(ExternalEngineEvent::Waiting);
break;
case MF_MEDIA_ENGINE_EVENT_SEEKED:
mOwner->NotifyEvent(ExternalEngineEvent::Seeked);
break;
case MF_MEDIA_ENGINE_EVENT_BUFFERINGSTARTED:
mOwner->NotifyEvent(ExternalEngineEvent::BufferingStarted);
break;
case MF_MEDIA_ENGINE_EVENT_BUFFERINGENDED:
mOwner->NotifyEvent(ExternalEngineEvent::BufferingEnded);
break;
case MF_MEDIA_ENGINE_EVENT_ENDED:
mOwner->NotifyEvent(ExternalEngineEvent::Ended);
break;
case MF_MEDIA_ENGINE_EVENT_PLAYING:
mOwner->NotifyEvent(ExternalEngineEvent::Playing);
break;
default:
NS_WARNING(
nsPrintfCString("Unhandled event=%s", MediaEngineEventToStr(aEvent))
.get());
break;
}
return IPC_OK();
}
mozilla::ipc::IPCResult MFMediaEngineChild::RecvNotifyError(
const MediaResult& aError) {
AssertOnManagerThread();
mOwner->NotifyError(aError);
return IPC_OK();
}
void MFMediaEngineChild::OwnerDestroyed() {
Unused << ManagerThread()->Dispatch(NS_NewRunnableFunction(
"MFMediaEngineChild::OwnerDestroy", [self = RefPtr{this}, this] {
@ -229,15 +281,21 @@ TimeUnit MFMediaEngineWrapper::GetCurrentPosition() {
void MFMediaEngineWrapper::UpdateCurrentTime(double aCurrentTimeInSecond) {
AssertOnManagerThread();
WLOG("Update current time %f", aCurrentTimeInSecond);
WLOGV("Update current time %f", aCurrentTimeInSecond);
mCurrentTimeInSecond = aCurrentTimeInSecond;
NotifyEvent(ExternalEngineEvent::Timeupdate);
}
void MFMediaEngineWrapper::NotifyEvent(ExternalEngineEvent aEvent) {
AssertOnManagerThread();
WLOG("Received event %s", ExternalEngineEventToStr(aEvent));
WLOGV("Received event %s", ExternalEngineEventToStr(aEvent));
mOwner->NotifyEvent(aEvent);
}
void MFMediaEngineWrapper::NotifyError(const MediaResult& aError) {
AssertOnManagerThread();
WLOG("Received error: %s", aError.Description().get());
mOwner->NotifyError(aError);
}
} // namespace mozilla

7
dom/media/ipc/MFMediaEngineChild.h
View File

@ -6,6 +6,7 @@
#define DOM_MEDIA_IPC_MFMEDIAENGINECHILD_H_
#include "ExternalEngineStateMachine.h"
#include "MFMediaEngineUtils.h"
#include "TimeUnits.h"
#include "mozilla/Atomics.h"
#include "mozilla/PMFMediaEngineChild.h"
@ -32,8 +33,11 @@ class MFMediaEngineChild final : public PMFMediaEngineChild {
RefPtr<GenericNonExclusivePromise> Init(bool aShouldPreload);
// Methods for PMFMediaEngineChild
mozilla::ipc::IPCResult RecvRequestSample(TrackInfo::TrackType aType);
mozilla::ipc::IPCResult RecvRequestSample(TrackInfo::TrackType aType,
bool aIsEnough);
mozilla::ipc::IPCResult RecvUpdateCurrentTime(double aCurrentTimeInSecond);
mozilla::ipc::IPCResult RecvNotifyEvent(MFMediaEngineEvent aEvent);
mozilla::ipc::IPCResult RecvNotifyError(const MediaResult& aError);
nsISerialEventTarget* ManagerThread() { return mManagerThread; }
void AssertOnManagerThread() const {
@ -94,6 +98,7 @@ class MFMediaEngineWrapper final : public ExternalPlaybackEngine {
bool IsInited() const { return mEngine->Id() != 0; }
void UpdateCurrentTime(double aCurrentTimeInSecond);
void NotifyEvent(ExternalEngineEvent aEvent);
void NotifyError(const MediaResult& aError);
const RefPtr<MFMediaEngineChild> mEngine;

252
dom/media/ipc/MFMediaEngineParent.cpp
View File

@ -4,13 +4,22 @@
#include "MFMediaEngineParent.h"
#include "MFMediaEngineAudioStream.h"
#include <audiosessiontypes.h>
#include <intsafe.h>
#include <mfapi.h>
#include "MFMediaEngineExtension.h"
#include "MFMediaEngineUtils.h"
#include "MFMediaEngineVideoStream.h"
#include "MFMediaEngineStream.h"
#include "MFMediaSource.h"
#include "RemoteDecoderManagerParent.h"
#include "WMF.h"
#include "mozilla/ClearOnShutdown.h"
#include "mozilla/ScopeExit.h"
#include "mozilla/StaticMutex.h"
#include "mozilla/StaticPrefs_media.h"
#include "mozilla/StaticPtr.h"
#include "mozilla/WindowsVersion.h"
namespace mozilla {
@ -22,6 +31,9 @@ namespace mozilla {
using MediaEngineMap = nsTHashMap<nsUint64HashKey, MFMediaEngineParent*>;
static StaticAutoPtr<MediaEngineMap> sMediaEngines;
using Microsoft::WRL::ComPtr;
using Microsoft::WRL::MakeAndInitialize;
StaticMutex sMediaEnginesLock;
static void RegisterMediaEngine(MFMediaEngineParent* aMediaEngine) {
@ -46,35 +58,191 @@ MFMediaEngineParent* MFMediaEngineParent::GetMediaEngineById(uint64_t aId) {
return sMediaEngines->Get(aId);
}
MFMediaEngineParent::MFMediaEngineParent(RemoteDecoderManagerParent* aManager)
: mMediaEngineId(++sMediaEngineIdx), mManager(aManager) {
MFMediaEngineParent::MFMediaEngineParent(RemoteDecoderManagerParent* aManager,
nsISerialEventTarget* aManagerThread)
: mMediaEngineId(++sMediaEngineIdx),
mManager(aManager),
mManagerThread(aManagerThread) {
MOZ_ASSERT(aManager);
MOZ_ASSERT(aManagerThread);
MOZ_ASSERT(mMediaEngineId != 0);
MOZ_ASSERT(XRE_IsRDDProcess());
LOG("Created MFMediaEngineParent");
RegisterMediaEngine(this);
mIPDLSelfRef = this;
CreateMediaEngine();
}
MFMediaEngineParent::~MFMediaEngineParent() {
LOG("Destoryed MFMediaEngineParent");
DestroyEngineIfExists();
UnregisterMedieEngine(this);
}
MFMediaEngineStream* MFMediaEngineParent::GetMediaEngineStream(
void MFMediaEngineParent::DestroyEngineIfExists(
const Maybe<MediaResult>& aError) {
LOG("DestroyEngineIfExists, hasError=%d", aError.isSome());
mMediaEngineNotify = nullptr;
mMediaEngineExtension = nullptr;
mMediaSource = nullptr;
if (mMediaEngine) {
mMediaEngine->Shutdown();
mMediaEngine = nullptr;
}
mMediaEngineEventListener.DisconnectIfExists();
mRequestSampleListener.DisconnectIfExists();
if (aError) {
Unused << SendNotifyError(*aError);
}
}
void MFMediaEngineParent::CreateMediaEngine() {
LOG("CreateMediaEngine");
auto errorExit = MakeScopeExit([&] {
MediaResult error(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Failed to create engine");
DestroyEngineIfExists(Some(error));
});
if (!wmf::MediaFoundationInitializer::HasInitialized()) {
NS_WARNING("Failed to initialize media foundation");
return;
}
// TODO : init DXGIDeviceManager and a virtual window when media has video.
// Create an attribute and set mandatory information that are required for a
// media engine creation.
ComPtr<IMFAttributes> creationAttributes;
RETURN_VOID_IF_FAILED(wmf::MFCreateAttributes(&creationAttributes, 6));
RETURN_VOID_IF_FAILED(
MakeAndInitialize<MFMediaEngineNotify>(&mMediaEngineNotify));
mMediaEngineEventListener = mMediaEngineNotify->MediaEngineEvent().Connect(
mManagerThread, this, &MFMediaEngineParent::HandleMediaEngineEvent);
RETURN_VOID_IF_FAILED(creationAttributes->SetUnknown(
MF_MEDIA_ENGINE_CALLBACK, mMediaEngineNotify.Get()));
RETURN_VOID_IF_FAILED(creationAttributes->SetUINT32(
MF_MEDIA_ENGINE_AUDIO_CATEGORY, AudioCategory_Media));
RETURN_VOID_IF_FAILED(
MakeAndInitialize<MFMediaEngineExtension>(&mMediaEngineExtension));
RETURN_VOID_IF_FAILED(creationAttributes->SetUnknown(
MF_MEDIA_ENGINE_EXTENSION, mMediaEngineExtension.Get()));
// TODO : SET MF_MEDIA_ENGINE_CONTENT_PROTECTION_FLAGS
ComPtr<IMFMediaEngineClassFactory> factory;
RETURN_VOID_IF_FAILED(CoCreateInstance(CLSID_MFMediaEngineClassFactory,
nullptr, CLSCTX_INPROC_SERVER,
IID_PPV_ARGS(&factory)));
const bool isLowLatency =
(StaticPrefs::media_wmf_low_latency_enabled() || IsWin10OrLater()) &&
!StaticPrefs::media_wmf_low_latency_force_disabled();
static const DWORD MF_MEDIA_ENGINE_DEFAULT = 0;
RETURN_VOID_IF_FAILED(factory->CreateInstance(
isLowLatency ? MF_MEDIA_ENGINE_REAL_TIME_MODE : MF_MEDIA_ENGINE_DEFAULT,
creationAttributes.Get(), &mMediaEngine));
// TODO : set DComp mode for video
// TODO : deal with encrypted content (set ContentProtectionManager and cdm
// proxy)
LOG("Created media engine successfully");
mIsCreatedMediaEngine = true;
errorExit.release();
}
void MFMediaEngineParent::HandleMediaEngineEvent(
MFMediaEngineEventWrapper aEvent) {
AssertOnManagerThread();
switch (aEvent.mEvent) {
case MF_MEDIA_ENGINE_EVENT_ERROR: {
MOZ_ASSERT(aEvent.mParam1 && aEvent.mParam2);
auto error = static_cast<MF_MEDIA_ENGINE_ERR>(*aEvent.mParam1);
auto result = static_cast<HRESULT>(*aEvent.mParam2);
NotifyError(error, result);
break;
}
case MF_MEDIA_ENGINE_EVENT_FIRSTFRAMEREADY:
case MF_MEDIA_ENGINE_EVENT_LOADEDDATA:
case MF_MEDIA_ENGINE_EVENT_WAITING:
case MF_MEDIA_ENGINE_EVENT_SEEKED:
case MF_MEDIA_ENGINE_EVENT_BUFFERINGSTARTED:
case MF_MEDIA_ENGINE_EVENT_BUFFERINGENDED:
case MF_MEDIA_ENGINE_EVENT_ENDED:
case MF_MEDIA_ENGINE_EVENT_PLAYING:
Unused << SendNotifyEvent(aEvent.mEvent);
break;
case MF_MEDIA_ENGINE_EVENT_TIMEUPDATE: {
auto currentTimeInSeconds = mMediaEngine->GetCurrentTime();
Unused << SendUpdateCurrentTime(currentTimeInSeconds);
break;
}
default:
LOG("Unhandled event=%s", MediaEngineEventToStr(aEvent.mEvent));
break;
}
}
void MFMediaEngineParent::NotifyError(MF_MEDIA_ENGINE_ERR aError,
HRESULT aResult) {
// TODO : handle HRESULT 0x8004CD12, DRM_E_TEE_INVALID_HWDRM_STATE, which can
// happen during OS sleep/resume, or moving video to different graphics
// adapters.
if (aError == MF_MEDIA_ENGINE_ERR_NOERROR) {
return;
}
LOG("Notify error '%s', hr=%lx", MFMediaEngineErrorToStr(aError), aResult);
switch (aError) {
case MF_MEDIA_ENGINE_ERR_ABORTED:
case MF_MEDIA_ENGINE_ERR_NETWORK:
// We ignore these two because we fetch data by ourselves.
return;
case MF_MEDIA_ENGINE_ERR_DECODE: {
MediaResult error(NS_ERROR_DOM_MEDIA_DECODE_ERR, "Decoder error");
Unused << SendNotifyError(error);
return;
}
case MF_MEDIA_ENGINE_ERR_SRC_NOT_SUPPORTED: {
MediaResult error(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Source not supported");
Unused << SendNotifyError(error);
return;
}
case MF_MEDIA_ENGINE_ERR_ENCRYPTED: {
MediaResult error(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Encrypted error");
Unused << SendNotifyError(error);
return;
}
default:
MOZ_ASSERT_UNREACHABLE("Unsupported error");
return;
}
}
MFMediaEngineStreamWrapper* MFMediaEngineParent::GetMediaEngineStream(
TrackType aType, const CreateDecoderParams& aParam) {
LOG("Create a media engine decoder for %s", TrackTypeToStr(aType));
// TODO : make those streams associated with their media engine and source.
MOZ_ASSERT(mMediaSource);
if (aType == TrackType::kAudioTrack) {
return new MFMediaEngineAudioStream(aParam);
auto* stream = mMediaSource->GetAudioStream();
return new MFMediaEngineStreamWrapper(stream, stream->GetTaskQueue());
}
MOZ_ASSERT(aType == TrackType::kVideoTrack);
return new MFMediaEngineVideoStream(aParam);
auto* stream = mMediaSource->GetVideoStream();
return new MFMediaEngineStreamWrapper(stream, stream->GetTaskQueue());
}
mozilla::ipc::IPCResult MFMediaEngineParent::RecvInitMediaEngine(
const MediaEngineInfoIPDL& aInfo, InitMediaEngineResolver&& aResolver) {
AssertOnManagerThread();
if (!mIsCreatedMediaEngine) {
aResolver(0);
return IPC_OK();
}
// Metadata preload is controlled by content process side before creating
// media engine.
if (aInfo.preload()) {
// TODO : really need this?
Unused << mMediaEngine->SetPreload(MF_MEDIA_ENGINE_PRELOAD_AUTOMATIC);
}
aResolver(mMediaEngineId);
return IPC_OK();
}
@ -82,32 +250,72 @@ mozilla::ipc::IPCResult MFMediaEngineParent::RecvInitMediaEngine(
mozilla::ipc::IPCResult MFMediaEngineParent::RecvNotifyMediaInfo(
const MediaInfoIPDL& aInfo) {
AssertOnManagerThread();
// TODO : implement this by using media engine
MOZ_ASSERT(mIsCreatedMediaEngine, "Hasn't created media engine?");
MOZ_ASSERT(!mMediaSource);
LOG("RecvNotifyMediaInfo");
auto errorExit = MakeScopeExit([&] {
MediaResult error(NS_ERROR_DOM_MEDIA_FATAL_ERR,
"Failed to setup media source");
DestroyEngineIfExists(Some(error));
});
// Create media source and set it to the media engine.
NS_ENSURE_TRUE(SUCCEEDED(MakeAndInitialize<MFMediaSource>(
&mMediaSource, aInfo.audioInfo(), aInfo.videoInfo())),
IPC_OK());
mMediaEngineExtension->SetMediaSource(mMediaSource.Get());
// We use the source scheme in order to let the media engine to load our
// custom source.
NS_ENSURE_TRUE(
SUCCEEDED(mMediaEngine->SetSource(SysAllocString(L"MFRendererSrc"))),
IPC_OK());
LOG("Finished setup our custom media source to the media engine");
mRequestSampleListener = mMediaSource->RequestSampleEvent().Connect(
mManagerThread, this, &MFMediaEngineParent::HandleRequestSample);
errorExit.release();
return IPC_OK();
}
mozilla::ipc::IPCResult MFMediaEngineParent::RecvPlay() {
AssertOnManagerThread();
// TODO : implement this by using media engine
if (mMediaEngine) {
LOG("Play");
NS_ENSURE_TRUE(SUCCEEDED(mMediaEngine->Play()), IPC_OK());
}
return IPC_OK();
}
mozilla::ipc::IPCResult MFMediaEngineParent::RecvPause() {
AssertOnManagerThread();
// TODO : implement this by using media engine
if (mMediaEngine) {
LOG("Pause");
NS_ENSURE_TRUE(SUCCEEDED(mMediaEngine->Pause()), IPC_OK());
}
return IPC_OK();
}
mozilla::ipc::IPCResult MFMediaEngineParent::RecvSeek(
double aTargetTimeInSecond) {
AssertOnManagerThread();
// TODO : implement this by using media engine
if (mMediaEngine) {
LOG("Seek to %f", aTargetTimeInSecond);
NS_ENSURE_TRUE(SUCCEEDED(mMediaEngine->SetCurrentTime(aTargetTimeInSecond)),
IPC_OK());
}
return IPC_OK();
}
mozilla::ipc::IPCResult MFMediaEngineParent::RecvSetVolume(double aVolume) {
AssertOnManagerThread();
// TODO : implement this by using media engine
if (mMediaEngine) {
LOG("SetVolume=%f", aVolume);
NS_ENSURE_TRUE(SUCCEEDED(mMediaEngine->SetVolume(aVolume)), IPC_OK());
}
return IPC_OK();
}
@ -120,20 +328,24 @@ mozilla::ipc::IPCResult MFMediaEngineParent::RecvSetPlaybackRate(
mozilla::ipc::IPCResult MFMediaEngineParent::RecvSetLooping(bool aLooping) {
AssertOnManagerThread();
// TODO : implement this by using media engine
// We handle looping by seeking back to the head by ourselves, so we don't
// rely on the media engine for looping.
return IPC_OK();
}
mozilla::ipc::IPCResult MFMediaEngineParent::RecvNotifyEndOfStream(
TrackInfo::TrackType aType) {
AssertOnManagerThread();
// TODO : implement this by using media engine
MOZ_ASSERT(mMediaSource);
LOG("NotifyEndOfStream, type=%s", TrackTypeToStr(aType));
mMediaSource->NotifyEndOfStream(aType);
return IPC_OK();
}
mozilla::ipc::IPCResult MFMediaEngineParent::RecvShutdown() {
AssertOnManagerThread();
LOG("Shutdown");
DestroyEngineIfExists();
return IPC_OK();
}
@ -142,10 +354,18 @@ void MFMediaEngineParent::Destroy() {
mIPDLSelfRef = nullptr;
}
void MFMediaEngineParent::HandleRequestSample(const SampleRequest& aRequest) {
AssertOnManagerThread();
MOZ_ASSERT(aRequest.mType == TrackInfo::TrackType::kAudioTrack ||
aRequest.mType == TrackInfo::TrackType::kVideoTrack);
Unused << SendRequestSample(aRequest.mType, aRequest.mIsEnough);
}
void MFMediaEngineParent::AssertOnManagerThread() const {
MOZ_ASSERT(mManager->OnManagerThread());
MOZ_ASSERT(mManagerThread->IsOnCurrentThread());
}
#undef LOG
#undef RETURN_IF_FAILED
} // namespace mozilla

38
dom/media/ipc/MFMediaEngineParent.h
View File

@ -5,13 +5,21 @@
#ifndef DOM_MEDIA_IPC_MFMEDIAENGINEPARENT_H_
#define DOM_MEDIA_IPC_MFMEDIAENGINEPARENT_H_
#include <wrl.h>
#include "MediaInfo.h"
#include "MFMediaEngineExtra.h"
#include "MFMediaEngineNotify.h"
#include "MFMediaEngineUtils.h"
#include "MFMediaSource.h"
#include "PlatformDecoderModule.h"
#include "mozilla/PMFMediaEngineParent.h"
namespace mozilla {
class MFMediaEngineStream;
class MFMediaEngineExtension;
class MFMediaEngineStreamWrapper;
class MFMediaSource;
class RemoteDecoderManagerParent;
/**
@ -25,14 +33,15 @@ class RemoteDecoderManagerParent;
class MFMediaEngineParent final : public PMFMediaEngineParent {
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MFMediaEngineParent);
explicit MFMediaEngineParent(RemoteDecoderManagerParent* aManager);
MFMediaEngineParent(RemoteDecoderManagerParent* aManager,
nsISerialEventTarget* aManagerThread);
using TrackType = TrackInfo::TrackType;
static MFMediaEngineParent* GetMediaEngineById(uint64_t aId);
MFMediaEngineStream* GetMediaEngineStream(TrackType aType,
const CreateDecoderParams& aParam);
MFMediaEngineStreamWrapper* GetMediaEngineStream(
TrackType aType, const CreateDecoderParams& aParam);
uint64_t Id() const { return mMediaEngineId; }
@ -54,8 +63,17 @@ class MFMediaEngineParent final : public PMFMediaEngineParent {
private:
~MFMediaEngineParent();
void CreateMediaEngine();
void AssertOnManagerThread() const;
void HandleMediaEngineEvent(MFMediaEngineEventWrapper aEvent);
void HandleRequestSample(const SampleRequest& aRequest);
void NotifyError(MF_MEDIA_ENGINE_ERR aError, HRESULT aResult = 0);
void DestroyEngineIfExists(const Maybe<MediaResult>& aError = Nothing());
// This generates unique id for each MFMediaEngineParent instance, and it
// would be increased monotonically.
static inline uint64_t sMediaEngineIdx = 0;
@ -68,6 +86,18 @@ class MFMediaEngineParent final : public PMFMediaEngineParent {
RefPtr<MFMediaEngineParent> mIPDLSelfRef;
const RefPtr<RemoteDecoderManagerParent> mManager;
const RefPtr<nsISerialEventTarget> mManagerThread;
// Required classes for working with the media engine.
Microsoft::WRL::ComPtr<IMFMediaEngine> mMediaEngine;
Microsoft::WRL::ComPtr<MFMediaEngineNotify> mMediaEngineNotify;
Microsoft::WRL::ComPtr<MFMediaEngineExtension> mMediaEngineExtension;
Microsoft::WRL::ComPtr<MFMediaSource> mMediaSource;
MediaEventListener mMediaEngineEventListener;
MediaEventListener mRequestSampleListener;
MediaEventListener mTimeUpdateListener;
bool mIsCreatedMediaEngine = false;
};
} // namespace mozilla

59
dom/media/ipc/MFMediaEngineUtils.cpp
View File

@ -47,6 +47,65 @@ const char* MediaEventTypeToStr(MediaEventType aType) {
}
}
const char* MediaEngineEventToStr(MF_MEDIA_ENGINE_EVENT aEvent) {
switch (aEvent) {
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_LOADSTART);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_PROGRESS);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_SUSPEND);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_ABORT);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_ERROR);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_EMPTIED);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_STALLED);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_PLAY);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_PAUSE);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_LOADEDMETADATA);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_LOADEDDATA);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_WAITING);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_PLAYING);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_CANPLAY);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_CANPLAYTHROUGH);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_SEEKING);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_SEEKED);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_TIMEUPDATE);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_ENDED);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_RATECHANGE);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_DURATIONCHANGE);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_VOLUMECHANGE);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_FORMATCHANGE);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_PURGEQUEUEDEVENTS);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_TIMELINE_MARKER);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_BALANCECHANGE);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_DOWNLOADCOMPLETE);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_BUFFERINGSTARTED);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_BUFFERINGENDED);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_FRAMESTEPCOMPLETED);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_NOTIFYSTABLESTATE);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_FIRSTFRAMEREADY);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_TRACKSCHANGE);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_OPMINFO);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_RESOURCELOST);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_DELAYLOADEVENT_CHANGED);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_STREAMRENDERINGERROR);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_SUPPORTEDRATES_CHANGED);
ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_AUDIOENDPOINTCHANGE);
default:
return "Unknown MF_MEDIA_ENGINE_EVENT";
}
}
const char* MFMediaEngineErrorToStr(MFMediaEngineError aError) {
switch (aError) {
ENUM_TO_STR(MF_MEDIA_ENGINE_ERR_NOERROR);
ENUM_TO_STR(MF_MEDIA_ENGINE_ERR_ABORTED);
ENUM_TO_STR(MF_MEDIA_ENGINE_ERR_NETWORK);
ENUM_TO_STR(MF_MEDIA_ENGINE_ERR_DECODE);
ENUM_TO_STR(MF_MEDIA_ENGINE_ERR_SRC_NOT_SUPPORTED);
ENUM_TO_STR(MF_MEDIA_ENGINE_ERR_ENCRYPTED);
default:
return "Unknown MFMediaEngineError";
}
}
const char* GUIDToStr(GUID aGUID) {
ENUM_TO_STR2(aGUID, MFAudioFormat_MP3)
ENUM_TO_STR2(aGUID, MFAudioFormat_AAC)

21
dom/media/ipc/MFMediaEngineUtils.h
View File

@ -49,6 +49,8 @@ using MFMediaEngineError = MF_MEDIA_ENGINE_ERR;
#endif
const char* MediaEventTypeToStr(MediaEventType aType);
const char* MediaEngineEventToStr(MF_MEDIA_ENGINE_EVENT aEvent);
const char* MFMediaEngineErrorToStr(MFMediaEngineError aError);
const char* GUIDToStr(GUID aGUID);
const char* MFVideoRotationFormatToStr(MFVideoRotationFormat aFormat);
const char* MFVideoTransferFunctionToStr(MFVideoTransferFunction aFunc);
@ -56,4 +58,23 @@ const char* MFVideoPrimariesToStr(MFVideoPrimaries aPrimaries);
} // namespace mozilla
namespace IPC {
template <>
struct ParamTraits<mozilla::MFMediaEngineError>
: public ContiguousEnumSerializerInclusive<
mozilla::MFMediaEngineError,
mozilla::MFMediaEngineError::MF_MEDIA_ENGINE_ERR_ABORTED,
mozilla::MFMediaEngineError::MF_MEDIA_ENGINE_ERR_ENCRYPTED> {};
template <>
struct ParamTraits<mozilla::MFMediaEngineEvent>
: public ContiguousEnumSerializerInclusive<
mozilla::MFMediaEngineEvent,
mozilla::MFMediaEngineEvent::MF_MEDIA_ENGINE_EVENT_LOADSTART,
mozilla::MFMediaEngineEvent::
MF_MEDIA_ENGINE_EVENT_AUDIOENDPOINTCHANGE> {};
} // namespace IPC
#endif // DOM_MEDIA_IPC_MFMEDIAENGINECHILD_H_

6
dom/media/ipc/PMFMediaEngine.ipdl
View File

@ -10,6 +10,8 @@ using mozilla::AudioInfo from "MediaInfo.h";
using mozilla::VideoInfo from "MediaInfo.h";
using mozilla::MediaResult from "MediaResult.h";
using mozilla::TrackInfo::TrackType from "MediaInfo.h";
using mozilla::MFMediaEngineError from "MFMediaEngineUtils.h";
using mozilla::MFMediaEngineEvent from "MFMediaEngineUtils.h";
namespace mozilla {
@ -43,8 +45,10 @@ parent:
async __delete__();
child:
async NotifyEvent(MFMediaEngineEvent event);
async NotifyError(MediaResult error);
async UpdateCurrentTime(double currentTimeInSecond);
async RequestSample(TrackType type);
async RequestSample(TrackType type, bool isEnough);
};
} // namespace mozilla

1
dom/media/ipc/PRemoteDecoderManager.ipdl
View File

@ -9,6 +9,7 @@
include protocol PTexture;
include protocol PRemoteDecoder;
include LayersSurfaces;
include PMediaDecoderParams;
include "mozilla/dom/MediaIPCUtils.h";
include "mozilla/layers/LayersMessageUtils.h";

2
dom/media/ipc/RemoteDecoderManagerParent.cpp
View File

@ -226,7 +226,7 @@ bool RemoteDecoderManagerParent::DeallocPRemoteDecoderParent(
PMFMediaEngineParent* RemoteDecoderManagerParent::AllocPMFMediaEngineParent() {
#ifdef MOZ_WMF
return new MFMediaEngineParent(this);
return new MFMediaEngineParent(this, sRemoteDecoderManagerParentThread);
#else
return nullptr;
#endif

84
dom/media/platforms/wmf/MFMediaEngineExtension.cpp Normal file
View File

@ -0,0 +1,84 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "MFMediaEngineExtension.h"
#include "MFMediaSource.h"
#include "MFMediaEngineUtils.h"
namespace mozilla {
#define LOG(msg, ...) \
MOZ_LOG(gMFMediaEngineLog, LogLevel::Debug, \
("MFMediaEngineExtension=%p, " msg, this, ##__VA_ARGS__))
using Microsoft::WRL::ComPtr;
void MFMediaEngineExtension::SetMediaSource(IMFMediaSource* aMediaSource) {
LOG("SetMediaSource=%p", aMediaSource);
mMediaSource = aMediaSource;
}
// https://docs.microsoft.com/en-us/windows/win32/api/mfmediaengine/nf-mfmediaengine-imfmediaengineextension-begincreateobject
IFACEMETHODIMP MFMediaEngineExtension::BeginCreateObject(
BSTR aUrl, IMFByteStream* aByteStream, MF_OBJECT_TYPE aType,
IUnknown** aCancelCookie, IMFAsyncCallback* aCallback, IUnknown* aState) {
if (aCancelCookie) {
// We don't support a cancel cookie.
*aCancelCookie = nullptr;
}
if (aType != MF_OBJECT_MEDIASOURCE) {
LOG("Only support media source type");
return MF_E_UNEXPECTED;
}
MOZ_ASSERT(mMediaSource);
ComPtr<IMFAsyncResult> result;
ComPtr<IUnknown> sourceUnknown = mMediaSource;
RETURN_IF_FAILED(wmf::MFCreateAsyncResult(sourceUnknown.Get(), aCallback,
aState, &result));
RETURN_IF_FAILED(result->SetStatus(S_OK));
LOG("Creating object");
mIsObjectCreating = true;
RETURN_IF_FAILED(aCallback->Invoke(result.Get()));
return S_OK;
}
IFACEMETHODIMP MFMediaEngineExtension::CancelObjectCreation(
IUnknown* aCancelCookie) {
return MF_E_UNEXPECTED;
}
IFACEMETHODIMP MFMediaEngineExtension::EndCreateObject(IMFAsyncResult* aResult,
IUnknown** aRetObj) {
*aRetObj = nullptr;
if (!mIsObjectCreating) {
LOG("No object is creating, not an expected call");
return MF_E_UNEXPECTED;
}
RETURN_IF_FAILED(aResult->GetStatus());
RETURN_IF_FAILED(aResult->GetObject(aRetObj));
LOG("End of creating object");
mIsObjectCreating = false;
return S_OK;
}
IFACEMETHODIMP MFMediaEngineExtension::CanPlayType(
BOOL aIsAudioOnly, BSTR aMimeType, MF_MEDIA_ENGINE_CANPLAY* aResult) {
// We use MF_MEDIA_ENGINE_EXTENSION to resolve as custom media source for
// MFMediaEngine, MIME types are not used.
*aResult = MF_MEDIA_ENGINE_CANPLAY_NOT_SUPPORTED;
return S_OK;
}
// TODO : break cycle of mMediaSource
#undef LOG
} // namespace mozilla

49
dom/media/platforms/wmf/MFMediaEngineExtension.h Normal file
View File

@ -0,0 +1,49 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINEEXTENSION_H
#define DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINEEXTENSION_H
#include <wrl.h>
#include "MFMediaEngineExtra.h"
namespace mozilla {
/**
* MFMediaEngineNotify is used to load media resources in the media engine.
* https://docs.microsoft.com/en-us/windows/win32/api/mfmediaengine/nn-mfmediaengine-imfmediaengineextension
*/
class MFMediaEngineExtension final
: public Microsoft::WRL::RuntimeClass<
Microsoft::WRL::RuntimeClassFlags<
Microsoft::WRL::RuntimeClassType::ClassicCom>,
IMFMediaEngineExtension> {
public:
MFMediaEngineExtension() = default;
HRESULT RuntimeClassInitialize() { return S_OK; }
void SetMediaSource(IMFMediaSource* aMediaSource);
// Method for MFMediaEngineExtension
IFACEMETHODIMP BeginCreateObject(BSTR aUrl, IMFByteStream* aByteStream,
MF_OBJECT_TYPE aType,
IUnknown** aCancelCookie,
IMFAsyncCallback* aCallback,
IUnknown* aState) override;
IFACEMETHODIMP CancelObjectCreation(IUnknown* aCancelCookie) override;
IFACEMETHODIMP EndCreateObject(IMFAsyncResult* aResult,
IUnknown** aRetObj) override;
IFACEMETHODIMP CanPlayType(BOOL aIsAudioOnly, BSTR aMimeType,
MF_MEDIA_ENGINE_CANPLAY* aResult) override;
private:
bool mIsObjectCreating = false;
Microsoft::WRL::ComPtr<IMFMediaSource> mMediaSource;
};
} // namespace mozilla
#endif // DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINEEXTENSION_H

32
dom/media/platforms/wmf/MFMediaEngineNotify.cpp Normal file
View File

@ -0,0 +1,32 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "MFMediaEngineNotify.h"
#include "MFMediaEngineUtils.h"
namespace mozilla {
#define LOG(msg, ...) \
MOZ_LOG(gMFMediaEngineLog, LogLevel::Debug, \
("MFMediaEngineNotify=%p, " msg, this, ##__VA_ARGS__))
IFACEMETHODIMP MFMediaEngineNotify::EventNotify(DWORD aEvent, DWORD_PTR aParam1,
DWORD aParam2) {
auto event = static_cast<MF_MEDIA_ENGINE_EVENT>(aEvent);
LOG("Received media engine event %s", MediaEngineEventToStr(event));
MFMediaEngineEventWrapper engineEvent{event};
if (event == MF_MEDIA_ENGINE_EVENT_ERROR ||
event == MF_MEDIA_ENGINE_EVENT_FORMATCHANGE ||
event == MF_MEDIA_ENGINE_EVENT_NOTIFYSTABLESTATE) {
engineEvent.mParam1 = Some(aParam1);
engineEvent.mParam2 = Some(aParam2);
}
mEngineEvents.Notify(engineEvent);
return S_OK;
}
#undef LOG
} // namespace mozilla

55
dom/media/platforms/wmf/MFMediaEngineNotify.h Normal file
View File

@ -0,0 +1,55 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINENOTIFY_H
#define DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINENOTIFY_H
#include <wrl.h>
#include "MediaEventSource.h"
#include "MFMediaEngineExtra.h"
#include "mozilla/Maybe.h"
namespace mozilla {
const char* MediaEngineEventToStr(MF_MEDIA_ENGINE_EVENT aEvent);
// https://docs.microsoft.com/en-us/windows/win32/api/mfmediaengine/ne-mfmediaengine-mf_media_engine_event
struct MFMediaEngineEventWrapper final {
explicit MFMediaEngineEventWrapper(MF_MEDIA_ENGINE_EVENT aEvent)
: mEvent(aEvent) {}
MF_MEDIA_ENGINE_EVENT mEvent;
Maybe<DWORD_PTR> mParam1;
Maybe<DWORD> mParam2;
};
/**
* MFMediaEngineNotify is used to handle the event sent from the media engine.
* https://docs.microsoft.com/en-us/windows/win32/api/mfmediaengine/nn-mfmediaengine-imfmediaenginenotify
*/
class MFMediaEngineNotify final
: public Microsoft::WRL::RuntimeClass<
Microsoft::WRL::RuntimeClassFlags<
Microsoft::WRL::RuntimeClassType::ClassicCom>,
IMFMediaEngineNotify> {
public:
MFMediaEngineNotify() = default;
HRESULT RuntimeClassInitialize() { return S_OK; }
// Method for IMFMediaEngineNotify
IFACEMETHODIMP EventNotify(DWORD aEvent, DWORD_PTR aParam1,
DWORD aParam2) override;
MediaEventSource<MFMediaEngineEventWrapper>& MediaEngineEvent() {
return mEngineEvents;
}
private:
MediaEventProducer<MFMediaEngineEventWrapper> mEngineEvents;
};
} // namespace mozilla
#endif // DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINENOTIFY_H

3
dom/media/platforms/wmf/MFMediaEngineStream.cpp
View File

@ -110,8 +110,7 @@ MFMediaEngineStream::MFMediaEngineStream()
: mIsShutdown(false),
mIsSelected(false),
mReceivedEOS(false),
mShouldServeSmamples(false),
mMutex("MFMediaEngineStream Mutex") {}
mShouldServeSmamples(false) {}
MFMediaEngineStream::~MFMediaEngineStream() { MOZ_ASSERT(IsShutdown()); }

13
dom/media/platforms/wmf/MFMediaSource.h
View File

@ -83,7 +83,6 @@ class MFMediaSource : public Microsoft::WRL::RuntimeClass<
self->NotifyEndOfStreamInternal(aType);
}));
}
void NotifyEndOfStreamInternal(TrackInfo::TrackType aType);
// Called from the MF stream to indicate that the stream has provided last
// encoded sample to the media engine.
@ -102,6 +101,8 @@ class MFMediaSource : public Microsoft::WRL::RuntimeClass<
void AssertOnTaskQueue() const;
void AssertOnMFThreadPool() const;
void NotifyEndOfStreamInternal(TrackInfo::TrackType aType);
bool IsSeekable() const;
MFMediaEngineStream* GetStreamByDescriptorId(DWORD aId) const;
@ -120,12 +121,18 @@ class MFMediaSource : public Microsoft::WRL::RuntimeClass<
MediaEventListener mAudioStreamEndedListener;
MediaEventListener mVideoStreamEndedListener;
// This class would be run on two threads, MF thread pool and the source's
// task queue. Following members would be used across both threads so they
// need to be thread-safe.
// True if the playback is ended. Use and modify on both task queue and MF
// thread pool.
Atomic<bool> mPresentationEnded;
// TODO : atomic
State mState;
// Modify on MF thread pool, read on any threads.
Atomic<State> mState;
// Thread-safe members END
};
} // namespace mozilla

2
dom/media/platforms/wmf/moz.build
View File

@ -27,6 +27,8 @@ UNIFIED_SOURCES += [
"DXVA2Manager.cpp",
"MFMediaEngineAudioStream.cpp",
"MFMediaEngineDecoderModule.cpp",
"MFMediaEngineExtension.cpp",
"MFMediaEngineNotify.cpp",
"MFMediaEngineStream.cpp",
"MFMediaEngineVideoStream.cpp",
"MFMediaSource.cpp",