From e1148e1aab2f904749c3c346177d63d742cbf512 Mon Sep 17 00:00:00 2001 From: Byron Campen Date: Thu, 20 Jul 2023 14:24:27 +0000 Subject: [PATCH] Bug 1631263: Implement RTCRtpScriptTransform. r=pehrsons,jib,asuth,emilio,saschanaz Differential Revision: https://phabricator.services.mozilla.com/D179735 --- dom/events/EventNameList.h | 1 + .../webrtc/jsapi/RTCEncodedAudioFrame.cpp | 94 ++++ dom/media/webrtc/jsapi/RTCEncodedAudioFrame.h | 52 ++ .../webrtc/jsapi/RTCEncodedFrameBase.cpp | 71 +++ dom/media/webrtc/jsapi/RTCEncodedFrameBase.h | 58 +++ .../webrtc/jsapi/RTCEncodedVideoFrame.cpp | 117 +++++ dom/media/webrtc/jsapi/RTCEncodedVideoFrame.h | 61 +++ dom/media/webrtc/jsapi/RTCRtpReceiver.cpp | 136 +++++- dom/media/webrtc/jsapi/RTCRtpReceiver.h | 16 + .../webrtc/jsapi/RTCRtpScriptTransform.cpp | 84 ++++ .../webrtc/jsapi/RTCRtpScriptTransform.h | 63 +++ .../webrtc/jsapi/RTCRtpScriptTransformer.cpp | 449 ++++++++++++++++++ .../webrtc/jsapi/RTCRtpScriptTransformer.h | 197 ++++++++ dom/media/webrtc/jsapi/RTCRtpSender.cpp | 125 ++++- dom/media/webrtc/jsapi/RTCRtpSender.h | 12 + dom/media/webrtc/jsapi/RTCRtpTransceiver.cpp | 94 +++- .../jsapi/RTCTransformEventRunnable.cpp | 79 +++ .../webrtc/jsapi/RTCTransformEventRunnable.h | 38 ++ dom/media/webrtc/jsapi/moz.build | 11 + .../webrtc/libwebrtcglue/AudioConduit.cpp | 123 ++++- dom/media/webrtc/libwebrtcglue/AudioConduit.h | 6 + .../webrtc/libwebrtcglue/FrameTransformer.cpp | 87 ++++ .../webrtc/libwebrtcglue/FrameTransformer.h | 79 +++ .../libwebrtcglue/FrameTransformerProxy.cpp | 258 ++++++++++ .../libwebrtcglue/FrameTransformerProxy.h | 124 +++++ .../libwebrtcglue/MediaConduitControl.h | 5 + .../libwebrtcglue/MediaConduitInterface.h | 5 + .../webrtc/libwebrtcglue/VideoConduit.cpp | 209 +++++++- dom/media/webrtc/libwebrtcglue/VideoConduit.h | 10 + dom/media/webrtc/libwebrtcglue/moz.build | 2 + .../mochitest/general/test_interfaces.js | 6 + dom/webidl/DedicatedWorkerGlobalScope.webidl | 6 + dom/webidl/RTCEncodedAudioFrame.webidl | 24 + dom/webidl/RTCEncodedVideoFrame.webidl | 41 ++ dom/webidl/RTCRtpReceiver.webidl | 6 + dom/webidl/RTCRtpScriptTransform.webidl | 20 + dom/webidl/RTCRtpScriptTransformer.webidl | 19 + dom/webidl/RTCRtpSender.webidl | 6 + dom/webidl/RTCTransformEvent.webidl | 20 + dom/webidl/moz.build | 6 + dom/workers/EventWithOptionsRunnable.cpp | 164 +++++++ dom/workers/EventWithOptionsRunnable.h | 55 +++ dom/workers/Worker.cpp | 35 ++ dom/workers/Worker.h | 7 + dom/workers/WorkerScope.h | 1 + dom/workers/moz.build | 2 + dom/workers/test/test_worker_interfaces.js | 10 + media/webrtc/signaling/gtest/Canonicals.h | 12 + media/webrtc/signaling/gtest/MockConduit.h | 1 + modules/libpref/init/StaticPrefList.yaml | 7 +- widget/EventMessageList.h | 1 + xpcom/ds/StaticAtoms.py | 2 + 52 files changed, 3028 insertions(+), 89 deletions(-) create mode 100644 dom/media/webrtc/jsapi/RTCEncodedAudioFrame.cpp create mode 100644 dom/media/webrtc/jsapi/RTCEncodedAudioFrame.h create mode 100644 dom/media/webrtc/jsapi/RTCEncodedFrameBase.cpp create mode 100644 dom/media/webrtc/jsapi/RTCEncodedFrameBase.h create mode 100644 dom/media/webrtc/jsapi/RTCEncodedVideoFrame.cpp create mode 100644 dom/media/webrtc/jsapi/RTCEncodedVideoFrame.h create mode 100644 dom/media/webrtc/jsapi/RTCRtpScriptTransform.cpp create mode 100644 dom/media/webrtc/jsapi/RTCRtpScriptTransform.h create mode 100644 dom/media/webrtc/jsapi/RTCRtpScriptTransformer.cpp create mode 100644 dom/media/webrtc/jsapi/RTCRtpScriptTransformer.h create mode 100644 dom/media/webrtc/jsapi/RTCTransformEventRunnable.cpp create mode 100644 dom/media/webrtc/jsapi/RTCTransformEventRunnable.h create mode 100644 dom/media/webrtc/libwebrtcglue/FrameTransformer.cpp create mode 100644 dom/media/webrtc/libwebrtcglue/FrameTransformer.h create mode 100644 dom/media/webrtc/libwebrtcglue/FrameTransformerProxy.cpp create mode 100644 dom/media/webrtc/libwebrtcglue/FrameTransformerProxy.h create mode 100644 dom/webidl/RTCEncodedAudioFrame.webidl create mode 100644 dom/webidl/RTCEncodedVideoFrame.webidl create mode 100644 dom/webidl/RTCRtpScriptTransform.webidl create mode 100644 dom/webidl/RTCRtpScriptTransformer.webidl create mode 100644 dom/webidl/RTCTransformEvent.webidl create mode 100644 dom/workers/EventWithOptionsRunnable.cpp create mode 100644 dom/workers/EventWithOptionsRunnable.h diff --git a/dom/events/EventNameList.h b/dom/events/EventNameList.h index 3cc34a680692..680d520496cd 100644 --- a/dom/events/EventNameList.h +++ b/dom/events/EventNameList.h @@ -295,6 +295,7 @@ WINDOW_EVENT(languagechange, eLanguageChange, // need a different macro to flag things like that (IDL, but not content // attributes on body/frameset), or is just using EventNameType_None enough? WINDOW_EVENT(message, eMessage, EventNameType_None, eBasicEventClass) +WINDOW_EVENT(rtctransform, eRTCTransform, EventNameType_None, eBasicEventClass) WINDOW_EVENT(messageerror, eMessageError, EventNameType_HTMLBodyOrFramesetOnly, eBasicEventClass) WINDOW_EVENT(offline, eOffline, diff --git a/dom/media/webrtc/jsapi/RTCEncodedAudioFrame.cpp b/dom/media/webrtc/jsapi/RTCEncodedAudioFrame.cpp new file mode 100644 index 000000000000..3a3395934425 --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCEncodedAudioFrame.cpp @@ -0,0 +1,94 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "jsapi/RTCEncodedAudioFrame.h" + +#include + +#include +#include + +#include "api/frame_transformer_interface.h" + +#include "jsapi/RTCEncodedFrameBase.h" +#include "jsapi/RTCRtpScriptTransform.h" +#include "mozilla/dom/RTCRtpScriptTransformer.h" +#include "mozilla/dom/RTCEncodedAudioFrameBinding.h" +#include "nsWrapperCache.h" +#include "nsISupports.h" +#include "nsCycleCollectionParticipant.h" +#include "nsIGlobalObject.h" +#include "nsContentUtils.h" +#include "mozilla/HoldDropJSObjects.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Unused.h" +#include "mozilla/fallible.h" +#include "js/RootingAPI.h" + +namespace mozilla::dom { + +NS_IMPL_CYCLE_COLLECTION_INHERITED(RTCEncodedAudioFrame, RTCEncodedFrameBase, + mOwner) +NS_IMPL_ADDREF_INHERITED(RTCEncodedAudioFrame, RTCEncodedFrameBase) +NS_IMPL_RELEASE_INHERITED(RTCEncodedAudioFrame, RTCEncodedFrameBase) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCEncodedAudioFrame) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY +NS_INTERFACE_MAP_END_INHERITING(RTCEncodedFrameBase) + +RTCEncodedAudioFrame::RTCEncodedAudioFrame( + nsIGlobalObject* aGlobal, + std::unique_ptr aFrame, + uint64_t aCounter, RTCRtpScriptTransformer* aOwner) + : RTCEncodedFrameBase(aGlobal, std::move(aFrame), aCounter), + mOwner(aOwner) { + mMetadata.mSynchronizationSource.Construct(mFrame->GetSsrc()); + mMetadata.mPayloadType.Construct(mFrame->GetPayloadType()); + // send frames are derived directly from TransformableFrameInterface, not + // TransformableAudioFrameInterface! Right now, send frames have no csrcs + // or sequence number + // TODO(bug 1835076): Fix this + if (mFrame->GetDirection() == + webrtc::TransformableFrameInterface::Direction::kReceiver) { + const auto& audioFrame( + static_cast(*mFrame)); + mMetadata.mContributingSources.Construct(); + for (const auto csrc : audioFrame.GetContributingSources()) { + Unused << mMetadata.mContributingSources.Value().AppendElement(csrc, + fallible); + } + mMetadata.mSequenceNumber.Construct(audioFrame.GetHeader().sequenceNumber); + } + + // Base class needs this, but can't do it itself because of an assertion in + // the cycle-collector. + mozilla::HoldJSObjects(this); +} + +RTCEncodedAudioFrame::~RTCEncodedAudioFrame() { + // Base class needs this, but can't do it itself because of an assertion in + // the cycle-collector. + mozilla::DropJSObjects(this); +} + +JSObject* RTCEncodedAudioFrame::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return RTCEncodedAudioFrame_Binding::Wrap(aCx, this, aGivenProto); +} + +nsIGlobalObject* RTCEncodedAudioFrame::GetParentObject() const { + return mGlobal; +} + +void RTCEncodedAudioFrame::GetMetadata( + RTCEncodedAudioFrameMetadata& aMetadata) const { + aMetadata = mMetadata; +} + +bool RTCEncodedAudioFrame::CheckOwner(RTCRtpScriptTransformer* aOwner) const { + return aOwner == mOwner; +} +} // namespace mozilla::dom diff --git a/dom/media/webrtc/jsapi/RTCEncodedAudioFrame.h b/dom/media/webrtc/jsapi/RTCEncodedAudioFrame.h new file mode 100644 index 000000000000..339231ccc27e --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCEncodedAudioFrame.h @@ -0,0 +1,52 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDAUDIOFRAME_H_ +#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDAUDIOFRAME_H_ + +#include "mozilla/RefPtr.h" +#include "nsIGlobalObject.h" +#include "jsapi/RTCEncodedFrameBase.h" +#include "mozilla/dom/RTCEncodedAudioFrameBinding.h" + +namespace mozilla::dom { + +// Wraps a libwebrtc frame, allowing the frame buffer to be modified, and +// providing read-only access to various metadata. After the libwebrtc frame is +// extracted (with RTCEncodedFrameBase::TakeFrame), the frame buffer is +// detached, but the metadata remains accessible. +class RTCEncodedAudioFrame final : public RTCEncodedFrameBase { + public: + explicit RTCEncodedAudioFrame( + nsIGlobalObject* aGlobal, + std::unique_ptr aFrame, + uint64_t aCounter, RTCRtpScriptTransformer* aOwner); + + // nsISupports + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(RTCEncodedAudioFrame, + RTCEncodedFrameBase) + + // webidl (timestamp and data accessors live in base class) + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + nsIGlobalObject* GetParentObject() const; + + void GetMetadata(RTCEncodedAudioFrameMetadata& aMetadata) const; + + bool CheckOwner(RTCRtpScriptTransformer* aOwner) const override; + + bool IsVideo() const override { return false; } + + private: + virtual ~RTCEncodedAudioFrame(); + RefPtr mOwner; + RTCEncodedAudioFrameMetadata mMetadata; +}; + +} // namespace mozilla::dom +#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDAUDIOFRAME_H_ diff --git a/dom/media/webrtc/jsapi/RTCEncodedFrameBase.cpp b/dom/media/webrtc/jsapi/RTCEncodedFrameBase.cpp new file mode 100644 index 000000000000..8abd0fbc4500 --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCEncodedFrameBase.cpp @@ -0,0 +1,71 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "jsapi/RTCEncodedFrameBase.h" + +#include "nsIGlobalObject.h" +#include "mozilla/dom/ScriptSettings.h" +#include "js/ArrayBuffer.h" + +namespace mozilla::dom { +NS_IMPL_CYCLE_COLLECTION_WITH_JS_MEMBERS(RTCEncodedFrameBase, (mGlobal), + (mData)) +NS_IMPL_CYCLE_COLLECTING_ADDREF(RTCEncodedFrameBase) +NS_IMPL_CYCLE_COLLECTING_RELEASE(RTCEncodedFrameBase) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCEncodedFrameBase) + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +RTCEncodedFrameBase::RTCEncodedFrameBase( + nsIGlobalObject* aGlobal, + std::unique_ptr aFrame, + uint64_t aCounter) + : mGlobal(aGlobal), + mFrame(std::move(aFrame)), + mCounter(aCounter), + mTimestamp(mFrame->GetTimestamp()) { + AutoJSAPI jsapi; + if (NS_WARN_IF(!jsapi.Init(mGlobal))) { + return; + } + + // Avoid a copy + mData = JS::NewArrayBufferWithUserOwnedContents( + jsapi.cx(), mFrame->GetData().size(), (void*)(mFrame->GetData().data())); +} + +RTCEncodedFrameBase::~RTCEncodedFrameBase() = default; + +unsigned long RTCEncodedFrameBase::Timestamp() const { return mTimestamp; } + +void RTCEncodedFrameBase::SetData(const ArrayBuffer& aData) { + mData.set(aData.Obj()); + if (mFrame) { + aData.ComputeState(); + mFrame->SetData(rtc::ArrayView( + static_cast(aData.Data()), aData.Length())); + } +} + +void RTCEncodedFrameBase::GetData(JSContext* aCx, JS::Rooted* aObj) { + aObj->set(mData); +} + +uint64_t RTCEncodedFrameBase::GetCounter() const { return mCounter; } + +std::unique_ptr +RTCEncodedFrameBase::TakeFrame() { + AutoJSAPI jsapi; + if (!jsapi.Init(mGlobal)) { + MOZ_CRASH("Could not init JSAPI!"); + } + JS::Rooted rootedData(jsapi.cx(), mData); + JS::DetachArrayBuffer(jsapi.cx(), rootedData); + return std::move(mFrame); +} + +} // namespace mozilla::dom diff --git a/dom/media/webrtc/jsapi/RTCEncodedFrameBase.h b/dom/media/webrtc/jsapi/RTCEncodedFrameBase.h new file mode 100644 index 000000000000..25546257be92 --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCEncodedFrameBase.h @@ -0,0 +1,58 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDFRAMEBASE_H_ +#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDFRAMEBASE_H_ + +#include "js/TypeDecls.h" +#include "mozilla/dom/TypedArray.h" // ArrayBuffer +#include "mozilla/Assertions.h" +#include "api/frame_transformer_interface.h" +#include + +class nsIGlobalObject; + +namespace mozilla::dom { + +class RTCRtpScriptTransformer; + +class RTCEncodedFrameBase : public nsISupports, public nsWrapperCache { + public: + explicit RTCEncodedFrameBase( + nsIGlobalObject* aGlobal, + std::unique_ptr aFrame, + uint64_t aCounter); + + // nsISupports + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_CLASS(RTCEncodedFrameBase) + + // Common webidl for RTCEncodedVideoFrame/RTCEncodedAudioFrame + unsigned long Timestamp() const; + + void SetData(const ArrayBuffer& aData); + + void GetData(JSContext* aCx, JS::Rooted* aObj); + + uint64_t GetCounter() const; + + virtual bool CheckOwner(RTCRtpScriptTransformer* aOwner) const = 0; + + std::unique_ptr TakeFrame(); + + virtual bool IsVideo() const = 0; + + protected: + virtual ~RTCEncodedFrameBase(); + RefPtr mGlobal; + std::unique_ptr mFrame; + const uint64_t mCounter = 0; + const unsigned long mTimestamp = 0; + JS::Heap mData; +}; + +} // namespace mozilla::dom +#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDFRAMEBASE_H_ diff --git a/dom/media/webrtc/jsapi/RTCEncodedVideoFrame.cpp b/dom/media/webrtc/jsapi/RTCEncodedVideoFrame.cpp new file mode 100644 index 000000000000..f3f8eb4a1542 --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCEncodedVideoFrame.cpp @@ -0,0 +1,117 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "jsapi/RTCEncodedVideoFrame.h" + +#include +#include +#include +#include + +#include "api/frame_transformer_interface.h" + +#include "jsapi/RTCEncodedFrameBase.h" +#include "mozilla/dom/RTCEncodedVideoFrameBinding.h" +#include "mozilla/dom/RTCRtpScriptTransformer.h" +#include "nsWrapperCache.h" +#include "nsISupports.h" +#include "nsCycleCollectionParticipant.h" +#include "nsIGlobalObject.h" +#include "nsContentUtils.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Unused.h" +#include "mozilla/fallible.h" +#include "mozilla/Maybe.h" +#include "js/RootingAPI.h" + +namespace mozilla::dom { + +NS_IMPL_CYCLE_COLLECTION_INHERITED(RTCEncodedVideoFrame, RTCEncodedFrameBase, + mOwner) +NS_IMPL_ADDREF_INHERITED(RTCEncodedVideoFrame, RTCEncodedFrameBase) +NS_IMPL_RELEASE_INHERITED(RTCEncodedVideoFrame, RTCEncodedFrameBase) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCEncodedVideoFrame) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY +NS_INTERFACE_MAP_END_INHERITING(RTCEncodedFrameBase) + +RTCEncodedVideoFrame::RTCEncodedVideoFrame( + nsIGlobalObject* aGlobal, + std::unique_ptr aFrame, + uint64_t aCounter, RTCRtpScriptTransformer* aOwner) + : RTCEncodedFrameBase(aGlobal, std::move(aFrame), aCounter), + mOwner(aOwner) { + const auto& videoFrame( + static_cast(*mFrame)); + mType = videoFrame.IsKeyFrame() ? RTCEncodedVideoFrameType::Key + : RTCEncodedVideoFrameType::Delta; + + if (videoFrame.GetMetadata().GetFrameId().has_value()) { + mMetadata.mFrameId.Construct(*videoFrame.GetMetadata().GetFrameId()); + } + mMetadata.mDependencies.Construct(); + for (const auto dep : videoFrame.GetMetadata().GetFrameDependencies()) { + Unused << mMetadata.mDependencies.Value().AppendElement( + static_cast(dep), fallible); + } + mMetadata.mWidth.Construct(videoFrame.GetMetadata().GetWidth()); + mMetadata.mHeight.Construct(videoFrame.GetMetadata().GetHeight()); + if (videoFrame.GetMetadata().GetSpatialIndex() >= 0) { + mMetadata.mSpatialIndex.Construct( + videoFrame.GetMetadata().GetSpatialIndex()); + } + if (videoFrame.GetMetadata().GetTemporalIndex() >= 0) { + mMetadata.mTemporalIndex.Construct( + videoFrame.GetMetadata().GetTemporalIndex()); + } + mMetadata.mSynchronizationSource.Construct(videoFrame.GetSsrc()); + mMetadata.mPayloadType.Construct(videoFrame.GetPayloadType()); + mMetadata.mContributingSources.Construct(); + for (const auto csrc : videoFrame.GetMetadata().GetCsrcs()) { + Unused << mMetadata.mContributingSources.Value().AppendElement(csrc, + fallible); + } + + // The metadata timestamp is different, and not presently present in the + // libwebrtc types + if (!videoFrame.GetRid().empty()) { + mRid = Some(videoFrame.GetRid()); + } + + // Base class needs this, but can't do it itself because of an assertion in + // the cycle-collector. + mozilla::HoldJSObjects(this); +} + +RTCEncodedVideoFrame::~RTCEncodedVideoFrame() { + // Base class needs this, but can't do it itself because of an assertion in + // the cycle-collector. + mozilla::DropJSObjects(this); +} + +JSObject* RTCEncodedVideoFrame::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return RTCEncodedVideoFrame_Binding::Wrap(aCx, this, aGivenProto); +} + +nsIGlobalObject* RTCEncodedVideoFrame::GetParentObject() const { + return mGlobal; +} + +RTCEncodedVideoFrameType RTCEncodedVideoFrame::Type() const { return mType; } + +void RTCEncodedVideoFrame::GetMetadata( + RTCEncodedVideoFrameMetadata& aMetadata) { + aMetadata = mMetadata; +} + +bool RTCEncodedVideoFrame::CheckOwner(RTCRtpScriptTransformer* aOwner) const { + return aOwner == mOwner; +} + +Maybe RTCEncodedVideoFrame::Rid() const { return mRid; } + +} // namespace mozilla::dom diff --git a/dom/media/webrtc/jsapi/RTCEncodedVideoFrame.h b/dom/media/webrtc/jsapi/RTCEncodedVideoFrame.h new file mode 100644 index 000000000000..7f1e04db96ad --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCEncodedVideoFrame.h @@ -0,0 +1,61 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDVIDEOFRAME_H_ +#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDVIDEOFRAME_H_ + +#include "mozilla/RefPtr.h" +#include "nsIGlobalObject.h" +#include "jsapi/RTCEncodedFrameBase.h" +#include "mozilla/dom/RTCEncodedVideoFrameBinding.h" + +namespace mozilla::dom { +class RTCRtpScriptTransformer; + +// Wraps a libwebrtc frame, allowing the frame buffer to be modified, and +// providing read-only access to various metadata. After the libwebrtc frame is +// extracted (with RTCEncodedFrameBase::TakeFrame), the frame buffer is +// detached, but the metadata remains accessible. +class RTCEncodedVideoFrame final : public RTCEncodedFrameBase { + public: + explicit RTCEncodedVideoFrame( + nsIGlobalObject* aGlobal, + std::unique_ptr aFrame, + uint64_t aCounter, RTCRtpScriptTransformer* aOwner); + + // nsISupports + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(RTCEncodedVideoFrame, + RTCEncodedFrameBase) + + // webidl (timestamp and data accessors live in base class) + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + nsIGlobalObject* GetParentObject() const; + + RTCEncodedVideoFrameType Type() const; + + void GetMetadata(RTCEncodedVideoFrameMetadata& aMetadata); + + bool CheckOwner(RTCRtpScriptTransformer* aOwner) const override; + + bool IsVideo() const override { return true; } + + // Not in webidl right now. Might change. + // https://github.com/w3c/webrtc-encoded-transform/issues/147 + Maybe Rid() const; + + private: + virtual ~RTCEncodedVideoFrame(); + RefPtr mOwner; + RTCEncodedVideoFrameType mType; + RTCEncodedVideoFrameMetadata mMetadata; + Maybe mRid; +}; + +} // namespace mozilla::dom +#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDVIDEOFRAME_H_ diff --git a/dom/media/webrtc/jsapi/RTCRtpReceiver.cpp b/dom/media/webrtc/jsapi/RTCRtpReceiver.cpp index 136aa5142f2a..7f4d6340d198 100644 --- a/dom/media/webrtc/jsapi/RTCRtpReceiver.cpp +++ b/dom/media/webrtc/jsapi/RTCRtpReceiver.cpp @@ -3,32 +3,82 @@ * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "RTCRtpReceiver.h" + +#include + +#include +#include +#include + +#include "call/call.h" +#include "call/audio_receive_stream.h" +#include "call/video_receive_stream.h" +#include "api/rtp_parameters.h" +#include "api/units/timestamp.h" +#include "api/units/time_delta.h" +#include "system_wrappers/include/clock.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + +#include "RTCRtpTransceiver.h" #include "PeerConnectionImpl.h" +#include "RTCStatsReport.h" +#include "mozilla/dom/RTCRtpReceiverBinding.h" +#include "mozilla/dom/RTCRtpSourcesBinding.h" +#include "mozilla/dom/RTCStatsReportBinding.h" +#include "jsep/JsepTransceiver.h" +#include "libwebrtcglue/MediaConduitControl.h" +#include "libwebrtcglue/MediaConduitInterface.h" +#include "transportbridge/MediaPipeline.h" +#include "sdp/SdpEnum.h" +#include "sdp/SdpAttribute.h" +#include "MediaTransportHandler.h" +#include "RemoteTrackSource.h" + #include "mozilla/dom/RTCRtpCapabilitiesBinding.h" -#include "transport/logging.h" #include "mozilla/dom/MediaStreamTrack.h" #include "mozilla/dom/Promise.h" +#include "mozilla/dom/Nullable.h" +#include "mozilla/dom/Document.h" +#include "mozilla/dom/AudioStreamTrack.h" +#include "mozilla/dom/VideoStreamTrack.h" +#include "mozilla/dom/RTCRtpScriptTransform.h" + #include "nsPIDOMWindow.h" #include "PrincipalHandle.h" #include "nsIPrincipal.h" -#include "mozilla/dom/Document.h" -#include "mozilla/NullPrincipal.h" #include "MediaTrackGraph.h" -#include "RemoteTrackSource.h" -#include "libwebrtcglue/RtpRtcpConfig.h" -#include "nsString.h" -#include "mozilla/dom/AudioStreamTrack.h" -#include "mozilla/dom/VideoStreamTrack.h" -#include "MediaTransportHandler.h" -#include "jsep/JsepTransceiver.h" -#include "mozilla/dom/RTCRtpReceiverBinding.h" -#include "mozilla/dom/RTCRtpSourcesBinding.h" -#include "RTCStatsReport.h" +#include "nsStringFwd.h" +#include "MediaSegment.h" +#include "nsLiteralString.h" +#include "nsTArray.h" +#include "nsDOMNavigationTiming.h" +#include "MainThreadUtils.h" +#include "ErrorList.h" +#include "nsWrapperCache.h" +#include "nsISupports.h" +#include "nsCOMPtr.h" +#include "nsIScriptObjectPrincipal.h" +#include "nsCycleCollectionParticipant.h" +#include "nsDebug.h" +#include "nsThreadUtils.h" +#include "PerformanceRecorder.h" + +#include "mozilla/NullPrincipal.h" #include "mozilla/Preferences.h" -#include "PeerConnectionCtx.h" -#include "RTCRtpTransceiver.h" -#include "libwebrtcglue/AudioConduit.h" -#include "call/call.h" +#include "mozilla/StateMirroring.h" +#include "mozilla/Logging.h" +#include "mozilla/RefPtr.h" +#include "mozilla/AbstractThread.h" +#include "mozilla/StateWatching.h" +#include "mozilla/Maybe.h" +#include "mozilla/Assertions.h" +#include "mozilla/AlreadyAddRefed.h" +#include "mozilla/MozPromise.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/fallible.h" +#include "mozilla/mozalloc_oom.h" +#include "mozilla/ErrorResult.h" +#include "js/RootingAPI.h" namespace mozilla::dom { @@ -40,8 +90,8 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(RTCRtpReceiver) NS_IMPL_CYCLE_COLLECTION_UNLINK_PRESERVED_WRAPPER NS_IMPL_CYCLE_COLLECTION_UNLINK_END NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(RTCRtpReceiver) - NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mWindow, mPc, mTransceiver, mTrack, - mTrackSource) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mWindow, mPc, mTransceiver, mTransform, + mTrack, mTrackSource) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END NS_IMPL_CYCLE_COLLECTING_ADDREF(RTCRtpReceiver) @@ -89,7 +139,8 @@ RTCRtpReceiver::RTCRtpReceiver( INIT_CANONICAL(mAudioCodecs, std::vector()), INIT_CANONICAL(mVideoCodecs, std::vector()), INIT_CANONICAL(mVideoRtpRtcpConfig, Nothing()), - INIT_CANONICAL(mReceiving, false) { + INIT_CANONICAL(mReceiving, false), + INIT_CANONICAL(mFrameTransformerProxy, nullptr) { PrincipalHandle principalHandle = GetPrincipalHandle(aWindow, aPrivacy); const bool isAudio = aConduit->type() == MediaSessionConduit::AUDIO; @@ -605,6 +656,9 @@ void RTCRtpReceiver::Shutdown() { mRtcpByeListener.DisconnectIfExists(); mRtcpTimeoutListener.DisconnectIfExists(); mUnmuteListener.DisconnectIfExists(); + if (mTransform) { + mTransform->GetProxy().SetReceiver(nullptr); + } } void RTCRtpReceiver::BreakCycles() { @@ -937,6 +991,48 @@ const JsepTransceiver& RTCRtpReceiver::GetJsepTransceiver() const { return mTransceiver->GetJsepTransceiver(); } +void RTCRtpReceiver::SetTransform(RTCRtpScriptTransform* aTransform, + ErrorResult& aError) { + if (aTransform == mTransform.get()) { + // Ok... smile and nod + // TODO: Depending on spec, this might throw + // https://github.com/w3c/webrtc-encoded-transform/issues/189 + return; + } + + if (aTransform && aTransform->IsClaimed()) { + aError.ThrowInvalidStateError("transform has already been used elsewhere"); + return; + } + + if (aTransform) { + mFrameTransformerProxy = &aTransform->GetProxy(); + } else { + mFrameTransformerProxy = nullptr; + } + + if (mTransform) { + mTransform->GetProxy().SetReceiver(nullptr); + } + + mTransform = const_cast(aTransform); + + if (mTransform) { + mTransform->GetProxy().SetReceiver(this); + mTransform->SetClaimed(); + } +} + +void RTCRtpReceiver::RequestKeyFrame() { + if (!mTransform || !mPipeline) { + return; + } + + mPipeline->mConduit->AsVideoSessionConduit().apply([&](const auto& conduit) { + conduit->RequestKeyFrame(&mTransform->GetProxy()); + }); +} + } // namespace mozilla::dom #undef LOGTAG diff --git a/dom/media/webrtc/jsapi/RTCRtpReceiver.h b/dom/media/webrtc/jsapi/RTCRtpReceiver.h index dc1ded11f854..cc11a0605397 100644 --- a/dom/media/webrtc/jsapi/RTCRtpReceiver.h +++ b/dom/media/webrtc/jsapi/RTCRtpReceiver.h @@ -38,6 +38,7 @@ struct RTCRtpCapabilities; struct RTCRtpContributingSource; struct RTCRtpSynchronizationSource; class RTCRtpTransceiver; +class RTCRtpScriptTransform; class RTCRtpReceiver : public nsISupports, public nsWrapperCache, @@ -72,6 +73,10 @@ class RTCRtpReceiver : public nsISupports, const uint32_t aSource, const DOMHighResTimeStamp aTimestamp, const uint32_t aRtpTimestamp, const bool aHasLevel, const uint8_t aLevel); + RTCRtpScriptTransform* GetTransform() const { return mTransform; } + + void SetTransform(RTCRtpScriptTransform* aTransform, ErrorResult& aError); + nsPIDOMWindowInner* GetParentObject() const; nsTArray> GetStatsInternal( bool aSkipIceStats = false); @@ -120,6 +125,9 @@ class RTCRtpReceiver : public nsISupports, // ALPN negotiation. void UpdatePrincipalPrivacy(PrincipalPrivacy aPrivacy); + // Called by FrameTransformerProxy + void RequestKeyFrame(); + void OnRtcpBye(); void OnRtcpTimeout(); @@ -141,11 +149,17 @@ class RTCRtpReceiver : public nsISupports, Canonical>& CanonicalVideoCodecs() { return mVideoCodecs; } + Canonical>& CanonicalVideoRtpRtcpConfig() { return mVideoRtpRtcpConfig; } + Canonical& CanonicalReceiving() override { return mReceiving; } + Canonical>& CanonicalFrameTransformerProxy() { + return mFrameTransformerProxy; + } + private: virtual ~RTCRtpReceiver(); @@ -168,6 +182,7 @@ class RTCRtpReceiver : public nsISupports, RefPtr mPipeline; RefPtr mTransportHandler; RefPtr mTransceiver; + RefPtr mTransform; // This is [[AssociatedRemoteMediaStreams]], basically. We do not keep the // streams themselves here, because that would require this object to know // where the stream list for the whole RTCPeerConnection lives.. @@ -191,6 +206,7 @@ class RTCRtpReceiver : public nsISupports, Canonical> mVideoCodecs; Canonical> mVideoRtpRtcpConfig; Canonical mReceiving; + Canonical> mFrameTransformerProxy; }; } // namespace dom diff --git a/dom/media/webrtc/jsapi/RTCRtpScriptTransform.cpp b/dom/media/webrtc/jsapi/RTCRtpScriptTransform.cpp new file mode 100644 index 000000000000..43f34c456fe0 --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCRtpScriptTransform.cpp @@ -0,0 +1,84 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "RTCRtpScriptTransform.h" + +#include "libwebrtcglue/FrameTransformerProxy.h" +#include "jsapi/RTCTransformEventRunnable.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/Worker.h" +#include "mozilla/dom/RTCRtpScriptTransformBinding.h" +#include "mozilla/dom/MessagePortBinding.h" +#include "mozilla/Logging.h" +#include "mozilla/AlreadyAddRefed.h" +#include "mozilla/RefPtr.h" +#include "nsPIDOMWindow.h" +#include "nsContentUtils.h" +#include "nsCOMPtr.h" +#include "nsDebug.h" +#include "ErrorList.h" +#include "nsWrapperCache.h" +#include "nsISupports.h" +#include "nsCycleCollectionParticipant.h" +#include "js/RootingAPI.h" + +namespace mozilla::dom { + +LazyLogModule gScriptTransformLog("RTCRtpScriptTransform"); + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(RTCRtpScriptTransform, mWindow) +NS_IMPL_CYCLE_COLLECTING_ADDREF(RTCRtpScriptTransform) +NS_IMPL_CYCLE_COLLECTING_RELEASE(RTCRtpScriptTransform) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCRtpScriptTransform) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +already_AddRefed RTCRtpScriptTransform::Constructor( + const GlobalObject& aGlobal, Worker& aWorker, + JS::Handle aOptions, + const Optional>& aTransfer, ErrorResult& aRv) { + nsCOMPtr ownerWindow = + do_QueryInterface(aGlobal.GetAsSupports()); + if (NS_WARN_IF(!ownerWindow)) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + auto newTransform = MakeRefPtr(ownerWindow); + RefPtr runnable = + new RTCTransformEventRunnable(aWorker, &newTransform->GetProxy()); + + if (aTransfer.WasPassed()) { + aWorker.PostEventWithOptions(aGlobal.Context(), aOptions, aTransfer.Value(), + runnable, aRv); + } else { + StructuredSerializeOptions transferOptions; + aWorker.PostEventWithOptions(aGlobal.Context(), aOptions, + transferOptions.mTransfer, runnable, aRv); + } + + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + return newTransform.forget(); +} + +RTCRtpScriptTransform::RTCRtpScriptTransform(nsPIDOMWindowInner* aWindow) + : mWindow(aWindow), mProxy(new FrameTransformerProxy) {} + +RTCRtpScriptTransform::~RTCRtpScriptTransform() { + mProxy->ReleaseScriptTransformer(); +} + +JSObject* RTCRtpScriptTransform::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return RTCRtpScriptTransform_Binding::Wrap(aCx, this, aGivenProto); +} + +} // namespace mozilla::dom + +#undef LOGTAG diff --git a/dom/media/webrtc/jsapi/RTCRtpScriptTransform.h b/dom/media/webrtc/jsapi/RTCRtpScriptTransform.h new file mode 100644 index 000000000000..362c59b76161 --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCRtpScriptTransform.h @@ -0,0 +1,63 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCRTPSCRIPTTRANSFORM_H_ +#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCRTPSCRIPTTRANSFORM_H_ + +#include "nsISupports.h" +#include "nsWrapperCache.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Maybe.h" +#include "js/RootingAPI.h" +#include "nsTArray.h" + +class nsPIDOMWindowInner; + +namespace mozilla { +class FrameTransformerProxy; +class ErrorResult; + +namespace dom { +class Worker; +class GlobalObject; +template +class Sequence; +template +class Optional; + +class RTCRtpScriptTransform : public nsISupports, public nsWrapperCache { + public: + static already_AddRefed Constructor( + const GlobalObject& aGlobal, Worker& aWorker, + JS::Handle aOptions, + const Optional>& aTransfer, ErrorResult& aRv); + + explicit RTCRtpScriptTransform(nsPIDOMWindowInner* aWindow); + + // nsISupports + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(RTCRtpScriptTransform) + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + nsPIDOMWindowInner* GetParentObject() const { return mWindow; } + + FrameTransformerProxy& GetProxy() { return *mProxy; } + + bool IsClaimed() const { return mClaimed; } + void SetClaimed() { mClaimed = true; } + + private: + virtual ~RTCRtpScriptTransform(); + RefPtr mWindow; + RefPtr mProxy; + bool mClaimed = false; +}; + +} // namespace dom +} // namespace mozilla +#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCRTPSCRIPTTRANSFORM_H_ diff --git a/dom/media/webrtc/jsapi/RTCRtpScriptTransformer.cpp b/dom/media/webrtc/jsapi/RTCRtpScriptTransformer.cpp new file mode 100644 index 000000000000..f02ef74e6bf7 --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCRtpScriptTransformer.cpp @@ -0,0 +1,449 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "RTCRtpScriptTransformer.h" + +#include + +#include +#include +#include + +#include "api/frame_transformer_interface.h" + +#include "nsString.h" +#include "nsCycleCollectionParticipant.h" +#include "nsISupports.h" +#include "ErrorList.h" +#include "nsDebug.h" +#include "nsCycleCollectionTraversalCallback.h" +#include "nsTArray.h" +#include "nsWrapperCache.h" +#include "nsIGlobalObject.h" +#include "nsCOMPtr.h" +#include "nsStringFwd.h" +#include "nsLiteralString.h" +#include "nsContentUtils.h" +#include "mozilla/RefPtr.h" +#include "mozilla/AlreadyAddRefed.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/Result.h" +#include "mozilla/HoldDropJSObjects.h" +#include "mozilla/Maybe.h" +#include "mozilla/Assertions.h" +#include "mozilla/Logging.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/Likely.h" +#include "mozilla/dom/RTCRtpScriptTransformerBinding.h" +#include "mozilla/dom/BindingUtils.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/PrototypeList.h" +#include "mozilla/dom/WorkerRef.h" +#include "mozilla/dom/RTCEncodedAudioFrame.h" +#include "mozilla/dom/RTCEncodedVideoFrame.h" +#include "mozilla/dom/UnderlyingSourceCallbackHelpers.h" +#include "mozilla/dom/ToJSValue.h" +#include "mozilla/dom/ReadableStream.h" +#include "mozilla/dom/WritableStream.h" +#include "mozilla/dom/UnderlyingSinkCallbackHelpers.h" +#include "mozilla/dom/WritableStreamDefaultController.h" +#include "mozilla/dom/ReadableStreamController.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/Promise-inl.h" +#include "js/RootingAPI.h" +#include "js/Value.h" +#include "js/CallArgs.h" +#include "libwebrtcglue/FrameTransformerProxy.h" +#include "sdp/SdpAttribute.h" // CheckRidValidity + +namespace mozilla::dom { + +LazyLogModule gScriptTransformerLog("RTCRtpScriptTransformer"); + +NS_IMPL_CYCLE_COLLECTION_INHERITED(nsISupportsStreamSource, + UnderlyingSourceAlgorithmsWrapper, mStream, + mThingQueuedPromise, mQueue) +NS_IMPL_ADDREF_INHERITED(nsISupportsStreamSource, + UnderlyingSourceAlgorithmsWrapper) +NS_IMPL_RELEASE_INHERITED(nsISupportsStreamSource, + UnderlyingSourceAlgorithmsWrapper) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(nsISupportsStreamSource) +NS_INTERFACE_MAP_END_INHERITING(UnderlyingSourceAlgorithmsWrapper) + +nsISupportsStreamSource::nsISupportsStreamSource() = default; + +nsISupportsStreamSource::~nsISupportsStreamSource() = default; + +void nsISupportsStreamSource::Init(ReadableStream* aStream) { + mStream = aStream; +} + +void nsISupportsStreamSource::Enqueue(nsISupports* aThing) { + if (!mThingQueuedPromise) { + mQueue.AppendElement(aThing); + return; + } + + // Maybe put a limit here? Or at least some sort of logging if this gets + // unreasonably long? + AutoJSAPI jsapi; + if (NS_WARN_IF(!jsapi.Init(mStream->GetParentObject()))) { + return; + } + + EnqueueToStream(jsapi.cx(), aThing); + mThingQueuedPromise->MaybeResolveWithUndefined(); + mThingQueuedPromise = nullptr; +} + +already_AddRefed nsISupportsStreamSource::PullCallbackImpl( + JSContext* aCx, ReadableStreamController& aController, ErrorResult& aRv) { + if (!mQueue.IsEmpty()) { + EnqueueOneThingFromQueue(aCx); + return nullptr; + } + + RefPtr self(this); + mThingQueuedPromise = Promise::CreateInfallible(mStream->GetParentObject()); + return do_AddRef(mThingQueuedPromise); +} + +void nsISupportsStreamSource::EnqueueToStream(JSContext* aCx, + nsISupports* aThing) { + JS::Rooted jsThing(aCx); + if (NS_WARN_IF(MOZ_UNLIKELY(!ToJSValue(aCx, *aThing, &jsThing)))) { + // Do we want to add error handling for this? + return; + } + IgnoredErrorResult rv; + // EnqueueNative is CAN_RUN_SCRIPT. Need a strong-ref temporary. + auto stream = mStream; + stream->EnqueueNative(aCx, jsThing, rv); +} + +void nsISupportsStreamSource::EnqueueOneThingFromQueue(JSContext* aCx) { + if (!mQueue.IsEmpty()) { + RefPtr thing = mQueue[0]; + mQueue.RemoveElementAt(0); + EnqueueToStream(aCx, thing); + } +} + +NS_IMPL_CYCLE_COLLECTION_INHERITED(WritableStreamRTCFrameSink, + UnderlyingSinkAlgorithmsWrapper, + mTransformer) +NS_IMPL_ADDREF_INHERITED(WritableStreamRTCFrameSink, + UnderlyingSinkAlgorithmsWrapper) +NS_IMPL_RELEASE_INHERITED(WritableStreamRTCFrameSink, + UnderlyingSinkAlgorithmsWrapper) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(WritableStreamRTCFrameSink) +NS_INTERFACE_MAP_END_INHERITING(UnderlyingSinkAlgorithmsWrapper) + +WritableStreamRTCFrameSink::WritableStreamRTCFrameSink( + RTCRtpScriptTransformer* aTransformer) + : mTransformer(aTransformer) {} + +WritableStreamRTCFrameSink::~WritableStreamRTCFrameSink() = default; + +already_AddRefed WritableStreamRTCFrameSink::WriteCallback( + JSContext* aCx, JS::Handle aChunk, + WritableStreamDefaultController& aController, ErrorResult& aError) { + // Spec does not say to do this right now. Might be a spec bug, needs + // clarification. + // https://github.com/w3c/webrtc-encoded-transform/issues/191 + if (NS_WARN_IF(!aChunk.isObject())) { + aError.ThrowTypeError( + "Wrong type for RTCRtpScriptTransformer.[[writeable]]: Not an object"); + return nullptr; + } + + // Lame. But, without a webidl base class, this is the only way. + RefPtr video; + UNWRAP_OBJECT(RTCEncodedVideoFrame, &aChunk.toObject(), video); + RefPtr audio; + UNWRAP_OBJECT(RTCEncodedAudioFrame, &aChunk.toObject(), audio); + + RefPtr frame; + if (video) { + frame = video; + } else if (audio) { + frame = audio; + } + + if (NS_WARN_IF(!frame)) { + aError.ThrowTypeError( + "Wrong type for RTCRtpScriptTransformer.[[writeable]]: Not an " + "RTCEncodedAudioFrame or RTCEncodedVideoFrame"); + return nullptr; + } + + return mTransformer->OnTransformedFrame(frame, aError); +} + +// There is not presently an implementation of these for nsTHashMap :( +inline void ImplCycleCollectionUnlink( + RTCRtpScriptTransformer::GenerateKeyFramePromises& aMap) { + for (auto& tableEntry : aMap) { + ImplCycleCollectionUnlink(*tableEntry.GetModifiableData()); + } + aMap.Clear(); +} + +inline void ImplCycleCollectionTraverse( + nsCycleCollectionTraversalCallback& aCallback, + RTCRtpScriptTransformer::GenerateKeyFramePromises& aMap, const char* aName, + uint32_t aFlags = 0) { + for (auto& tableEntry : aMap) { + ImplCycleCollectionTraverse(aCallback, *tableEntry.GetModifiableData(), + aName, aFlags); + } +} + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_WITH_JS_MEMBERS( + RTCRtpScriptTransformer, + (mGlobal, mReadableSource, mReadable, mWritable, mWritableSink, + mKeyFrameRequestPromises, mGenerateKeyFramePromises), + (mOptions)) + +NS_IMPL_CYCLE_COLLECTING_ADDREF(RTCRtpScriptTransformer) +NS_IMPL_CYCLE_COLLECTING_RELEASE(RTCRtpScriptTransformer) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCRtpScriptTransformer) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +RTCRtpScriptTransformer::RTCRtpScriptTransformer(nsIGlobalObject* aGlobal) + : mGlobal(aGlobal), + mReadableSource(new nsISupportsStreamSource), + mWritableSink(new WritableStreamRTCFrameSink(this)), + mOptions(JS::UndefinedHandleValue) { + mozilla::HoldJSObjects(this); +} + +RTCRtpScriptTransformer::~RTCRtpScriptTransformer() { + mozilla::DropJSObjects(this); +} + +nsresult RTCRtpScriptTransformer::Init(JSContext* aCx, + JS::Handle aOptions, + WorkerPrivate* aWorkerPrivate, + FrameTransformerProxy* aProxy) { + ErrorResult rv; + RefPtr global(mGlobal); + auto source = mReadableSource; + auto sink = mWritableSink; + + // NOTE: We do not transfer these streams from mainthread, as the spec says, + // because there's no JS observable reason to. The spec is likely to change + // here, because it is overspecifying implementation details. + mReadable = ReadableStream::CreateNative(aCx, global, *source, Some(1.0), + nullptr, rv); + if (rv.Failed()) { + return rv.StealNSResult(); + } + mReadableSource->Init(mReadable); + + // WritableStream::CreateNative takes a nsIGlobalObject&, but + // ReadableStream::CreateNative takes a nsIGlobalObject*? + mWritable = + WritableStream::CreateNative(aCx, *global, *sink, Nothing(), nullptr, rv); + if (rv.Failed()) { + return rv.StealNSResult(); + } + + mOptions = aOptions; + mProxy = aProxy; + // This will return null if the worker is already shutting down. + // A call to ReleaseScriptTransformer will eventually result in a call to + // NotifyReleased. + mWorkerRef = StrongWorkerRef::Create( + aWorkerPrivate, "RTCRtpScriptTransformer", + [this, self = RefPtr(this)]() { mProxy->ReleaseScriptTransformer(); }); + if (mWorkerRef) { + mProxy->SetScriptTransformer(*this); + } + return NS_OK; +} + +void RTCRtpScriptTransformer::NotifyReleased() { + RejectPendingPromises(); + mWorkerRef = nullptr; + mProxy = nullptr; +} + +void RTCRtpScriptTransformer::RejectPendingPromises() { + for (const auto& promise : mKeyFrameRequestPromises) { + ErrorResult rv; + rv.ThrowInvalidStateError( + "RTCRtpScriptTransformer is not associated with a receiver"); + promise->MaybeReject(std::move(rv)); + } + mKeyFrameRequestPromises.Clear(); + + // GenerateKeyFrame promises are indexed by rid + for (auto& ridAndPromises : mGenerateKeyFramePromises) { + for (const auto& promise : ridAndPromises.GetData()) { + ErrorResult rv; + rv.ThrowInvalidStateError( + "RTCRtpScriptTransformer is not associated with a sender"); + promise->MaybeReject(std::move(rv)); + } + } + mGenerateKeyFramePromises.Clear(); +} + +void RTCRtpScriptTransformer::TransformFrame( + std::unique_ptr aFrame) { + if (!mVideo.isSome()) { + // First frame. mProxy will know whether it's video or not by now. + mVideo = mProxy->IsVideo(); + MOZ_ASSERT(mVideo.isSome()); + } + + RefPtr domFrame; + if (*mVideo) { + // If this is a send video keyframe, resolve any pending GenerateKeyFrame + // promises for its rid. + if (aFrame->GetDirection() == + webrtc::TransformableFrameInterface::Direction::kSender) { + auto* videoFrame = + static_cast(aFrame.get()); + if (videoFrame->IsKeyFrame()) { + ResolveGenerateKeyFramePromises(videoFrame->GetRid(), + videoFrame->GetTimestamp()); + if (!videoFrame->GetRid().empty() && + videoFrame->GetMetadata().GetSimulcastIdx() == 0) { + ResolveGenerateKeyFramePromises("", videoFrame->GetTimestamp()); + } + } + } + domFrame = new RTCEncodedVideoFrame(mGlobal, std::move(aFrame), + ++mLastEnqueuedFrameCounter, this); + } else { + domFrame = new RTCEncodedAudioFrame(mGlobal, std::move(aFrame), + ++mLastEnqueuedFrameCounter, this); + } + mReadableSource->Enqueue(domFrame); +} + +void RTCRtpScriptTransformer::GetOptions(JSContext* aCx, + JS::MutableHandle aVal, + ErrorResult& aError) { + if (!ToJSValue(aCx, mOptions, aVal)) { + aError.NoteJSContextException(aCx); + } +} + +already_AddRefed RTCRtpScriptTransformer::GenerateKeyFrame( + const Optional& aRid) { + Maybe utf8Rid; + if (aRid.WasPassed()) { + utf8Rid = Some(NS_ConvertUTF16toUTF8(aRid.Value()).get()); + std::string error; + if (!SdpRidAttributeList::CheckRidValidity(*utf8Rid, &error)) { + ErrorResult rv; + nsCString nsError(error.c_str()); + rv.ThrowNotAllowedError(nsError); + return Promise::CreateRejectedWithErrorResult(GetParentObject(), rv); + } + } + + nsCString key; + if (utf8Rid.isSome()) { + key.Assign(utf8Rid->data(), utf8Rid->size()); + } + + nsTArray>& promises = + mGenerateKeyFramePromises.LookupOrInsert(key); + if (!promises.Length()) { + // No pending keyframe generation request for this rid. Make one. + if (!mProxy || !mProxy->GenerateKeyFrame(utf8Rid)) { + ErrorResult rv; + rv.ThrowInvalidStateError( + "RTCRtpScriptTransformer is not associated with a video sender"); + return Promise::CreateRejectedWithErrorResult(GetParentObject(), rv); + } + } + RefPtr promise = Promise::CreateInfallible(GetParentObject()); + promises.AppendElement(promise); + return promise.forget(); +} + +void RTCRtpScriptTransformer::ResolveGenerateKeyFramePromises( + const std::string& aRid, uint64_t aTimestamp) { + nsCString key(aRid.data(), aRid.size()); + nsTArray> promises; + mGenerateKeyFramePromises.Remove(key, &promises); + for (auto& promise : promises) { + promise->MaybeResolve(aTimestamp); + } +} + +void RTCRtpScriptTransformer::GenerateKeyFrameError( + const Maybe& aRid, const CopyableErrorResult& aResult) { + nsCString key; + if (aRid.isSome()) { + key.Assign(aRid->data(), aRid->size()); + } + nsTArray> promises; + mGenerateKeyFramePromises.Remove(key, &promises); + for (auto& promise : promises) { + CopyableErrorResult rv(aResult); + promise->MaybeReject(std::move(rv)); + } +} + +already_AddRefed RTCRtpScriptTransformer::SendKeyFrameRequest() { + if (!mKeyFrameRequestPromises.Length()) { + if (!mProxy || !mProxy->RequestKeyFrame()) { + ErrorResult rv; + rv.ThrowInvalidStateError( + "RTCRtpScriptTransformer is not associated with a video receiver"); + return Promise::CreateRejectedWithErrorResult(GetParentObject(), rv); + } + } + RefPtr promise = Promise::CreateInfallible(GetParentObject()); + mKeyFrameRequestPromises.AppendElement(promise); + return promise.forget(); +} + +void RTCRtpScriptTransformer::KeyFrameRequestDone(bool aSuccess) { + auto promises = std::move(mKeyFrameRequestPromises); + if (aSuccess) { + for (const auto& promise : promises) { + promise->MaybeResolveWithUndefined(); + } + } else { + for (const auto& promise : promises) { + ErrorResult rv; + rv.ThrowInvalidStateError( + "Depacketizer is not defined, or not processing"); + promise->MaybeReject(std::move(rv)); + } + } +} + +JSObject* RTCRtpScriptTransformer::WrapObject( + JSContext* aCx, JS::Handle aGivenProto) { + return RTCRtpScriptTransformer_Binding::Wrap(aCx, this, aGivenProto); +} + +already_AddRefed RTCRtpScriptTransformer::OnTransformedFrame( + RTCEncodedFrameBase* aFrame, ErrorResult& aError) { + // Spec says to skip frames that are out of order or have wrong owner + if (aFrame->GetCounter() > mLastReceivedFrameCounter && + aFrame->CheckOwner(this) && mProxy) { + mLastReceivedFrameCounter = aFrame->GetCounter(); + mProxy->OnTransformedFrame(aFrame->TakeFrame()); + } + + return Promise::CreateResolvedWithUndefined(GetParentObject(), aError); +} + +} // namespace mozilla::dom + +#undef LOGTAG diff --git a/dom/media/webrtc/jsapi/RTCRtpScriptTransformer.h b/dom/media/webrtc/jsapi/RTCRtpScriptTransformer.h new file mode 100644 index 000000000000..6d61ac3cd576 --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCRtpScriptTransformer.h @@ -0,0 +1,197 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCRTPSCRIPTTRANSFORMER_H_ +#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCRTPSCRIPTTRANSFORMER_H_ + +#include "nsISupports.h" +#include "nsWrapperCache.h" +#include "mozilla/RefPtr.h" +#include "mozilla/dom/ReadableStream.h" +#include "mozilla/dom/WritableStream.h" +#include "mozilla/Maybe.h" +#include "js/RootingAPI.h" +#include "nsTArray.h" +#include "nsCOMArray.h" +#include +#include "nsTHashSet.h" +#include "nsCycleCollectionParticipant.h" + +class nsPIDOMWindowInner; + +namespace webrtc { +class TransformableFrameInterface; +} + +namespace mozilla { +class FrameTransformerProxy; + +namespace dom { +class Worker; +class WorkerPrivate; + +// Dirt simple source for ReadableStream that accepts nsISupports +// Might be suitable to move someplace else, with some polish. +class nsISupportsStreamSource final : public UnderlyingSourceAlgorithmsWrapper { + public: + nsISupportsStreamSource(); + nsISupportsStreamSource(const nsISupportsStreamSource&) = delete; + nsISupportsStreamSource(nsISupportsStreamSource&&) = delete; + nsISupportsStreamSource& operator=(const nsISupportsStreamSource&) = delete; + nsISupportsStreamSource& operator=(nsISupportsStreamSource&&) = delete; + + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(nsISupportsStreamSource, + UnderlyingSourceAlgorithmsWrapper) + + void Init(ReadableStream* aStream); + + void Enqueue(nsISupports* aThing); + + // From UnderlyingSourceAlgorithmsWrapper + already_AddRefed PullCallbackImpl( + JSContext* aCx, ReadableStreamController& aController, + ErrorResult& aRv) override; + + void EnqueueOneThingFromQueue(JSContext* aCx); + + private: + virtual ~nsISupportsStreamSource(); + + // Calls ReadableStream::EnqueueNative, which is MOZ_CAN_RUN_SCRIPT. + MOZ_CAN_RUN_SCRIPT_BOUNDARY void EnqueueToStream(JSContext* aCx, + nsISupports* aThing); + + RefPtr mStream; + RefPtr mThingQueuedPromise; + // mozilla::Queue is not cycle-collector friendly :( + nsCOMArray mQueue; +}; + +class RTCRtpScriptTransformer; + +class WritableStreamRTCFrameSink final + : public UnderlyingSinkAlgorithmsWrapper { + public: + explicit WritableStreamRTCFrameSink(RTCRtpScriptTransformer* aTransformer); + WritableStreamRTCFrameSink(const WritableStreamRTCFrameSink&) = delete; + WritableStreamRTCFrameSink(WritableStreamRTCFrameSink&&) = delete; + WritableStreamRTCFrameSink& operator=(const WritableStreamRTCFrameSink&) = + delete; + WritableStreamRTCFrameSink& operator=(WritableStreamRTCFrameSink&&) = delete; + + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(WritableStreamRTCFrameSink, + UnderlyingSinkAlgorithmsWrapper) + + already_AddRefed WriteCallback( + JSContext* aCx, JS::Handle aChunk, + WritableStreamDefaultController& aController, + ErrorResult& aError) override; + + private: + virtual ~WritableStreamRTCFrameSink(); + RefPtr mTransformer; +}; + +class RTCEncodedFrameBase; + +// Here's the basic flow. All of this happens on the worker thread. +// 0. We register with a FrameTransformerProxy. +// 1. That FrameTransformerProxy dispatches webrtc::TransformableFrameInterface +// to us (from either the encoder/depacketizer thread), via our +// TransformFrame method. +// 2. We wrap these frames in RTCEncodedAudio/VideoFrame, and feed them to +// mReadableSource, which queues them. +// 3. mReadableSource.PullCallbackImpl consumes that queue, and feeds the +// frames to mReadable. +// 4. JS worker code consumes from mReadable, does whatever transformation it +// wants, then writes the frames to mWritable. +// 5. mWritableSink.WriteCallback passes those frames to us. +// 6. We unwrap the webrtc::TransformableFrameInterface from these frames. +// 7. We pass these unwrapped frames back to the FrameTransformerProxy. +// (FrameTransformerProxy handles any dispatching/synchronization necessary) +// 8. Eventually, that FrameTransformerProxy calls NotifyReleased (possibly at +// our prompting). +class RTCRtpScriptTransformer final : public nsISupports, + public nsWrapperCache { + public: + explicit RTCRtpScriptTransformer(nsIGlobalObject* aGlobal); + + MOZ_CAN_RUN_SCRIPT_BOUNDARY nsresult Init(JSContext* aCx, + JS::Handle aOptions, + WorkerPrivate* aWorkerPrivate, + FrameTransformerProxy* aProxy); + + void NotifyReleased(); + + void TransformFrame( + std::unique_ptr aFrame); + already_AddRefed OnTransformedFrame(RTCEncodedFrameBase* aFrame, + ErrorResult& aError); + + // nsISupports + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_CLASS(RTCRtpScriptTransformer) + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + nsIGlobalObject* GetParentObject() const { return mGlobal; } + + // WebIDL Interface + already_AddRefed Readable() const { + return do_AddRef(mReadable); + } + already_AddRefed Writable() const { + return do_AddRef(mWritable); + } + + void GetOptions(JSContext* aCx, JS::MutableHandle aVal, + ErrorResult& aError); + + already_AddRefed GenerateKeyFrame(const Optional& aRid); + void GenerateKeyFrameError(const Maybe& aRid, + const CopyableErrorResult& aResult); + already_AddRefed SendKeyFrameRequest(); + void KeyFrameRequestDone(bool aSuccess); + + // Public to ease implementation of cycle collection functions + using GenerateKeyFramePromises = + nsTHashMap>>; + + private: + virtual ~RTCRtpScriptTransformer(); + void RejectPendingPromises(); + // empty string means no rid + void ResolveGenerateKeyFramePromises(const std::string& aRid, + uint64_t aTimestamp); + + nsCOMPtr mGlobal; + + RefPtr mProxy; + RefPtr mReadableSource; + RefPtr mReadable; + RefPtr mWritable; + RefPtr mWritableSink; + + JS::Heap mOptions; + uint64_t mLastEnqueuedFrameCounter = 0; + uint64_t mLastReceivedFrameCounter = 0; + nsTArray> mKeyFrameRequestPromises; + // Contains the promise returned for each call to GenerateKeyFrame(rid), in + // the order in which it was called, keyed by the rid (empty string if not + // passed). If there is already a promise in here for a given rid, we do not + // ask the FrameTransformerProxy again, and just bulk resolve/reject. + GenerateKeyFramePromises mGenerateKeyFramePromises; + Maybe mVideo; + RefPtr mWorkerRef; +}; + +} // namespace dom +} // namespace mozilla + +#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCRTPSCRIPTTRANSFORMER_H_ diff --git a/dom/media/webrtc/jsapi/RTCRtpSender.cpp b/dom/media/webrtc/jsapi/RTCRtpSender.cpp index 3d8e3e605342..e7c651c4fa53 100644 --- a/dom/media/webrtc/jsapi/RTCRtpSender.cpp +++ b/dom/media/webrtc/jsapi/RTCRtpSender.cpp @@ -3,22 +3,77 @@ * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "RTCRtpSender.h" -#include "transport/logging.h" -#include "mozilla/dom/MediaStreamTrack.h" -#include "mozilla/dom/Promise.h" -#include "mozilla/glean/GleanMetrics.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "system_wrappers/include/clock.h" +#include "call/call.h" +#include "api/rtp_parameters.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video_codecs/video_codec.h" +#include "api/video/video_codec_constants.h" +#include "call/audio_send_stream.h" +#include "call/video_send_stream.h" +#include "modules/rtp_rtcp/include/report_block_data.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + #include "nsPIDOMWindow.h" #include "nsString.h" +#include "MainThreadUtils.h" +#include "nsCOMPtr.h" +#include "nsContentUtils.h" +#include "nsCycleCollectionParticipant.h" +#include "nsDebug.h" +#include "nsISupports.h" +#include "nsLiteralString.h" +#include "nsStringFwd.h" +#include "nsTArray.h" +#include "nsThreadUtils.h" +#include "nsWrapperCache.h" +#include "mozilla/AbstractThread.h" +#include "mozilla/AlreadyAddRefed.h" +#include "mozilla/Assertions.h" +#include "mozilla/Attributes.h" +#include "mozilla/fallible.h" +#include "mozilla/Logging.h" +#include "mozilla/mozalloc_oom.h" +#include "mozilla/MozPromise.h" +#include "mozilla/OwningNonNull.h" +#include "mozilla/RefPtr.h" +#include "mozilla/StateWatching.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/Unused.h" +#include "mozilla/StateMirroring.h" +#include "mozilla/Maybe.h" +#include "mozilla/dom/MediaStreamTrack.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/RTCRtpScriptTransform.h" #include "mozilla/dom/VideoStreamTrack.h" -#include "jsep/JsepTransceiver.h" #include "mozilla/dom/RTCRtpSenderBinding.h" +#include "mozilla/dom/MediaStreamTrackBinding.h" +#include "mozilla/dom/Nullable.h" +#include "mozilla/dom/RTCRtpParametersBinding.h" +#include "mozilla/dom/RTCStatsReportBinding.h" +#include "mozilla/glean/GleanMetrics.h" +#include "js/RootingAPI.h" +#include "jsep/JsepTransceiver.h" #include "RTCStatsReport.h" -#include "mozilla/Preferences.h" #include "RTCRtpTransceiver.h" #include "PeerConnectionImpl.h" -#include "libwebrtcglue/AudioConduit.h" -#include -#include "call/call.h" +#include "libwebrtcglue/CodecConfig.h" +#include "libwebrtcglue/MediaConduitControl.h" +#include "libwebrtcglue/MediaConduitInterface.h" +#include "sdp/SdpAttribute.h" +#include "sdp/SdpEnum.h" namespace mozilla::dom { @@ -31,7 +86,7 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(RTCRtpSender) NS_IMPL_CYCLE_COLLECTION_UNLINK_END NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(RTCRtpSender) NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mWindow, mPc, mSenderTrack, mTransceiver, - mStreams, mDtmf) + mStreams, mTransform, mDtmf) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END NS_IMPL_CYCLE_COLLECTING_ADDREF(RTCRtpSender) @@ -66,7 +121,8 @@ RTCRtpSender::RTCRtpSender(nsPIDOMWindowInner* aWindow, PeerConnectionImpl* aPc, INIT_CANONICAL(mVideoRtpRtcpConfig, Nothing()), INIT_CANONICAL(mVideoCodecMode, webrtc::VideoCodecMode::kRealtimeVideo), INIT_CANONICAL(mCname, std::string()), - INIT_CANONICAL(mTransmitting, false) { + INIT_CANONICAL(mTransmitting, false), + INIT_CANONICAL(mFrameTransformerProxy, nullptr) { mPipeline = MediaPipelineTransmit::Create( mPc->GetHandle(), aTransportHandler, aCallThread, aStsThread, aConduit->type() == MediaSessionConduit::VIDEO, aConduit); @@ -1260,6 +1316,9 @@ void RTCRtpSender::Shutdown() { mWatchManager.Shutdown(); mPipeline->Shutdown(); mPipeline = nullptr; + if (mTransform) { + mTransform->GetProxy().SetSender(nullptr); + } } void RTCRtpSender::BreakCycles() { @@ -1671,6 +1730,50 @@ void RTCRtpSender::UpdateDtmfSender() { mDtmf->StopPlayout(); } +void RTCRtpSender::SetTransform(RTCRtpScriptTransform* aTransform, + ErrorResult& aError) { + if (aTransform == mTransform.get()) { + // Ok... smile and nod + // TODO: Depending on spec, this might throw + // https://github.com/w3c/webrtc-encoded-transform/issues/189 + return; + } + + if (aTransform && aTransform->IsClaimed()) { + aError.ThrowInvalidStateError("transform has already been used elsewhere"); + return; + } + + // Seamless switch for frames + if (aTransform) { + mFrameTransformerProxy = &aTransform->GetProxy(); + } else { + mFrameTransformerProxy = nullptr; + } + + if (mTransform) { + mTransform->GetProxy().SetSender(nullptr); + } + + mTransform = const_cast(aTransform); + + if (mTransform) { + mTransform->GetProxy().SetSender(this); + mTransform->SetClaimed(); + } +} + +bool RTCRtpSender::GenerateKeyFrame(const Maybe& aRid) { + if (!mTransform || !mPipeline || !mSenderTrack) { + return false; + } + + mPipeline->mConduit->AsVideoSessionConduit().apply([&](const auto& conduit) { + conduit->GenerateKeyFrame(aRid, &mTransform->GetProxy()); + }); + return true; +} + } // namespace mozilla::dom #undef LOGTAG diff --git a/dom/media/webrtc/jsapi/RTCRtpSender.h b/dom/media/webrtc/jsapi/RTCRtpSender.h index 3cef3ff9a80c..04411019b117 100644 --- a/dom/media/webrtc/jsapi/RTCRtpSender.h +++ b/dom/media/webrtc/jsapi/RTCRtpSender.h @@ -36,6 +36,7 @@ class RTCDtlsTransport; class RTCDTMFSender; struct RTCRtpCapabilities; class RTCRtpTransceiver; +class RTCRtpScriptTransform; class RTCRtpSender : public nsISupports, public nsWrapperCache, @@ -75,6 +76,11 @@ class RTCRtpSender : public nsISupports, Sequence& aEncodings, bool aVideo, ErrorResult& aRv); + RTCRtpScriptTransform* GetTransform() const { return mTransform; } + + void SetTransform(RTCRtpScriptTransform* aTransform, ErrorResult& aError); + bool GenerateKeyFrame(const Maybe& aRid); + nsPIDOMWindowInner* GetParentObject() const; nsTArray> GetStatsInternal( bool aSkipIceStats = false); @@ -126,6 +132,10 @@ class RTCRtpSender : public nsISupports, Canonical& CanonicalCname() { return mCname; } Canonical& CanonicalTransmitting() override { return mTransmitting; } + Canonical>& CanonicalFrameTransformerProxy() { + return mFrameTransformerProxy; + } + bool HasPendingSetParameters() const { return mPendingParameters.isSome(); } void InvalidateLastReturnedParameters() { mLastReturnedParameters = Nothing(); @@ -171,6 +181,7 @@ class RTCRtpSender : public nsISupports, RefPtr mTransportHandler; RefPtr mTransceiver; nsTArray> mStreams; + RefPtr mTransform; bool mHaveSetupTransport = false; // TODO(bug 1803388): Remove this stuff once it is no longer needed. bool mAllowOldSetParameters = false; @@ -251,6 +262,7 @@ class RTCRtpSender : public nsISupports, Canonical mVideoCodecMode; Canonical mCname; Canonical mTransmitting; + Canonical> mFrameTransformerProxy; }; } // namespace dom diff --git a/dom/media/webrtc/jsapi/RTCRtpTransceiver.cpp b/dom/media/webrtc/jsapi/RTCRtpTransceiver.cpp index 9edfcfd82451..a8973d1fa11c 100644 --- a/dom/media/webrtc/jsapi/RTCRtpTransceiver.cpp +++ b/dom/media/webrtc/jsapi/RTCRtpTransceiver.cpp @@ -3,37 +3,84 @@ * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "jsapi/RTCRtpTransceiver.h" -#include "mozilla/UniquePtr.h" + +#include + #include #include #include -#include "libwebrtcglue/AudioConduit.h" -#include "libwebrtcglue/VideoConduit.h" -#include "MediaTrackGraph.h" -#include "transportbridge/MediaPipeline.h" -#include "transportbridge/MediaPipelineFilter.h" -#include "jsep/JsepTrack.h" -#include "sdp/SdpHelper.h" -#include "MediaTrackGraphImpl.h" -#include "transport/logging.h" -#include "MediaEngine.h" -#include "nsIPrincipal.h" -#include "MediaSegment.h" -#include "RemoteTrackSource.h" -#include "libwebrtcglue/RtpRtcpConfig.h" -#include "MediaTransportHandler.h" +#include +#include +#include +#include + +#include "api/video_codecs/video_codec.h" + +#include "nsCOMPtr.h" +#include "nsContentUtils.h" +#include "nsCycleCollectionParticipant.h" +#include "nsDebug.h" +#include "nsISerialEventTarget.h" +#include "nsISupports.h" +#include "nsProxyRelease.h" +#include "nsStringFwd.h" +#include "nsString.h" +#include "nsTArray.h" +#include "nsThreadUtils.h" +#include "nsWrapperCache.h" +#include "PrincipalHandle.h" +#include "ErrorList.h" +#include "MainThreadUtils.h" +#include "MediaEventSource.h" +#include "mozilla/AbstractThread.h" +#include "mozilla/Assertions.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/fallible.h" +#include "mozilla/Maybe.h" +#include "mozilla/mozalloc_oom.h" +#include "mozilla/MozPromise.h" +#include "mozilla/Preferences.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/StateMirroring.h" +#include "mozilla/RefPtr.h" +#include "mozilla/dom/Nullable.h" +#include "mozilla/dom/RTCStatsReportBinding.h" #include "mozilla/dom/RTCRtpReceiverBinding.h" #include "mozilla/dom/RTCRtpSenderBinding.h" #include "mozilla/dom/RTCRtpTransceiverBinding.h" #include "mozilla/dom/Promise.h" +#include "utils/PerformanceRecorder.h" +#include "systemservices/MediaUtils.h" +#include "MediaTrackGraph.h" +#include "js/RootingAPI.h" +#include "libwebrtcglue/AudioConduit.h" +#include "libwebrtcglue/VideoConduit.h" +#include "transportbridge/MediaPipeline.h" +#include "jsep/JsepTrack.h" +#include "sdp/SdpHelper.h" +#include "transport/logging.h" +#include "RemoteTrackSource.h" +#include "libwebrtcglue/RtpRtcpConfig.h" +#include "MediaTransportHandler.h" #include "RTCDtlsTransport.h" #include "RTCRtpReceiver.h" #include "RTCRtpSender.h" #include "RTCDTMFSender.h" -#include "systemservices/MediaUtils.h" +#include "PeerConnectionImpl.h" +#include "RTCStatsIdGenerator.h" #include "libwebrtcglue/WebrtcCallWrapper.h" -#include "libwebrtcglue/WebrtcGmpVideoCodec.h" -#include "utils/PerformanceRecorder.h" +#include "libwebrtcglue/FrameTransformerProxy.h" +#include "jsep/JsepCodecDescription.h" +#include "jsep/JsepSession.h" +#include "jsep/JsepTrackEncoding.h" +#include "libwebrtcglue/CodecConfig.h" +#include "libwebrtcglue/MediaConduitControl.h" +#include "libwebrtcglue/MediaConduitInterface.h" +#include "RTCStatsReport.h" +#include "sdp/SdpAttribute.h" +#include "sdp/SdpEnum.h" +#include "sdp/SdpMediaSection.h" +#include "transport/transportlayer.h" namespace mozilla { @@ -119,6 +166,15 @@ struct ConduitControlState : public AudioConduitControlInterface, Canonical& CanonicalVideoCodecMode() override { return mSender->CanonicalVideoCodecMode(); } + Canonical>& CanonicalFrameTransformerProxySend() + override { + return mSender->CanonicalFrameTransformerProxy(); + } + + Canonical>& CanonicalFrameTransformerProxyRecv() + override { + return mReceiver->CanonicalFrameTransformerProxy(); + } }; } // namespace diff --git a/dom/media/webrtc/jsapi/RTCTransformEventRunnable.cpp b/dom/media/webrtc/jsapi/RTCTransformEventRunnable.cpp new file mode 100644 index 000000000000..2a96801f91e9 --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCTransformEventRunnable.cpp @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "RTCTransformEventRunnable.h" + +#include "nsIGlobalObject.h" +#include "ErrorList.h" +#include "nsError.h" +#include "nsDebug.h" +#include "nsLiteralString.h" +#include "mozilla/RefPtr.h" +#include "mozilla/AlreadyAddRefed.h" +// This needs to come before RTCTransformEvent.h, since webidl codegen doesn't +// include-what-you-use or forward declare. +#include "mozilla/dom/RTCRtpScriptTransformer.h" +#include "mozilla/dom/RTCTransformEvent.h" +#include "mozilla/dom/RTCTransformEventBinding.h" +#include "mozilla/dom/EventWithOptionsRunnable.h" +#include "mozilla/dom/Event.h" +#include "mozilla/dom/EventTarget.h" +#include "mozilla/dom/RootedDictionary.h" +#include "js/RootingAPI.h" +#include "js/Value.h" +#include "libwebrtcglue/FrameTransformerProxy.h" + +namespace mozilla::dom { + +RTCTransformEventRunnable::RTCTransformEventRunnable( + Worker& aWorker, FrameTransformerProxy* aProxy) + : EventWithOptionsRunnable(aWorker), mProxy(aProxy) {} + +RTCTransformEventRunnable::~RTCTransformEventRunnable() = default; + +already_AddRefed RTCTransformEventRunnable::BuildEvent( + JSContext* aCx, nsIGlobalObject* aGlobal, EventTarget* aTarget, + JS::Handle aTransformerOptions) { + // Let transformerOptions be the result of + // StructuredDeserialize(serializedOptions, the current Realm). + + // NOTE: We do not do this streams stuff. Spec will likely change here. + // The gist here is that we hook [[readable]] and [[writable]] up to the frame + // source/sink, which in out case is FrameTransformerProxy. + // Let readable be the result of StructuredDeserialize(serializedReadable, the + // current Realm). Let writable be the result of + // StructuredDeserialize(serializedWritable, the current Realm). + + // Let transformer be a new RTCRtpScriptTransformer. + + // Set transformer.[[options]] to transformerOptions. + + // Set transformer.[[readable]] to readable. + + // Set transformer.[[writable]] to writable. + RefPtr transformer = + new RTCRtpScriptTransformer(aGlobal); + nsresult nrv = + transformer->Init(aCx, aTransformerOptions, mWorkerPrivate, mProxy); + if (NS_WARN_IF(NS_FAILED(nrv))) { + // TODO: Error handling. Currently unspecified. + return nullptr; + } + + // Fire an event named rtctransform using RTCTransformEvent with transformer + // set to transformer on worker’s global scope. + RootedDictionary init(aCx); + init.mBubbles = false; + init.mCancelable = false; + init.mTransformer = transformer; + + RefPtr event = + RTCTransformEvent::Constructor(aTarget, u"rtctransform"_ns, init); + event->SetTrusted(true); + return event.forget(); +} + +} // namespace mozilla::dom diff --git a/dom/media/webrtc/jsapi/RTCTransformEventRunnable.h b/dom/media/webrtc/jsapi/RTCTransformEventRunnable.h new file mode 100644 index 000000000000..763d7c923773 --- /dev/null +++ b/dom/media/webrtc/jsapi/RTCTransformEventRunnable.h @@ -0,0 +1,38 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCTRANSFORMEVENTRUNNABLE_H_ +#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCTRANSFORMEVENTRUNNABLE_H_ + +#include "mozilla/dom/EventWithOptionsRunnable.h" + +namespace mozilla { + +class FrameTransformerProxy; + +namespace dom { + +// Cargo-culted from MesssageEventRunnable. +// TODO: Maybe could subclass WorkerRunnable instead? Comments on +// WorkerDebuggeeRunnable indicate that firing an event at JS means we need that +// class. +class RTCTransformEventRunnable final : public EventWithOptionsRunnable { + public: + RTCTransformEventRunnable(Worker& aWorker, FrameTransformerProxy* aProxy); + + already_AddRefed BuildEvent( + JSContext* aCx, nsIGlobalObject* aGlobal, EventTarget* aTarget, + JS::Handle aTransformerOptions) override; + + private: + virtual ~RTCTransformEventRunnable(); + RefPtr mProxy; +}; + +} // namespace dom +} // namespace mozilla + +#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCTRANSFORMEVENTRUNNABLE_H_ diff --git a/dom/media/webrtc/jsapi/moz.build b/dom/media/webrtc/jsapi/moz.build index 2c1dbe79f19a..78a6241cd65e 100644 --- a/dom/media/webrtc/jsapi/moz.build +++ b/dom/media/webrtc/jsapi/moz.build @@ -28,12 +28,18 @@ UNIFIED_SOURCES += [ "RemoteTrackSource.cpp", "RTCDtlsTransport.cpp", "RTCDTMFSender.cpp", + "RTCEncodedAudioFrame.cpp", + "RTCEncodedFrameBase.cpp", + "RTCEncodedVideoFrame.cpp", "RTCRtpReceiver.cpp", + "RTCRtpScriptTransform.cpp", + "RTCRtpScriptTransformer.cpp", "RTCRtpSender.cpp", "RTCRtpTransceiver.cpp", "RTCSctpTransport.cpp", "RTCStatsIdGenerator.cpp", "RTCStatsReport.cpp", + "RTCTransformEventRunnable.cpp", "WebrtcGlobalInformation.cpp", "WebrtcGlobalStatsHistory.cpp", ] @@ -41,7 +47,12 @@ UNIFIED_SOURCES += [ EXPORTS.mozilla.dom += [ "RTCDtlsTransport.h", "RTCDTMFSender.h", + "RTCEncodedAudioFrame.h", + "RTCEncodedFrameBase.h", + "RTCEncodedVideoFrame.h", "RTCRtpReceiver.h", + "RTCRtpScriptTransform.h", + "RTCRtpScriptTransformer.h", "RTCRtpSender.h", "RTCRtpTransceiver.h", "RTCSctpTransport.h", diff --git a/dom/media/webrtc/libwebrtcglue/AudioConduit.cpp b/dom/media/webrtc/libwebrtcglue/AudioConduit.cpp index 57c3108ce24a..8f74aca45826 100644 --- a/dom/media/webrtc/libwebrtcglue/AudioConduit.cpp +++ b/dom/media/webrtc/libwebrtcglue/AudioConduit.cpp @@ -6,16 +6,57 @@ #include "common/browser_logging/CSFLog.h" #include "MediaConduitControl.h" -#include "mozilla/media/MediaUtils.h" -#include "mozilla/Telemetry.h" -#include "transport/runnable_utils.h" #include "transport/SrtpFlow.h" // For SRTP_MAX_EXPANSION #include "WebrtcCallWrapper.h" +#include "libwebrtcglue/FrameTransformer.h" +#include +#include "CodecConfig.h" +#include "mozilla/StateMirroring.h" +#include +#include "mozilla/MozPromise.h" +#include "mozilla/RefPtr.h" +#include "mozilla/RWLock.h" // libwebrtc includes #include "api/audio_codecs/builtin_audio_encoder_factory.h" #include "audio/audio_receive_stream.h" #include "media/base/media_constants.h" +#include "rtc_base/ref_counted_object.h" + +#include "api/audio/audio_frame.h" +#include "api/audio/audio_mixer.h" +#include "api/audio_codecs/audio_format.h" +#include "api/call/transport.h" +#include "api/media_types.h" +#include "api/rtp_headers.h" +#include "api/rtp_parameters.h" +#include "api/transport/rtp/rtp_source.h" +#include +#include "call/audio_receive_stream.h" +#include "call/audio_send_stream.h" +#include "call/call_basic_stats.h" +#include "domstubs.h" +#include "jsapi/RTCStatsReport.h" +#include +#include "MainThreadUtils.h" +#include +#include "MediaConduitErrors.h" +#include "MediaConduitInterface.h" +#include +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "mozilla/Assertions.h" +#include "mozilla/Atomics.h" +#include "mozilla/Maybe.h" +#include "mozilla/StateWatching.h" +#include "nsCOMPtr.h" +#include "nsError.h" +#include "nsISerialEventTarget.h" +#include "nsThreadUtils.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/network/sent_packet.h" +#include +#include +#include "transport/mediapacket.h" // for ntohs #ifdef HAVE_NETINET_IN_H @@ -71,7 +112,9 @@ WebrtcAudioConduit::Control::Control(const RefPtr& aCallThread) INIT_MIRROR(mLocalRecvRtpExtensions, RtpExtList()), INIT_MIRROR(mLocalSendRtpExtensions, RtpExtList()), INIT_MIRROR(mSendCodec, Nothing()), - INIT_MIRROR(mRecvCodecs, std::vector()) {} + INIT_MIRROR(mRecvCodecs, std::vector()), + INIT_MIRROR(mFrameTransformerProxySend, nullptr), + INIT_MIRROR(mFrameTransformerProxyRecv, nullptr) {} #undef INIT_MIRROR RefPtr WebrtcAudioConduit::Shutdown() { @@ -81,28 +124,30 @@ RefPtr WebrtcAudioConduit::Shutdown() { return InvokeAsync(mCallThread, "WebrtcAudioConduit::Shutdown (main thread)", [this, self = RefPtr(this)] { - mControl.mReceiving.DisconnectIfConnected(); - mControl.mTransmitting.DisconnectIfConnected(); - mControl.mLocalSsrcs.DisconnectIfConnected(); - mControl.mLocalCname.DisconnectIfConnected(); - mControl.mMid.DisconnectIfConnected(); - mControl.mRemoteSsrc.DisconnectIfConnected(); - mControl.mSyncGroup.DisconnectIfConnected(); - mControl.mLocalRecvRtpExtensions.DisconnectIfConnected(); - mControl.mLocalSendRtpExtensions.DisconnectIfConnected(); - mControl.mSendCodec.DisconnectIfConnected(); - mControl.mRecvCodecs.DisconnectIfConnected(); - mWatchManager.Shutdown(); + mControl.mReceiving.DisconnectIfConnected(); + mControl.mTransmitting.DisconnectIfConnected(); + mControl.mLocalSsrcs.DisconnectIfConnected(); + mControl.mLocalCname.DisconnectIfConnected(); + mControl.mMid.DisconnectIfConnected(); + mControl.mRemoteSsrc.DisconnectIfConnected(); + mControl.mSyncGroup.DisconnectIfConnected(); + mControl.mLocalRecvRtpExtensions.DisconnectIfConnected(); + mControl.mLocalSendRtpExtensions.DisconnectIfConnected(); + mControl.mSendCodec.DisconnectIfConnected(); + mControl.mRecvCodecs.DisconnectIfConnected(); + mControl.mFrameTransformerProxySend.DisconnectIfConnected(); + mControl.mFrameTransformerProxyRecv.DisconnectIfConnected(); + mWatchManager.Shutdown(); - { - AutoWriteLock lock(mLock); - DeleteSendStream(); - DeleteRecvStream(); - } + { + AutoWriteLock lock(mLock); + DeleteSendStream(); + DeleteRecvStream(); + } - return GenericPromise::CreateAndResolve( - true, "WebrtcAudioConduit::Shutdown (call thread)"); - }); + return GenericPromise::CreateAndResolve( + true, "WebrtcAudioConduit::Shutdown (call thread)"); + }); } WebrtcAudioConduit::WebrtcAudioConduit( @@ -163,6 +208,10 @@ void WebrtcAudioConduit::InitControl(AudioConduitControlInterface* aControl) { mControl.mLocalSendRtpExtensions); CONNECT(aControl->CanonicalAudioSendCodec(), mControl.mSendCodec); CONNECT(aControl->CanonicalAudioRecvCodecs(), mControl.mRecvCodecs); + CONNECT(aControl->CanonicalFrameTransformerProxySend(), + mControl.mFrameTransformerProxySend); + CONNECT(aControl->CanonicalFrameTransformerProxyRecv(), + mControl.mFrameTransformerProxyRecv); mControl.mOnDtmfEventListener = aControl->OnDtmfEvent().Connect( mCall->mCallThread, this, &WebrtcAudioConduit::OnDtmfEvent); } @@ -288,6 +337,32 @@ void WebrtcAudioConduit::OnControlConfigChange() { recvStreamReconfigureNeeded = true; } + if (mControl.mConfiguredFrameTransformerProxySend.get() != + mControl.mFrameTransformerProxySend.Ref().get()) { + mControl.mConfiguredFrameTransformerProxySend = + mControl.mFrameTransformerProxySend.Ref(); + if (!mSendStreamConfig.frame_transformer) { + mSendStreamConfig.frame_transformer = + new rtc::RefCountedObject(false); + sendStreamRecreationNeeded = true; + } + static_cast(mSendStreamConfig.frame_transformer.get()) + ->SetProxy(mControl.mConfiguredFrameTransformerProxySend); + } + + if (mControl.mConfiguredFrameTransformerProxyRecv.get() != + mControl.mFrameTransformerProxyRecv.Ref().get()) { + mControl.mConfiguredFrameTransformerProxyRecv = + mControl.mFrameTransformerProxyRecv.Ref(); + if (!mRecvStreamConfig.frame_transformer) { + mRecvStreamConfig.frame_transformer = + new rtc::RefCountedObject(false); + recvStreamRecreationNeeded = true; + } + static_cast(mRecvStreamConfig.frame_transformer.get()) + ->SetProxy(mControl.mConfiguredFrameTransformerProxyRecv); + } + if (!recvStreamReconfigureNeeded && !sendStreamReconfigureNeeded && !recvStreamRecreationNeeded && !sendStreamRecreationNeeded && mControl.mReceiving == mRecvStreamRunning && diff --git a/dom/media/webrtc/libwebrtcglue/AudioConduit.h b/dom/media/webrtc/libwebrtcglue/AudioConduit.h index e8de331e12bc..807f6c8b4057 100644 --- a/dom/media/webrtc/libwebrtcglue/AudioConduit.h +++ b/dom/media/webrtc/libwebrtcglue/AudioConduit.h @@ -256,6 +256,8 @@ class WebrtcAudioConduit : public AudioSessionConduit, Mirror mLocalSendRtpExtensions; Mirror> mSendCodec; Mirror> mRecvCodecs; + Mirror> mFrameTransformerProxySend; + Mirror> mFrameTransformerProxyRecv; MediaEventListener mOnDtmfEventListener; // For caching mRemoteSsrc, since another caller may change the remote ssrc @@ -266,6 +268,10 @@ class WebrtcAudioConduit : public AudioSessionConduit, // For tracking changes to mRecvCodecs. std::vector mConfiguredRecvCodecs; + // For change tracking. Callthread only. + RefPtr mConfiguredFrameTransformerProxySend; + RefPtr mConfiguredFrameTransformerProxyRecv; + Control() = delete; explicit Control(const RefPtr& aCallThread); } mControl; diff --git a/dom/media/webrtc/libwebrtcglue/FrameTransformer.cpp b/dom/media/webrtc/libwebrtcglue/FrameTransformer.cpp new file mode 100644 index 000000000000..23688a7d88f9 --- /dev/null +++ b/dom/media/webrtc/libwebrtcglue/FrameTransformer.cpp @@ -0,0 +1,87 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "libwebrtcglue/FrameTransformer.h" +#include "api/frame_transformer_interface.h" +#include "mozilla/Mutex.h" +#include +#include +#include "api/scoped_refptr.h" +#include +#include "libwebrtcglue/FrameTransformerProxy.h" + +namespace mozilla { + +FrameTransformer::FrameTransformer(bool aVideo) + : webrtc::FrameTransformerInterface(), + mVideo(aVideo), + mCallbacksMutex("FrameTransformer::mCallbacksMutex"), + mProxyMutex("FrameTransformer::mProxyMutex") {} + +FrameTransformer::~FrameTransformer() { + if (mProxy) { + mProxy->SetLibwebrtcTransformer(nullptr); + } +} + +void FrameTransformer::Transform( + std::unique_ptr aFrame) { + MutexAutoLock lock(mProxyMutex); + if (mProxy) { + mProxy->Transform(std::move(aFrame)); + return; + } + + // No transformer, just passthrough + OnTransformedFrame(std::move(aFrame)); +} + +void FrameTransformer::RegisterTransformedFrameCallback( + rtc::scoped_refptr aCallback) { + MutexAutoLock lock(mCallbacksMutex); + mCallback = aCallback; +} + +void FrameTransformer::UnregisterTransformedFrameCallback() { + MutexAutoLock lock(mCallbacksMutex); + mCallback = nullptr; +} + +void FrameTransformer::RegisterTransformedFrameSinkCallback( + rtc::scoped_refptr aCallback, + uint32_t aSsrc) { + MutexAutoLock lock(mCallbacksMutex); + mCallbacksBySsrc[aSsrc] = aCallback; +} + +void FrameTransformer::UnregisterTransformedFrameSinkCallback(uint32_t aSsrc) { + MutexAutoLock lock(mCallbacksMutex); + mCallbacksBySsrc.erase(aSsrc); +} + +void FrameTransformer::OnTransformedFrame( + std::unique_ptr aFrame) { + MutexAutoLock lock(mCallbacksMutex); + if (mCallback) { + mCallback->OnTransformedFrame(std::move(aFrame)); + } else if (auto it = mCallbacksBySsrc.find(aFrame->GetSsrc()); + it != mCallbacksBySsrc.end()) { + it->second->OnTransformedFrame(std::move(aFrame)); + } +} + +void FrameTransformer::SetProxy(FrameTransformerProxy* aProxy) { + MutexAutoLock lock(mProxyMutex); + if (mProxy) { + mProxy->SetLibwebrtcTransformer(nullptr); + } + mProxy = aProxy; + if (mProxy) { + mProxy->SetLibwebrtcTransformer(this); + } +} + +} // namespace mozilla diff --git a/dom/media/webrtc/libwebrtcglue/FrameTransformer.h b/dom/media/webrtc/libwebrtcglue/FrameTransformer.h new file mode 100644 index 000000000000..0c93d0f77f9b --- /dev/null +++ b/dom/media/webrtc/libwebrtcglue/FrameTransformer.h @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_DOM_MEDIA_WEBRTC_LIBWEBRTCGLUE_FRAMETRANSFORMER_H_ +#define MOZILLA_DOM_MEDIA_WEBRTC_LIBWEBRTCGLUE_FRAMETRANSFORMER_H_ + +#include "api/frame_transformer_interface.h" +#include "libwebrtcglue/FrameTransformerProxy.h" +#include "nsISupportsImpl.h" +#include "mozilla/Mutex.h" +#include "jsapi/RTCRtpScriptTransformer.h" + +namespace mozilla { + +// There is one of these per RTCRtpSender and RTCRtpReceiver, for its entire +// lifetime. SetProxy is used to activate/deactivate it. In the inactive state +// (the default), this is just a synchronous passthrough. +class FrameTransformer : public webrtc::FrameTransformerInterface { + public: + explicit FrameTransformer(bool aVideo); + virtual ~FrameTransformer(); + + // This is set when RTCRtpSender/Receiver.transform is set, and unset when + // RTCRtpSender/Receiver.transform is unset. + void SetProxy(FrameTransformerProxy* aProxy); + + // If no proxy is set (ie; RTCRtpSender/Receiver.transform is not set), this + // synchronously calls OnTransformedFrame with no modifcation. If a proxy is + // set, we send the frame to it, and eventually that frame should come back + // to OnTransformedFrame. + void Transform( + std::unique_ptr aFrame) override; + void OnTransformedFrame( + std::unique_ptr aFrame); + + // When libwebrtc uses the same callback for all ssrcs + // (right now, this is used for audio, but we do not care in this class) + void RegisterTransformedFrameCallback( + rtc::scoped_refptr aCallback) override; + void UnregisterTransformedFrameCallback() override; + + // When libwebrtc uses a different callback for each ssrc + // (right now, this is used for video, but we do not care in this class) + void RegisterTransformedFrameSinkCallback( + rtc::scoped_refptr aCallback, + uint32_t aSsrc) override; + void UnregisterTransformedFrameSinkCallback(uint32_t aSsrc) override; + + bool IsVideo() const { return mVideo; } + + private: + const bool mVideo; + Mutex mCallbacksMutex; + // Written on a libwebrtc thread, read on the worker thread. + rtc::scoped_refptr mCallback + MOZ_GUARDED_BY(mCallbacksMutex); + std::map> + mCallbacksBySsrc MOZ_GUARDED_BY(mCallbacksMutex); + + Mutex mProxyMutex; + // Written on the call thread, read on a libwebrtc/gmp/mediadataencoder/call + // thread (which one depends on the media type and direction). Right now, + // these are: + // Send video: VideoStreamEncoder::encoder_queue_, + // WebrtcMediaDataEncoder::mTaskQueue, or GMP encoder thread. + // Recv video: Call::worker_thread_ + // Send audio: ChannelSend::encoder_queue_ + // Recv audio: ChannelReceive::worker_thread_ + // This should have little to no lock contention + // This corresponds to the RTCRtpScriptTransform/RTCRtpScriptTransformer. + RefPtr mProxy MOZ_GUARDED_BY(mProxyMutex); +}; // FrameTransformer + +} // namespace mozilla + +#endif // MOZILLA_DOM_MEDIA_WEBRTC_LIBWEBRTCGLUE_FRAMETRANSFORMER_H_ diff --git a/dom/media/webrtc/libwebrtcglue/FrameTransformerProxy.cpp b/dom/media/webrtc/libwebrtcglue/FrameTransformerProxy.cpp new file mode 100644 index 000000000000..f374cda6998d --- /dev/null +++ b/dom/media/webrtc/libwebrtcglue/FrameTransformerProxy.cpp @@ -0,0 +1,258 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "libwebrtcglue/FrameTransformerProxy.h" +#include "libwebrtcglue/FrameTransformer.h" +#include "mozilla/dom/RTCRtpSender.h" +#include "mozilla/dom/RTCRtpReceiver.h" +#include "mozilla/Logging.h" +#include "mozilla/Mutex.h" +#include "jsapi/RTCRtpScriptTransformer.h" +#include "nsThreadUtils.h" +#include "mozilla/Assertions.h" +#include +#include "mozilla/Maybe.h" +#include "mozilla/RefPtr.h" +#include "nscore.h" +#include "ErrorList.h" +#include "nsIRunnable.h" +#include "nsIEventTarget.h" +#include "api/frame_transformer_interface.h" +#include +#include "nsDebug.h" +#include "nsISupports.h" +#include + +namespace mozilla { + +LazyLogModule gFrameTransformerProxyLog("FrameTransformerProxy"); + +FrameTransformerProxy::FrameTransformerProxy() + : mMutex("FrameTransformerProxy::mMutex") {} + +FrameTransformerProxy::~FrameTransformerProxy() = default; + +void FrameTransformerProxy::SetScriptTransformer( + dom::RTCRtpScriptTransformer& aTransformer) { + MutexAutoLock lock(mMutex); + if (mReleaseScriptTransformerCalled) { + MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Warning, + ("RTCRtpScriptTransformer is ready, but ReleaseScriptTransformer " + "has already been called.")); + // The mainthread side has torn down while the worker init was pending. + // Don't grab a reference to the worker thread, or the script transformer. + // Also, let the script transformer know that we do not need it after all. + aTransformer.NotifyReleased(); + return; + } + + MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Info, + ("RTCRtpScriptTransformer is ready!")); + mWorkerThread = GetCurrentSerialEventTarget(); + MOZ_ASSERT(mWorkerThread); + + MOZ_ASSERT(!mScriptTransformer); + mScriptTransformer = &aTransformer; + while (!mQueue.empty()) { + mScriptTransformer->TransformFrame(std::move(mQueue.front())); + mQueue.pop_front(); + } +} + +Maybe FrameTransformerProxy::IsVideo() const { + MutexAutoLock lock(mMutex); + return mVideo; +} + +void FrameTransformerProxy::ReleaseScriptTransformer() { + MutexAutoLock lock(mMutex); + MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Debug, ("In %s", __FUNCTION__)); + if (mReleaseScriptTransformerCalled) { + return; + } + mReleaseScriptTransformerCalled = true; + + if (mWorkerThread) { + mWorkerThread->Dispatch(NS_NewRunnableFunction( + __func__, [this, self = RefPtr(this)] { + if (mScriptTransformer) { + mScriptTransformer->NotifyReleased(); + mScriptTransformer = nullptr; + } + + // Make sure cycles are broken; this unset might have been caused by + // something other than the sender/receiver being unset. + GetMainThreadSerialEventTarget()->Dispatch( + NS_NewRunnableFunction(__func__, [this, self] { + MutexAutoLock lock(mMutex); + mSender = nullptr; + mReceiver = nullptr; + })); + })); + mWorkerThread = nullptr; + } +} + +void FrameTransformerProxy::SetLibwebrtcTransformer( + FrameTransformer* aLibwebrtcTransformer) { + MutexAutoLock lock(mMutex); + mLibwebrtcTransformer = aLibwebrtcTransformer; + if (mLibwebrtcTransformer) { + MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Info, + ("mLibwebrtcTransformer is now set!")); + mVideo = Some(mLibwebrtcTransformer->IsVideo()); + } +} + +void FrameTransformerProxy::Transform( + std::unique_ptr aFrame) { + MutexAutoLock lock(mMutex); + MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Debug, ("In %s", __FUNCTION__)); + if (!mWorkerThread && !mReleaseScriptTransformerCalled) { + MOZ_LOG( + gFrameTransformerProxyLog, LogLevel::Info, + ("In %s, queueing frame because RTCRtpScriptTransformer is not ready", + __FUNCTION__)); + // We are still waiting for the script transformer to be created on the + // worker thread. + mQueue.push_back(std::move(aFrame)); + return; + } + + if (mWorkerThread) { + MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Debug, + ("Queueing call to RTCRtpScriptTransformer::TransformFrame")); + mWorkerThread->Dispatch(NS_NewRunnableFunction( + __func__, [this, self = RefPtr(this), + frame = std::move(aFrame)]() mutable { + if (NS_WARN_IF(!mScriptTransformer)) { + // Could happen due to errors. Is there some + // other processing we ought to do? + return; + } + mScriptTransformer->TransformFrame(std::move(frame)); + })); + } +} + +void FrameTransformerProxy::OnTransformedFrame( + std::unique_ptr aFrame) { + MutexAutoLock lock(mMutex); + // If the worker thread has changed, we drop the frame, to avoid frames + // arriving out of order. + if (mLibwebrtcTransformer) { + // This will lock, lock order is mMutex, FrameTransformer::mLibwebrtcMutex + mLibwebrtcTransformer->OnTransformedFrame(std::move(aFrame)); + } +} + +void FrameTransformerProxy::SetSender(dom::RTCRtpSender* aSender) { + { + MutexAutoLock lock(mMutex); + MOZ_ASSERT(!mReceiver); + mSender = aSender; + } + if (!aSender) { + MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Info, ("Sender set to null")); + ReleaseScriptTransformer(); + } +} + +void FrameTransformerProxy::SetReceiver(dom::RTCRtpReceiver* aReceiver) { + { + MutexAutoLock lock(mMutex); + MOZ_ASSERT(!mSender); + mReceiver = aReceiver; + } + if (!aReceiver) { + MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Info, + ("Receiver set to null")); + ReleaseScriptTransformer(); + } +} + +bool FrameTransformerProxy::RequestKeyFrame() { + { + // Spec wants this to reject synchronously if the RTCRtpScriptTransformer + // is not associated with a video receiver. This may change to an async + // check? + MutexAutoLock lock(mMutex); + if (!mReceiver || !mVideo.isSome() || !*mVideo) { + return false; + } + } + + // Thread hop to main, and then the conduit thread-hops to the call thread. + GetMainThreadSerialEventTarget()->Dispatch(NS_NewRunnableFunction( + __func__, [this, self = RefPtr(this)] { + MutexAutoLock lock(mMutex); + if (mReceiver && mVideo.isSome() && *mVideo) { + mReceiver->RequestKeyFrame(); + } + })); + return true; +} + +void FrameTransformerProxy::KeyFrameRequestDone(bool aSuccess) { + MutexAutoLock lock(mMutex); + if (mWorkerThread) { + mWorkerThread->Dispatch(NS_NewRunnableFunction( + __func__, [this, self = RefPtr(this), aSuccess] { + if (mScriptTransformer) { + mScriptTransformer->KeyFrameRequestDone(aSuccess); + } + })); + } +} + +bool FrameTransformerProxy::GenerateKeyFrame(const Maybe& aRid) { + { + // Spec wants this to reject synchronously if the RTCRtpScriptTransformer + // is not associated with a video sender. This may change to an async + // check? + MutexAutoLock lock(mMutex); + if (!mSender || !mVideo.isSome() || !*mVideo) { + return false; + } + } + + // Thread hop to main, and then the conduit thread-hops to the call thread. + GetMainThreadSerialEventTarget()->Dispatch(NS_NewRunnableFunction( + __func__, [this, self = RefPtr(this), aRid] { + MutexAutoLock lock(mMutex); + if (!mSender || !mVideo.isSome() || !*mVideo || + !mSender->GenerateKeyFrame(aRid)) { + CopyableErrorResult rv; + rv.ThrowInvalidStateError("Not sending video"); + if (mWorkerThread) { + mWorkerThread->Dispatch(NS_NewRunnableFunction( + __func__, + [this, self = RefPtr(this), aRid, rv] { + if (mScriptTransformer) { + mScriptTransformer->GenerateKeyFrameError(aRid, rv); + } + })); + } + } + })); + return true; +} + +void FrameTransformerProxy::GenerateKeyFrameError( + const Maybe& aRid, const CopyableErrorResult& aResult) { + MutexAutoLock lock(mMutex); + if (mWorkerThread) { + mWorkerThread->Dispatch(NS_NewRunnableFunction( + __func__, + [this, self = RefPtr(this), aRid, aResult] { + if (mScriptTransformer) { + mScriptTransformer->GenerateKeyFrameError(aRid, aResult); + } + })); + } +} + +} // namespace mozilla diff --git a/dom/media/webrtc/libwebrtcglue/FrameTransformerProxy.h b/dom/media/webrtc/libwebrtcglue/FrameTransformerProxy.h new file mode 100644 index 000000000000..72617fcde9ff --- /dev/null +++ b/dom/media/webrtc/libwebrtcglue/FrameTransformerProxy.h @@ -0,0 +1,124 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_DOM_MEDIA_WEBRTC_LIBWEBRTCGLUE_FRAMETRANSFORMERPROXY_H_ +#define MOZILLA_DOM_MEDIA_WEBRTC_LIBWEBRTCGLUE_FRAMETRANSFORMERPROXY_H_ + +#include "nsISupportsImpl.h" +#include "mozilla/Mutex.h" +#include "mozilla/Maybe.h" +#include +#include + +class nsIEventTarget; + +namespace webrtc { +class TransformableFrameInterface; +class VideoReceiveStreamInterface; +} // namespace webrtc + +namespace mozilla { + +class FrameTransformer; +class WebrtcVideoConduit; +class CopyableErrorResult; + +namespace dom { +class RTCRtpScriptTransformer; +class RTCRtpSender; +class RTCRtpReceiver; +} // namespace dom + +// This corresponds to a single RTCRtpScriptTransform (and its +// RTCRtpScriptTransformer, once that is created on the worker thread). This +// is intended to decouple threading/lifecycle/include-dependencies between +// FrameTransformer (on the libwebrtc side of things), RTCRtpScriptTransformer +// (on the worker side of things), RTCRtpScriptTransform and +// RTCRtpSender/Receiver (on the main thread), and prevents frames from being +// lost while we're setting things up on the worker. In other words, this +// handles the inconvenient stuff. +class FrameTransformerProxy { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(FrameTransformerProxy); + + FrameTransformerProxy(); + FrameTransformerProxy(const FrameTransformerProxy& aRhs) = delete; + FrameTransformerProxy(FrameTransformerProxy&& aRhs) = delete; + FrameTransformerProxy& operator=(const FrameTransformerProxy& aRhs) = delete; + FrameTransformerProxy& operator=(FrameTransformerProxy&& aRhs) = delete; + + // Called at most once (might not be called if the worker is shutting down), + // on the worker thread. + void SetScriptTransformer(dom::RTCRtpScriptTransformer& aTransformer); + + // Can be called from the worker thread (if the worker is shutting down), or + // main (if RTCRtpSender/RTCRtpReceiver is done with us). + void ReleaseScriptTransformer(); + + // RTCRtpScriptTransformer calls this when it is done transforming a frame. + void OnTransformedFrame( + std::unique_ptr aFrame); + + Maybe IsVideo() const; + + // Called by FrameTransformer, on main. Only one FrameTransformer will ever + // be registered over the lifetime of this object. This is where we route + // transformed frames. If this is set, we can also expect to receive calls to + // Transform. + void SetLibwebrtcTransformer(FrameTransformer* aLibwebrtcTransformer); + + // FrameTransformer calls this while we're registered with it (by + // SetLibwebrtcTransformer) + void Transform(std::unique_ptr aFrame); + + void SetSender(dom::RTCRtpSender* aSender); + void SetReceiver(dom::RTCRtpReceiver* aReceiver); + + // Called on worker thread + bool RequestKeyFrame(); + // Called on call thread + void KeyFrameRequestDone(bool aSuccess); + + bool GenerateKeyFrame(const Maybe& aRid); + void GenerateKeyFrameError(const Maybe& aRid, + const CopyableErrorResult& aResult); + + private: + virtual ~FrameTransformerProxy(); + + // Worker thread only. Set at most once. + // Does not need any mutex protection. + RefPtr mScriptTransformer; + + mutable Mutex mMutex; + // Written on the worker thread. Read on libwebrtc threads, mainthread, and + // the worker thread. + RefPtr mWorkerThread MOZ_GUARDED_BY(mMutex); + // We need a flag for this in case the ReleaseScriptTransformer call comes + // _before_ the script transformer is set, to disable SetScriptTransformer. + // Could be written on main or the worker thread. Read on main, worker, and + // libwebrtc threads. + bool mReleaseScriptTransformerCalled MOZ_GUARDED_BY(mMutex) = false; + // Used when frames arrive before the script transformer is created, which + // should be pretty rare. Accessed on worker and libwebrtc threads. + std::list> mQueue + MOZ_GUARDED_BY(mMutex); + // Written on main, read on the worker thread. + FrameTransformer* mLibwebrtcTransformer MOZ_GUARDED_BY(mMutex) = nullptr; + + // TODO: Will be used to route GenerateKeyFrame. Details TBD. + RefPtr mSender MOZ_GUARDED_BY(mMutex); + // Set on mainthread. This is where we route RequestKeyFrame calls from the + // worker thread. Mutex protected because spec wants sync errors if the + // receiver is not set (or the right type). If spec drops this requirement, + // this could be mainthread only and non-mutex-protected. + RefPtr mReceiver MOZ_GUARDED_BY(mMutex); + Maybe mVideo MOZ_GUARDED_BY(mMutex); +}; + +} // namespace mozilla + +#endif // MOZILLA_DOM_MEDIA_WEBRTC_LIBWEBRTCGLUE_FRAMETRANSFORMERPROXY_H_ diff --git a/dom/media/webrtc/libwebrtcglue/MediaConduitControl.h b/dom/media/webrtc/libwebrtcglue/MediaConduitControl.h index 5df3f87bc161..ab38d8d623ad 100644 --- a/dom/media/webrtc/libwebrtcglue/MediaConduitControl.h +++ b/dom/media/webrtc/libwebrtcglue/MediaConduitControl.h @@ -16,6 +16,7 @@ #include "CodecConfig.h" // For Audio/VideoCodecConfig #include "api/rtp_parameters.h" // For webrtc::RtpExtension #include "api/video_codecs/video_codec.h" // For webrtc::VideoCodecMode +#include "FrameTransformerProxy.h" namespace mozilla { @@ -45,6 +46,10 @@ class MediaConduitControlInterface { virtual Canonical& CanonicalSyncGroup() = 0; virtual Canonical& CanonicalLocalRecvRtpExtensions() = 0; virtual Canonical& CanonicalLocalSendRtpExtensions() = 0; + virtual Canonical>& + CanonicalFrameTransformerProxySend() = 0; + virtual Canonical>& + CanonicalFrameTransformerProxyRecv() = 0; }; class AudioConduitControlInterface : public MediaConduitControlInterface { diff --git a/dom/media/webrtc/libwebrtcglue/MediaConduitInterface.h b/dom/media/webrtc/libwebrtcglue/MediaConduitInterface.h index dc78efb8d8b4..4d73e82c91c5 100644 --- a/dom/media/webrtc/libwebrtcglue/MediaConduitInterface.h +++ b/dom/media/webrtc/libwebrtcglue/MediaConduitInterface.h @@ -55,6 +55,7 @@ enum class MediaSessionConduitLocalDirection : int { kSend, kRecv }; class VideoSessionConduit; class AudioSessionConduit; class WebrtcCallWrapper; +class FrameTransformerProxy; /** * 1. Abstract renderer for video data @@ -413,6 +414,10 @@ class VideoSessionConduit : public MediaSessionConduit { }; virtual Maybe GetLastResolution() const = 0; + virtual void RequestKeyFrame(FrameTransformerProxy* aProxy) = 0; + virtual void GenerateKeyFrame(const Maybe& aRid, + FrameTransformerProxy* aProxy) = 0; + protected: /* RTCP feedback settings, for unit testing purposes */ FrameRequestType mFrameRequestMethod; diff --git a/dom/media/webrtc/libwebrtcglue/VideoConduit.cpp b/dom/media/webrtc/libwebrtcglue/VideoConduit.cpp index fc51a9e51868..e97f8761b91b 100644 --- a/dom/media/webrtc/libwebrtcglue/VideoConduit.cpp +++ b/dom/media/webrtc/libwebrtcglue/VideoConduit.cpp @@ -5,28 +5,28 @@ #include "VideoConduit.h" #include -#include #include #include "common/browser_logging/CSFLog.h" #include "common/YuvStamper.h" -#include "GmpVideoCodec.h" #include "MediaConduitControl.h" -#include "MediaDataCodec.h" -#include "mozilla/dom/RTCRtpSourcesBinding.h" -#include "mozilla/media/MediaUtils.h" -#include "mozilla/StaticPrefs_media.h" -#include "mozilla/TemplateLib.h" #include "nsIGfxInfo.h" -#include "nsIPrefBranch.h" -#include "nsIPrefService.h" #include "nsServiceManagerUtils.h" #include "RtpRtcpConfig.h" #include "transport/SrtpFlow.h" // For SRTP_MAX_EXPANSION #include "Tracing.h" #include "VideoStreamFactory.h" #include "WebrtcCallWrapper.h" -#include "WebrtcGmpVideoCodec.h" +#include "libwebrtcglue/FrameTransformer.h" +#include "libwebrtcglue/FrameTransformerProxy.h" +#include "mozilla/StateMirroring.h" +#include "mozilla/RefPtr.h" +#include "nsThreadUtils.h" +#include "mozilla/Maybe.h" +#include "mozilla/ErrorResult.h" +#include +#include +#include // libwebrtc includes #include "api/transport/bitrate_settings.h" @@ -36,8 +36,70 @@ #include "media/base/media_constants.h" #include "media/engine/simulcast_encoder_adapter.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" -#include "modules/video_coding/codecs/vp8/include/vp8.h" -#include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "rtc_base/ref_counted_object.h" + +#include "api/call/transport.h" +#include "api/media_types.h" +#include "api/rtp_headers.h" +#include "api/rtp_parameters.h" +#include "api/scoped_refptr.h" +#include "api/transport/rtp/rtp_source.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video/video_codec_constants.h" +#include "api/video/video_codec_type.h" +#include "api/video/video_frame_buffer.h" +#include "api/video/video_sink_interface.h" +#include "api/video/video_source_interface.h" +#include +#include "call/call.h" +#include "call/rtp_config.h" +#include "call/video_receive_stream.h" +#include "call/video_send_stream.h" +#include "CodecConfig.h" +#include "common_video/include/video_frame_buffer_pool.h" +#include "domstubs.h" +#include +#include +#include "jsapi/RTCStatsReport.h" +#include +#include "MainThreadUtils.h" +#include +#include "MediaConduitErrors.h" +#include "MediaConduitInterface.h" +#include "MediaEventSource.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "mozilla/Assertions.h" +#include "mozilla/Atomics.h" +#include "mozilla/DataMutex.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/RTCStatsReportBinding.h" +#include "mozilla/fallible.h" +#include "mozilla/mozalloc_oom.h" +#include "mozilla/MozPromise.h" +#include "mozilla/Mutex.h" +#include "mozilla/ProfilerState.h" +#include "mozilla/ReentrantMonitor.h" +#include "mozilla/ReverseIterator.h" +#include "mozilla/StateWatching.h" +#include "mozilla/Telemetry.h" +#include "mozilla/TelemetryHistogramEnums.h" +#include "mozilla/TelemetryScalarEnums.h" +#include "mozilla/Types.h" +#include "mozilla/UniquePtr.h" +#include "nsCOMPtr.h" +#include "nsDebug.h" +#include "nsError.h" +#include "nsIDirectTaskDispatcher.h" +#include "nsISerialEventTarget.h" +#include "nsStringFwd.h" +#include "PerformanceRecorder.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/network/sent_packet.h" +#include +#include +#include "transport/mediapacket.h" +#include "video/config/video_encoder_config.h" +#include "WebrtcVideoCodecFactory.h" #ifdef MOZ_WIDGET_ANDROID # include "VideoEngine.h" @@ -329,7 +391,9 @@ WebrtcVideoConduit::Control::Control(const RefPtr& aCallThread) INIT_MIRROR(mSendRtpRtcpConfig, Nothing()), INIT_MIRROR(mRecvCodecs, std::vector()), INIT_MIRROR(mRecvRtpRtcpConfig, Nothing()), - INIT_MIRROR(mCodecMode, webrtc::VideoCodecMode::kRealtimeVideo) {} + INIT_MIRROR(mCodecMode, webrtc::VideoCodecMode::kRealtimeVideo), + INIT_MIRROR(mFrameTransformerProxySend, nullptr), + INIT_MIRROR(mFrameTransformerProxyRecv, nullptr) {} #undef INIT_MIRROR WebrtcVideoConduit::WebrtcVideoConduit( @@ -368,7 +432,6 @@ WebrtcVideoConduit::WebrtcVideoConduit( WebrtcVideoConduit::~WebrtcVideoConduit() { CSFLogDebug(LOGTAG, "%s ", __FUNCTION__); - MOZ_ASSERT(!mSendStream && !mRecvStream, "Call DeleteStreams prior to ~WebrtcVideoConduit."); } @@ -408,6 +471,10 @@ void WebrtcVideoConduit::InitControl(VideoConduitControlInterface* aControl) { CONNECT(aControl->CanonicalVideoRecvRtpRtcpConfig(), mControl.mRecvRtpRtcpConfig); CONNECT(aControl->CanonicalVideoCodecMode(), mControl.mCodecMode); + CONNECT(aControl->CanonicalFrameTransformerProxySend(), + mControl.mFrameTransformerProxySend); + CONNECT(aControl->CanonicalFrameTransformerProxyRecv(), + mControl.mFrameTransformerProxyRecv); } #undef CONNECT @@ -765,6 +832,32 @@ void WebrtcVideoConduit::OnControlConfigChange() { } } + if (mControl.mConfiguredFrameTransformerProxySend.get() != + mControl.mFrameTransformerProxySend.Ref().get()) { + mControl.mConfiguredFrameTransformerProxySend = + mControl.mFrameTransformerProxySend.Ref(); + if (!mSendStreamConfig.frame_transformer) { + mSendStreamConfig.frame_transformer = + new rtc::RefCountedObject(true); + sendStreamRecreationNeeded = true; + } + static_cast(mSendStreamConfig.frame_transformer.get()) + ->SetProxy(mControl.mConfiguredFrameTransformerProxySend); + } + + if (mControl.mConfiguredFrameTransformerProxyRecv.get() != + mControl.mFrameTransformerProxyRecv.Ref().get()) { + mControl.mConfiguredFrameTransformerProxyRecv = + mControl.mFrameTransformerProxyRecv.Ref(); + if (!mRecvStreamConfig.frame_transformer) { + mRecvStreamConfig.frame_transformer = + new rtc::RefCountedObject(true); + } + static_cast(mRecvStreamConfig.frame_transformer.get()) + ->SetProxy(mControl.mConfiguredFrameTransformerProxyRecv); + // No flag to set, we always recreate recv streams + } + if (remoteSsrcUpdateNeeded) { SetRemoteSSRCConfig(mControl.mConfiguredRemoteSsrc, mControl.mConfiguredRemoteRtxSsrc); @@ -1219,6 +1312,8 @@ RefPtr WebrtcVideoConduit::Shutdown() { mControl.mRecvCodecs.DisconnectIfConnected(); mControl.mRecvRtpRtcpConfig.DisconnectIfConnected(); mControl.mCodecMode.DisconnectIfConnected(); + mControl.mFrameTransformerProxySend.DisconnectIfConnected(); + mControl.mFrameTransformerProxyRecv.DisconnectIfConnected(); mWatchManager.Shutdown(); mCall->UnregisterConduit(this); @@ -1860,6 +1955,92 @@ std::vector WebrtcVideoConduit::GetUpstreamRtpSources() return sources; } +void WebrtcVideoConduit::RequestKeyFrame(FrameTransformerProxy* aProxy) { + mCallThread->Dispatch(NS_NewRunnableFunction( + __func__, [this, self = RefPtr(this), + proxy = RefPtr(aProxy)] { + bool success = false; + if (mRecvStream && mEngineReceiving) { + // This is a misnomer. This requests a keyframe from the other side. + mRecvStream->GenerateKeyFrame(); + success = true; + } + proxy->KeyFrameRequestDone(success); + })); +} + +void WebrtcVideoConduit::GenerateKeyFrame(const Maybe& aRid, + FrameTransformerProxy* aProxy) { + // libwebrtc does not implement error handling in the way that + // webrtc-encoded-transform specifies. So, we'll need to do that here. + // Also, spec wants us to synchronously check whether there's an encoder, but + // that's not something that can be checked synchronously. + + mCallThread->Dispatch(NS_NewRunnableFunction( + __func__, [this, self = RefPtr(this), + proxy = RefPtr(aProxy), aRid] { + // If encoder is undefined, reject promise with InvalidStateError, + // abort these steps. + + // If encoder is not processing video frames, reject promise with + // InvalidStateError, abort these steps. + if (!mSendStream || !mCurSendCodecConfig || !mEngineTransmitting) { + CopyableErrorResult result; + result.ThrowInvalidStateError("No encoders"); + proxy->GenerateKeyFrameError(aRid, result); + return; + } + + // Gather a list of video encoders, named videoEncoders from encoder, + // ordered according negotiated RIDs if any. + // NOTE: This is represented by mCurSendCodecConfig->mEncodings + + // If rid is defined, remove from videoEncoders any video encoder that + // does not match rid. + + // If rid is undefined, remove from videoEncoders all video encoders + // except the first one. + bool found = false; + std::vector rids; + if (!aRid.isSome()) { + // If rid is undefined, set rid to the RID value corresponding to + // videoEncoder. + if (!mCurSendCodecConfig->mEncodings.empty()) { + if (!mCurSendCodecConfig->mEncodings[0].rid.empty()) { + rids.push_back(mCurSendCodecConfig->mEncodings[0].rid); + } + found = true; + } + } else { + for (const auto& encoding : mCurSendCodecConfig->mEncodings) { + if (encoding.rid == *aRid) { + found = true; + rids.push_back(encoding.rid); + break; + } + } + } + + // If videoEncoders is empty, reject promise with NotFoundError and + // abort these steps. videoEncoders is expected to be empty if the + // corresponding RTCRtpSender is not active, or the corresponding + // RTCRtpSender track is ended. + if (!found) { + CopyableErrorResult result; + result.ThrowNotFoundError("Rid not in use"); + proxy->GenerateKeyFrameError(aRid, result); + } + + // NOTE: We don't do this stuff, because libwebrtc's interface is + // rid-based. + // Let videoEncoder be the first encoder in videoEncoders. + // If rid is undefined, set rid to the RID value corresponding to + // videoEncoder. + + mSendStream->GenerateKeyFrame(rids); + })); +} + bool WebrtcVideoConduit::HasCodecPluginID(uint64_t aPluginID) const { MOZ_ASSERT(NS_IsMainThread()); diff --git a/dom/media/webrtc/libwebrtcglue/VideoConduit.h b/dom/media/webrtc/libwebrtcglue/VideoConduit.h index 11a07a1f944e..f4e6583a21b1 100644 --- a/dom/media/webrtc/libwebrtcglue/VideoConduit.h +++ b/dom/media/webrtc/libwebrtcglue/VideoConduit.h @@ -231,6 +231,10 @@ class WebrtcVideoConduit std::vector GetUpstreamRtpSources() const override; + void RequestKeyFrame(FrameTransformerProxy* aProxy) override; + void GenerateKeyFrame(const Maybe& aRid, + FrameTransformerProxy* aProxy) override; + private: // Don't allow copying/assigning. WebrtcVideoConduit(const WebrtcVideoConduit&) = delete; @@ -298,6 +302,8 @@ class WebrtcVideoConduit Mirror> mRecvCodecs; Mirror> mRecvRtpRtcpConfig; Mirror mCodecMode; + Mirror> mFrameTransformerProxySend; + Mirror> mFrameTransformerProxyRecv; // For caching mRemoteSsrc and mRemoteRtxSsrc, since another caller may // change the remote ssrc in the stream config directly. @@ -310,6 +316,10 @@ class WebrtcVideoConduit std::vector mConfiguredRecvCodecs; Maybe mConfiguredRecvRtpRtcpConfig; + // For change tracking. Callthread only. + RefPtr mConfiguredFrameTransformerProxySend; + RefPtr mConfiguredFrameTransformerProxyRecv; + Control() = delete; explicit Control(const RefPtr& aCallThread); } mControl; diff --git a/dom/media/webrtc/libwebrtcglue/moz.build b/dom/media/webrtc/libwebrtcglue/moz.build index 62bac2ffe616..6f650e8da9ff 100644 --- a/dom/media/webrtc/libwebrtcglue/moz.build +++ b/dom/media/webrtc/libwebrtcglue/moz.build @@ -19,6 +19,8 @@ LOCAL_INCLUDES += [ UNIFIED_SOURCES += [ "AudioConduit.cpp", + "FrameTransformer.cpp", + "FrameTransformerProxy.cpp", "GmpVideoCodec.cpp", "MediaConduitInterface.cpp", "MediaDataCodec.cpp", diff --git a/dom/tests/mochitest/general/test_interfaces.js b/dom/tests/mochitest/general/test_interfaces.js index 422fc54c46c9..6834d81be07b 100644 --- a/dom/tests/mochitest/general/test_interfaces.js +++ b/dom/tests/mochitest/general/test_interfaces.js @@ -1064,6 +1064,10 @@ let interfaceNamesInGlobalScope = [ // IMPORTANT: Do not change this list without review from a DOM peer! { name: "RTCDTMFToneChangeEvent", insecureContext: true }, // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "RTCEncodedAudioFrame", insecureContext: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "RTCEncodedVideoFrame", insecureContext: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! { name: "RTCIceCandidate", insecureContext: true }, // IMPORTANT: Do not change this list without review from a DOM peer! { name: "RTCPeerConnection", insecureContext: true }, @@ -1072,6 +1076,8 @@ let interfaceNamesInGlobalScope = [ // IMPORTANT: Do not change this list without review from a DOM peer! { name: "RTCRtpReceiver", insecureContext: true }, // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "RTCRtpScriptTransform", insecureContext: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! { name: "RTCRtpSender", insecureContext: true }, // IMPORTANT: Do not change this list without review from a DOM peer! { name: "RTCRtpTransceiver", insecureContext: true }, diff --git a/dom/webidl/DedicatedWorkerGlobalScope.webidl b/dom/webidl/DedicatedWorkerGlobalScope.webidl index e32e23c0cd06..68c2c0fc640d 100644 --- a/dom/webidl/DedicatedWorkerGlobalScope.webidl +++ b/dom/webidl/DedicatedWorkerGlobalScope.webidl @@ -37,3 +37,9 @@ interface DedicatedWorkerGlobalScope : WorkerGlobalScope { [Pref="dom.workers.requestAnimationFrame", Throws] undefined cancelAnimationFrame(long handle); }; + +// https://w3c.github.io/webrtc-encoded-transform/#RTCEncodedAudioFrame-methods +partial interface DedicatedWorkerGlobalScope { + [Pref="media.peerconnection.enabled", + Pref="media.peerconnection.scripttransform.enabled"] attribute EventHandler onrtctransform; +}; diff --git a/dom/webidl/RTCEncodedAudioFrame.webidl b/dom/webidl/RTCEncodedAudioFrame.webidl new file mode 100644 index 000000000000..a2a25f0f284f --- /dev/null +++ b/dom/webidl/RTCEncodedAudioFrame.webidl @@ -0,0 +1,24 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. + * + * The origin of this IDL file is + * https://w3c.github.io/webrtc-encoded-transform + */ + +dictionary RTCEncodedAudioFrameMetadata { + unsigned long synchronizationSource; + octet payloadType; + sequence contributingSources; + short sequenceNumber; +}; + +[Pref="media.peerconnection.enabled", + Pref="media.peerconnection.scripttransform.enabled", + Exposed=(Window,DedicatedWorker)] +interface RTCEncodedAudioFrame { + readonly attribute unsigned long timestamp; + attribute ArrayBuffer data; + RTCEncodedAudioFrameMetadata getMetadata(); +}; diff --git a/dom/webidl/RTCEncodedVideoFrame.webidl b/dom/webidl/RTCEncodedVideoFrame.webidl new file mode 100644 index 000000000000..564252fd6ed0 --- /dev/null +++ b/dom/webidl/RTCEncodedVideoFrame.webidl @@ -0,0 +1,41 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. + * + * The origin of this IDL file is + * https://www.w3.org/TR/webrtc-encoded-transform + */ + +// New enum for video frame types. Will eventually re-use the equivalent defined +// by WebCodecs. +enum RTCEncodedVideoFrameType { + "empty", + "key", + "delta", +}; + +dictionary RTCEncodedVideoFrameMetadata { + unsigned long long frameId; + sequence dependencies; + unsigned short width; + unsigned short height; + unsigned long spatialIndex; + unsigned long temporalIndex; + unsigned long synchronizationSource; + octet payloadType; + sequence contributingSources; + long long timestamp; // microseconds +}; + +// New interfaces to define encoded video and audio frames. Will eventually +// re-use or extend the equivalent defined in WebCodecs. +[Pref="media.peerconnection.enabled", + Pref="media.peerconnection.scripttransform.enabled", + Exposed=(Window,DedicatedWorker)] +interface RTCEncodedVideoFrame { + readonly attribute RTCEncodedVideoFrameType type; + readonly attribute unsigned long timestamp; + attribute ArrayBuffer data; + RTCEncodedVideoFrameMetadata getMetadata(); +}; diff --git a/dom/webidl/RTCRtpReceiver.webidl b/dom/webidl/RTCRtpReceiver.webidl index 59a4f1340fd9..d8002e6511ac 100644 --- a/dom/webidl/RTCRtpReceiver.webidl +++ b/dom/webidl/RTCRtpReceiver.webidl @@ -32,3 +32,9 @@ partial interface RTCRtpReceiver { [Throws] attribute DOMHighResTimeStamp? jitterBufferTarget; }; + +// https://w3c.github.io/webrtc-encoded-transform/#specification +partial interface RTCRtpReceiver { + [SetterThrows, + Pref="media.peerconnection.scripttransform.enabled"] attribute RTCRtpTransform? transform; +}; diff --git a/dom/webidl/RTCRtpScriptTransform.webidl b/dom/webidl/RTCRtpScriptTransform.webidl new file mode 100644 index 000000000000..91fe7b39edb5 --- /dev/null +++ b/dom/webidl/RTCRtpScriptTransform.webidl @@ -0,0 +1,20 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. + * + * The origin of this IDL file is + * https://www.w3.org/TR/webrtc-encoded-transform + */ + +// Spec version is commented out (uncomment if SFrameTransform is implemented) +// typedef (SFrameTransform or RTCRtpScriptTransform) RTCRtpTransform; +typedef RTCRtpScriptTransform RTCRtpTransform; + +[Pref="media.peerconnection.enabled", + Pref="media.peerconnection.scripttransform.enabled", + Exposed=Window] +interface RTCRtpScriptTransform { + [Throws] + constructor(Worker worker, optional any options, optional sequence transfer); +}; diff --git a/dom/webidl/RTCRtpScriptTransformer.webidl b/dom/webidl/RTCRtpScriptTransformer.webidl new file mode 100644 index 000000000000..3a047ac9a015 --- /dev/null +++ b/dom/webidl/RTCRtpScriptTransformer.webidl @@ -0,0 +1,19 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. + * + * The origin of this IDL file is + * https://www.w3.org/TR/webrtc-encoded-transform + */ + +[Pref="media.peerconnection.enabled", + Pref="media.peerconnection.scripttransform.enabled", + Exposed=DedicatedWorker] +interface RTCRtpScriptTransformer { + readonly attribute ReadableStream readable; + readonly attribute WritableStream writable; + [Throws] readonly attribute any options; + Promise generateKeyFrame(optional DOMString rid); + Promise sendKeyFrameRequest(); +}; diff --git a/dom/webidl/RTCRtpSender.webidl b/dom/webidl/RTCRtpSender.webidl index 7867d019e50c..c6e70a376f19 100644 --- a/dom/webidl/RTCRtpSender.webidl +++ b/dom/webidl/RTCRtpSender.webidl @@ -30,3 +30,9 @@ interface RTCRtpSender { [ChromeOnly] undefined setTrack(MediaStreamTrack? track); }; + +// https://w3c.github.io/webrtc-encoded-transform/#specification +partial interface RTCRtpSender { + [SetterThrows, + Pref="media.peerconnection.scripttransform.enabled"] attribute RTCRtpTransform? transform; +}; diff --git a/dom/webidl/RTCTransformEvent.webidl b/dom/webidl/RTCTransformEvent.webidl new file mode 100644 index 000000000000..b8f48faab626 --- /dev/null +++ b/dom/webidl/RTCTransformEvent.webidl @@ -0,0 +1,20 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. + * + * The origin of this IDL file is + * https://www.w3.org/TR/webrtc-encoded-transform + */ + +[Pref="media.peerconnection.enabled", + Pref="media.peerconnection.scripttransform.enabled", + Exposed=DedicatedWorker] +interface RTCTransformEvent : Event { + constructor(DOMString type, RTCTransformEventInit eventInitDict); + readonly attribute RTCRtpScriptTransformer transformer; +}; + +dictionary RTCTransformEventInit : EventInit { + required RTCRtpScriptTransformer transformer; +}; diff --git a/dom/webidl/moz.build b/dom/webidl/moz.build index 401e72096e98..a5222e7b240e 100644 --- a/dom/webidl/moz.build +++ b/dom/webidl/moz.build @@ -1060,6 +1060,8 @@ if CONFIG["MOZ_WEBRTC"]: "RTCDataChannel.webidl", "RTCDtlsTransport.webidl", "RTCDTMFSender.webidl", + "RTCEncodedAudioFrame.webidl", + "RTCEncodedVideoFrame.webidl", "RTCIceCandidate.webidl", "RTCIdentityAssertion.webidl", "RTCIdentityProvider.webidl", @@ -1068,12 +1070,15 @@ if CONFIG["MOZ_WEBRTC"]: "RTCRtpCapabilities.webidl", "RTCRtpParameters.webidl", "RTCRtpReceiver.webidl", + "RTCRtpScriptTransform.webidl", + "RTCRtpScriptTransformer.webidl", "RTCRtpSender.webidl", "RTCRtpSources.webidl", "RTCRtpTransceiver.webidl", "RTCSctpTransport.webidl", "RTCSessionDescription.webidl", "RTCStatsReport.webidl", + "RTCTransformEvent.webidl", "WebrtcGlobalInformation.webidl", ] @@ -1181,6 +1186,7 @@ GENERATED_EVENTS_WEBIDL_FILES = [ "PositionStateEvent.webidl", "ProgressEvent.webidl", "PromiseRejectionEvent.webidl", + "RTCTransformEvent.webidl", "ScrollViewChangeEvent.webidl", "SecurityPolicyViolationEvent.webidl", "StyleSheetApplicableStateChangeEvent.webidl", diff --git a/dom/workers/EventWithOptionsRunnable.cpp b/dom/workers/EventWithOptionsRunnable.cpp new file mode 100644 index 000000000000..c3d81491abc9 --- /dev/null +++ b/dom/workers/EventWithOptionsRunnable.cpp @@ -0,0 +1,164 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "EventWithOptionsRunnable.h" +#include "WorkerScope.h" +#include "mozilla/dom/WorkerRunnable.h" +#include "mozilla/dom/StructuredCloneHolder.h" +#include "js/StructuredClone.h" +#include "js/RootingAPI.h" +#include "js/Value.h" +#include "nsJSPrincipals.h" +#include "nsContentUtils.h" +#include "nsDebug.h" +#include "MainThreadUtils.h" +#include "mozilla/Assertions.h" +#include "nsGlobalWindowInner.h" +#include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/ErrorResult.h" +#include "nsIGlobalObject.h" +#include "nsCOMPtr.h" +#include "js/GlobalObject.h" +#include "xpcpublic.h" +#include "mozilla/dom/MessagePortBinding.h" +#include "mozilla/dom/MessagePort.h" +#include "mozilla/OwningNonNull.h" +#include "mozilla/RefPtr.h" +#include "mozilla/dom/Event.h" +#include "mozilla/dom/WorkerCommon.h" + +namespace mozilla::dom { +EventWithOptionsRunnable::EventWithOptionsRunnable(Worker& aWorker) + : WorkerDebuggeeRunnable(aWorker.mWorkerPrivate, + WorkerRunnable::WorkerThreadModifyBusyCount), + StructuredCloneHolder(CloningSupported, TransferringSupported, + StructuredCloneScope::SameProcess) {} + +EventWithOptionsRunnable::~EventWithOptionsRunnable() = default; + +void EventWithOptionsRunnable::InitOptions( + JSContext* aCx, JS::Handle aOptions, + const Sequence& aTransferable, ErrorResult& aRv) { + JS::Rooted transferable(aCx, JS::UndefinedValue()); + + aRv = nsContentUtils::CreateJSValueFromSequenceOfObject(aCx, aTransferable, + &transferable); + if (NS_WARN_IF(aRv.Failed())) { + return; + } + + JS::CloneDataPolicy clonePolicy; + // DedicatedWorkers are always part of the same agent cluster. + clonePolicy.allowIntraClusterClonableSharedObjects(); + + MOZ_ASSERT(NS_IsMainThread()); + nsGlobalWindowInner* win = nsContentUtils::IncumbentInnerWindow(); + if (win && win->IsSharedMemoryAllowed()) { + clonePolicy.allowSharedMemoryObjects(); + } + + Write(aCx, aOptions, transferable, clonePolicy, aRv); +} + +// Cargo-culted from MesssageEventRunnable. +bool EventWithOptionsRunnable::BuildAndFireEvent( + JSContext* aCx, WorkerPrivate* aWorkerPrivate, + DOMEventTargetHelper* aTarget) { + IgnoredErrorResult rv; + nsCOMPtr parent = aTarget->GetParentObject(); + + // For some workers without window, parent is null and we try to find it + // from the JS Context. + if (!parent) { + JS::Rooted globalObject(aCx, JS::CurrentGlobalOrNull(aCx)); + if (NS_WARN_IF(!globalObject)) { + rv.ThrowDataCloneError("failed to get global object"); + OptionsDeserializeFailed(rv); + return false; + } + + parent = xpc::NativeGlobal(globalObject); + if (NS_WARN_IF(!parent)) { + rv.ThrowDataCloneError("failed to get parent"); + OptionsDeserializeFailed(rv); + return false; + } + } + + MOZ_ASSERT(parent); + + JS::Rooted options(aCx); + + JS::CloneDataPolicy cloneDataPolicy; + if (parent->GetClientInfo().isSome() && + parent->GetClientInfo()->AgentClusterId().isSome() && + parent->GetClientInfo()->AgentClusterId()->Equals( + aWorkerPrivate->AgentClusterId())) { + cloneDataPolicy.allowIntraClusterClonableSharedObjects(); + } + + if (aWorkerPrivate->IsSharedMemoryAllowed()) { + cloneDataPolicy.allowSharedMemoryObjects(); + } + + Read(parent, aCx, &options, cloneDataPolicy, rv); + + if (NS_WARN_IF(rv.Failed())) { + OptionsDeserializeFailed(rv); + return false; + } + + Sequence> ports; + if (NS_WARN_IF(!TakeTransferredPortsAsSequence(ports))) { + // TODO: Is this an appropriate type? What does this actually do? + rv.ThrowDataCloneError("TakeTransferredPortsAsSequence failed"); + OptionsDeserializeFailed(rv); + return false; + } + + RefPtr event = BuildEvent(aCx, parent, aTarget, options); + + if (NS_WARN_IF(!event)) { + return false; + } + + aTarget->DispatchEvent(*event); + return true; +} + +bool EventWithOptionsRunnable::WorkerRun(JSContext* aCx, + WorkerPrivate* aWorkerPrivate) { + if (mBehavior == ParentThreadUnchangedBusyCount) { + // Don't fire this event if the JS object has been disconnected from the + // private object. + if (!aWorkerPrivate->IsAcceptingEvents()) { + return true; + } + + // Once a window has frozen its workers, their + // mMainThreadDebuggeeEventTargets should be paused, and their + // WorkerDebuggeeRunnables should not be being executed. The same goes for + // WorkerDebuggeeRunnables sent from child to parent workers, but since a + // frozen parent worker runs only control runnables anyway, that is taken + // care of naturally. + MOZ_ASSERT(!aWorkerPrivate->IsFrozen()); + + // Similarly for paused windows; all its workers should have been informed. + // (Subworkers are unaffected by paused windows.) + MOZ_ASSERT(!aWorkerPrivate->IsParentWindowPaused()); + + aWorkerPrivate->AssertInnerWindowIsCorrect(); + + return BuildAndFireEvent(aCx, aWorkerPrivate, + aWorkerPrivate->ParentEventTargetRef()); + } + + MOZ_ASSERT(aWorkerPrivate == GetWorkerPrivateFromContext(aCx)); + + return BuildAndFireEvent(aCx, aWorkerPrivate, aWorkerPrivate->GlobalScope()); +} + +} // namespace mozilla::dom diff --git a/dom/workers/EventWithOptionsRunnable.h b/dom/workers/EventWithOptionsRunnable.h new file mode 100644 index 000000000000..424a4e157925 --- /dev/null +++ b/dom/workers/EventWithOptionsRunnable.h @@ -0,0 +1,55 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_DOM_WORKERS_EVENTWITHOPTIONSRUNNABLE_H_ +#define MOZILLA_DOM_WORKERS_EVENTWITHOPTIONSRUNNABLE_H_ + +#include "WorkerCommon.h" +#include "WorkerRunnable.h" +#include "mozilla/dom/StructuredCloneHolder.h" + +namespace mozilla { +class DOMEventTargetHelper; + +namespace dom { +class Event; +class EventTarget; +class Worker; +class WorkerPrivate; + +// Cargo-culted from MesssageEventRunnable. +// Intended to be used for the idiom where arbitrary options are transferred to +// the worker thread (with optional transfer functions), which are then used to +// build an event, which is then fired on the global worker scope. +class EventWithOptionsRunnable : public WorkerDebuggeeRunnable, + public StructuredCloneHolder { + public: + explicit EventWithOptionsRunnable(Worker& aWorker); + void InitOptions(JSContext* aCx, JS::Handle aOptions, + const Sequence& aTransferable, ErrorResult& aRv); + + // Called on the worker thread. The event returned will be fired on the + // worker's global scope. If a StrongWorkerRef needs to be retained, the + // implementation can do so with the WorkerPrivate. + virtual already_AddRefed BuildEvent( + JSContext* aCx, nsIGlobalObject* aGlobal, EventTarget* aTarget, + JS::Handle aOptions) = 0; + + // Called on the worker thread + virtual void OptionsDeserializeFailed(ErrorResult& aRv) {} + + protected: + virtual ~EventWithOptionsRunnable(); + + private: + bool WorkerRun(JSContext* aCx, WorkerPrivate* aWorkerPrivate) override; + bool BuildAndFireEvent(JSContext* aCx, WorkerPrivate* aWorkerPrivate, + DOMEventTargetHelper* aTarget); +}; +} // namespace dom +} // namespace mozilla + +#endif // MOZILLA_DOM_WORKERS_EVENTWITHOPTIONSRUNNABLE_H_ diff --git a/dom/workers/Worker.cpp b/dom/workers/Worker.cpp index 3452859c4c6d..2dea04bf5582 100644 --- a/dom/workers/Worker.cpp +++ b/dom/workers/Worker.cpp @@ -16,6 +16,13 @@ #include "nsContentUtils.h" #include "nsGlobalWindowOuter.h" #include "WorkerPrivate.h" +#include "EventWithOptionsRunnable.h" +#include "js/RootingAPI.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "nsISupports.h" +#include "nsDebug.h" +#include "mozilla/dom/WorkerStatus.h" +#include "mozilla/RefPtr.h" #ifdef XP_WIN # undef PostMessage @@ -177,6 +184,34 @@ void Worker::PostMessage(JSContext* aCx, JS::Handle aMessage, PostMessage(aCx, aMessage, aOptions.mTransfer, aRv); } +void Worker::PostEventWithOptions(JSContext* aCx, + JS::Handle aOptions, + const Sequence& aTransferable, + EventWithOptionsRunnable* aRunnable, + ErrorResult& aRv) { + NS_ASSERT_OWNINGTHREAD(Worker); + + if (NS_WARN_IF(!mWorkerPrivate || + mWorkerPrivate->ParentStatusProtected() > Running)) { + return; + } + RefPtr workerPrivate = mWorkerPrivate; + Unused << workerPrivate; + + aRunnable->InitOptions(aCx, aOptions, aTransferable, aRv); + + if (NS_WARN_IF(!mWorkerPrivate || + mWorkerPrivate->ParentStatusProtected() > Running)) { + return; + } + + if (NS_WARN_IF(aRv.Failed())) { + return; + } + + Unused << NS_WARN_IF(!aRunnable->Dispatch()); +} + void Worker::Terminate() { NS_ASSERT_OWNINGTHREAD(Worker); diff --git a/dom/workers/Worker.h b/dom/workers/Worker.h index b1b068668865..14d0630f284a 100644 --- a/dom/workers/Worker.h +++ b/dom/workers/Worker.h @@ -19,6 +19,7 @@ namespace mozilla::dom { +class EventWithOptionsRunnable; struct StructuredSerializeOptions; struct WorkerOptions; class WorkerPrivate; @@ -48,6 +49,11 @@ class Worker : public DOMEventTargetHelper, public SupportsWeakPtr { const StructuredSerializeOptions& aOptions, ErrorResult& aRv); + void PostEventWithOptions(JSContext* aCx, JS::Handle aOptions, + const Sequence& aTransferable, + EventWithOptionsRunnable* aRunnable, + ErrorResult& aRv); + void Terminate(); IMPL_EVENT_HANDLER(error) @@ -59,6 +65,7 @@ class Worker : public DOMEventTargetHelper, public SupportsWeakPtr { already_AddRefed aWorkerPrivate); ~Worker(); + friend class EventWithOptionsRunnable; RefPtr mWorkerPrivate; }; diff --git a/dom/workers/WorkerScope.h b/dom/workers/WorkerScope.h index e29413f503ed..6d4ea76ef1fc 100644 --- a/dom/workers/WorkerScope.h +++ b/dom/workers/WorkerScope.h @@ -424,6 +424,7 @@ class DedicatedWorkerGlobalScope final IMPL_EVENT_HANDLER(message) IMPL_EVENT_HANDLER(messageerror) + IMPL_EVENT_HANDLER(rtctransform) private: ~DedicatedWorkerGlobalScope() = default; diff --git a/dom/workers/moz.build b/dom/workers/moz.build index ed6950bd7011..4568e63ea430 100644 --- a/dom/workers/moz.build +++ b/dom/workers/moz.build @@ -12,6 +12,7 @@ DIRS += ["remoteworkers", "sharedworkers", "loader"] # Public stuff. EXPORTS.mozilla.dom += [ "ChromeWorker.h", + "EventWithOptionsRunnable.h", "JSExecutionManager.h", "Worker.h", "WorkerChannelInfo.h", @@ -51,6 +52,7 @@ XPIDL_SOURCES += [ UNIFIED_SOURCES += [ "ChromeWorker.cpp", "ChromeWorkerScope.cpp", + "EventWithOptionsRunnable.cpp", "JSExecutionManager.cpp", "MessageEventRunnable.cpp", "RegisterBindings.cpp", diff --git a/dom/workers/test/test_worker_interfaces.js b/dom/workers/test/test_worker_interfaces.js index 87c0cda696ca..8743f3c4a293 100644 --- a/dom/workers/test/test_worker_interfaces.js +++ b/dom/workers/test/test_worker_interfaces.js @@ -307,6 +307,14 @@ let interfaceNamesInGlobalScope = [ // IMPORTANT: Do not change this list without review from a DOM peer! { name: "Response", insecureContext: true }, // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "RTCEncodedAudioFrame", insecureContext: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "RTCEncodedVideoFrame", insecureContext: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "RTCRtpScriptTransformer", insecureContext: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "RTCTransformEvent", insecureContext: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! { name: "Scheduler", insecureContext: true, nightly: true }, // IMPORTANT: Do not change this list without review from a DOM peer! { name: "StorageManager", fennec: false }, @@ -426,6 +434,8 @@ let interfaceNamesInGlobalScope = [ // IMPORTANT: Do not change this list without review from a DOM peer! { name: "onmessageerror", insecureContext: true }, // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "onrtctransform", insecureContext: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! { name: "postMessage", insecureContext: true }, // IMPORTANT: Do not change this list without review from a DOM peer! { name: "requestAnimationFrame", insecureContext: true }, diff --git a/media/webrtc/signaling/gtest/Canonicals.h b/media/webrtc/signaling/gtest/Canonicals.h index d9be46faa5b5..e7bab2d4fd6a 100644 --- a/media/webrtc/signaling/gtest/Canonicals.h +++ b/media/webrtc/signaling/gtest/Canonicals.h @@ -29,6 +29,8 @@ class ConcreteCanonicals { INIT_CANONICAL(mLocalRecvRtpExtensions, RtpExtList()), INIT_CANONICAL(mRemoteSsrc, 0), INIT_CANONICAL(mRemoteVideoRtxSsrc, 0), + INIT_CANONICAL(mFrameTransformerProxySend, nullptr), + INIT_CANONICAL(mFrameTransformerProxyRecv, nullptr), INIT_CANONICAL(mAudioRecvCodecs, std::vector()), INIT_CANONICAL(mAudioSendCodec, Nothing()), INIT_CANONICAL(mVideoRecvCodecs, std::vector()), @@ -49,6 +51,8 @@ class ConcreteCanonicals { Canonical mLocalRecvRtpExtensions; Canonical mRemoteSsrc; Canonical mRemoteVideoRtxSsrc; + Canonical> mFrameTransformerProxySend; + Canonical> mFrameTransformerProxyRecv; Canonical> mAudioRecvCodecs; Canonical> mAudioSendCodec; @@ -103,6 +107,14 @@ class ConcreteControl : public AudioConduitControlInterface, Canonical& CanonicalLocalSendRtpExtensions() override { return mLocalSendRtpExtensions; } + Canonical>& CanonicalFrameTransformerProxySend() + override { + return mFrameTransformerProxySend; + } + Canonical>& CanonicalFrameTransformerProxyRecv() + override { + return mFrameTransformerProxyRecv; + } // AudioConduitControlInterface Canonical>& CanonicalAudioSendCodec() override { diff --git a/media/webrtc/signaling/gtest/MockConduit.h b/media/webrtc/signaling/gtest/MockConduit.h index d0c43f7c45aa..c2c5becd1ba5 100644 --- a/media/webrtc/signaling/gtest/MockConduit.h +++ b/media/webrtc/signaling/gtest/MockConduit.h @@ -9,6 +9,7 @@ #include "gmock/gmock.h" #include "MediaConduitInterface.h" +#include "libwebrtcglue/FrameTransformer.h" namespace webrtc { std::ostream& operator<<(std::ostream& aStream, diff --git a/modules/libpref/init/StaticPrefList.yaml b/modules/libpref/init/StaticPrefList.yaml index 64201641b36e..2e931c661824 100644 --- a/modules/libpref/init/StaticPrefList.yaml +++ b/modules/libpref/init/StaticPrefList.yaml @@ -10750,7 +10750,12 @@ # navigator.mediaDevices and getUserMedia() support as well. # See also media.navigator.enabled - name: media.peerconnection.enabled - type: bool + type: RelaxedAtomicBool + value: true + mirror: always + +- name: media.peerconnection.scripttransform.enabled + type: RelaxedAtomicBool value: true mirror: always diff --git a/widget/EventMessageList.h b/widget/EventMessageList.h index 592ed84cf0d0..ae02d04676af 100644 --- a/widget/EventMessageList.h +++ b/widget/EventMessageList.h @@ -359,6 +359,7 @@ NS_EVENT_MESSAGE(eAfterPrint) NS_EVENT_MESSAGE(eMessage) NS_EVENT_MESSAGE(eMessageError) +NS_EVENT_MESSAGE(eRTCTransform) // Menu open event NS_EVENT_MESSAGE(eOpen) diff --git a/xpcom/ds/StaticAtoms.py b/xpcom/ds/StaticAtoms.py index 06410d4ca666..e114c7ce95ba 100644 --- a/xpcom/ds/StaticAtoms.py +++ b/xpcom/ds/StaticAtoms.py @@ -1934,6 +1934,8 @@ STATIC_ATOMS = [ Atom("ondevicelight", "ondevicelight"), # MediaDevices device change event Atom("ondevicechange", "ondevicechange"), + # WebRTC events + Atom("onrtctransform", "onrtctransform"), # Internal Visual Viewport events Atom("onmozvisualresize", "onmozvisualresize"), Atom("onmozvisualscroll", "onmozvisualscroll"),