Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-07 09:54:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1631263: Implement RTCRtpScriptTransform. r=pehrsons,jib,asuth,emilio,saschanaz

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D179735
This commit is contained in:
Byron Campen 2023-07-20 14:24:27 +00:00
parent 1b0ed5f17c
commit e1148e1aab
52 changed files with 3028 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dom/events/EventNameList.h
View File

@ -295,6 +295,7 @@ WINDOW_EVENT(languagechange, eLanguageChange,
// need a different macro to flag things like that (IDL, but not content
// attributes on body/frameset), or is just using EventNameType_None enough?
WINDOW_EVENT(message, eMessage, EventNameType_None, eBasicEventClass)
WINDOW_EVENT(rtctransform, eRTCTransform, EventNameType_None, eBasicEventClass)
WINDOW_EVENT(messageerror, eMessageError, EventNameType_HTMLBodyOrFramesetOnly,
eBasicEventClass)
WINDOW_EVENT(offline, eOffline,

94
dom/media/webrtc/jsapi/RTCEncodedAudioFrame.cpp Normal file
View File

@ -0,0 +1,94 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "jsapi/RTCEncodedAudioFrame.h"
#include <stdint.h>
#include <memory>
#include <utility>
#include "api/frame_transformer_interface.h"
#include "jsapi/RTCEncodedFrameBase.h"
#include "jsapi/RTCRtpScriptTransform.h"
#include "mozilla/dom/RTCRtpScriptTransformer.h"
#include "mozilla/dom/RTCEncodedAudioFrameBinding.h"
#include "nsWrapperCache.h"
#include "nsISupports.h"
#include "nsCycleCollectionParticipant.h"
#include "nsIGlobalObject.h"
#include "nsContentUtils.h"
#include "mozilla/HoldDropJSObjects.h"
#include "mozilla/RefPtr.h"
#include "mozilla/Unused.h"
#include "mozilla/fallible.h"
#include "js/RootingAPI.h"
namespace mozilla::dom {
NS_IMPL_CYCLE_COLLECTION_INHERITED(RTCEncodedAudioFrame, RTCEncodedFrameBase,
mOwner)
NS_IMPL_ADDREF_INHERITED(RTCEncodedAudioFrame, RTCEncodedFrameBase)
NS_IMPL_RELEASE_INHERITED(RTCEncodedAudioFrame, RTCEncodedFrameBase)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCEncodedAudioFrame)
NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
NS_INTERFACE_MAP_END_INHERITING(RTCEncodedFrameBase)
RTCEncodedAudioFrame::RTCEncodedAudioFrame(
nsIGlobalObject* aGlobal,
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
uint64_t aCounter, RTCRtpScriptTransformer* aOwner)
: RTCEncodedFrameBase(aGlobal, std::move(aFrame), aCounter),
mOwner(aOwner) {
mMetadata.mSynchronizationSource.Construct(mFrame->GetSsrc());
mMetadata.mPayloadType.Construct(mFrame->GetPayloadType());
// send frames are derived directly from TransformableFrameInterface, not
// TransformableAudioFrameInterface! Right now, send frames have no csrcs
// or sequence number
// TODO(bug 1835076): Fix this
if (mFrame->GetDirection() ==
webrtc::TransformableFrameInterface::Direction::kReceiver) {
const auto& audioFrame(
static_cast<webrtc::TransformableAudioFrameInterface&>(*mFrame));
mMetadata.mContributingSources.Construct();
for (const auto csrc : audioFrame.GetContributingSources()) {
Unused << mMetadata.mContributingSources.Value().AppendElement(csrc,
fallible);
}
mMetadata.mSequenceNumber.Construct(audioFrame.GetHeader().sequenceNumber);
}
// Base class needs this, but can't do it itself because of an assertion in
// the cycle-collector.
mozilla::HoldJSObjects(this);
}
RTCEncodedAudioFrame::~RTCEncodedAudioFrame() {
// Base class needs this, but can't do it itself because of an assertion in
// the cycle-collector.
mozilla::DropJSObjects(this);
}
JSObject* RTCEncodedAudioFrame::WrapObject(JSContext* aCx,
JS::Handle<JSObject*> aGivenProto) {
return RTCEncodedAudioFrame_Binding::Wrap(aCx, this, aGivenProto);
}
nsIGlobalObject* RTCEncodedAudioFrame::GetParentObject() const {
return mGlobal;
}
void RTCEncodedAudioFrame::GetMetadata(
RTCEncodedAudioFrameMetadata& aMetadata) const {
aMetadata = mMetadata;
}
bool RTCEncodedAudioFrame::CheckOwner(RTCRtpScriptTransformer* aOwner) const {
return aOwner == mOwner;
}
} // namespace mozilla::dom

52
dom/media/webrtc/jsapi/RTCEncodedAudioFrame.h Normal file
View File

@ -0,0 +1,52 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDAUDIOFRAME_H_
#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDAUDIOFRAME_H_
#include "mozilla/RefPtr.h"
#include "nsIGlobalObject.h"
#include "jsapi/RTCEncodedFrameBase.h"
#include "mozilla/dom/RTCEncodedAudioFrameBinding.h"
namespace mozilla::dom {
// Wraps a libwebrtc frame, allowing the frame buffer to be modified, and
// providing read-only access to various metadata. After the libwebrtc frame is
// extracted (with RTCEncodedFrameBase::TakeFrame), the frame buffer is
// detached, but the metadata remains accessible.
class RTCEncodedAudioFrame final : public RTCEncodedFrameBase {
public:
explicit RTCEncodedAudioFrame(
nsIGlobalObject* aGlobal,
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
uint64_t aCounter, RTCRtpScriptTransformer* aOwner);
// nsISupports
NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(RTCEncodedAudioFrame,
RTCEncodedFrameBase)
// webidl (timestamp and data accessors live in base class)
JSObject* WrapObject(JSContext* aCx,
JS::Handle<JSObject*> aGivenProto) override;
nsIGlobalObject* GetParentObject() const;
void GetMetadata(RTCEncodedAudioFrameMetadata& aMetadata) const;
bool CheckOwner(RTCRtpScriptTransformer* aOwner) const override;
bool IsVideo() const override { return false; }
private:
virtual ~RTCEncodedAudioFrame();
RefPtr<RTCRtpScriptTransformer> mOwner;
RTCEncodedAudioFrameMetadata mMetadata;
};
} // namespace mozilla::dom
#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDAUDIOFRAME_H_

71
dom/media/webrtc/jsapi/RTCEncodedFrameBase.cpp Normal file
View File

@ -0,0 +1,71 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "jsapi/RTCEncodedFrameBase.h"
#include "nsIGlobalObject.h"
#include "mozilla/dom/ScriptSettings.h"
#include "js/ArrayBuffer.h"
namespace mozilla::dom {
NS_IMPL_CYCLE_COLLECTION_WITH_JS_MEMBERS(RTCEncodedFrameBase, (mGlobal),
(mData))
NS_IMPL_CYCLE_COLLECTING_ADDREF(RTCEncodedFrameBase)
NS_IMPL_CYCLE_COLLECTING_RELEASE(RTCEncodedFrameBase)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCEncodedFrameBase)
NS_INTERFACE_MAP_ENTRY(nsISupports)
NS_INTERFACE_MAP_END
RTCEncodedFrameBase::RTCEncodedFrameBase(
nsIGlobalObject* aGlobal,
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
uint64_t aCounter)
: mGlobal(aGlobal),
mFrame(std::move(aFrame)),
mCounter(aCounter),
mTimestamp(mFrame->GetTimestamp()) {
AutoJSAPI jsapi;
if (NS_WARN_IF(!jsapi.Init(mGlobal))) {
return;
}
// Avoid a copy
mData = JS::NewArrayBufferWithUserOwnedContents(
jsapi.cx(), mFrame->GetData().size(), (void*)(mFrame->GetData().data()));
}
RTCEncodedFrameBase::~RTCEncodedFrameBase() = default;
unsigned long RTCEncodedFrameBase::Timestamp() const { return mTimestamp; }
void RTCEncodedFrameBase::SetData(const ArrayBuffer& aData) {
mData.set(aData.Obj());
if (mFrame) {
aData.ComputeState();
mFrame->SetData(rtc::ArrayView<const uint8_t>(
static_cast<const uint8_t*>(aData.Data()), aData.Length()));
}
}
void RTCEncodedFrameBase::GetData(JSContext* aCx, JS::Rooted<JSObject*>* aObj) {
aObj->set(mData);
}
uint64_t RTCEncodedFrameBase::GetCounter() const { return mCounter; }
std::unique_ptr<webrtc::TransformableFrameInterface>
RTCEncodedFrameBase::TakeFrame() {
AutoJSAPI jsapi;
if (!jsapi.Init(mGlobal)) {
MOZ_CRASH("Could not init JSAPI!");
}
JS::Rooted<JSObject*> rootedData(jsapi.cx(), mData);
JS::DetachArrayBuffer(jsapi.cx(), rootedData);
return std::move(mFrame);
}
} // namespace mozilla::dom

58
dom/media/webrtc/jsapi/RTCEncodedFrameBase.h Normal file
View File

@ -0,0 +1,58 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDFRAMEBASE_H_
#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDFRAMEBASE_H_
#include "js/TypeDecls.h"
#include "mozilla/dom/TypedArray.h" // ArrayBuffer
#include "mozilla/Assertions.h"
#include "api/frame_transformer_interface.h"
#include <memory>
class nsIGlobalObject;
namespace mozilla::dom {
class RTCRtpScriptTransformer;
class RTCEncodedFrameBase : public nsISupports, public nsWrapperCache {
public:
explicit RTCEncodedFrameBase(
nsIGlobalObject* aGlobal,
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
uint64_t aCounter);
// nsISupports
NS_DECL_CYCLE_COLLECTING_ISUPPORTS
NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_CLASS(RTCEncodedFrameBase)
// Common webidl for RTCEncodedVideoFrame/RTCEncodedAudioFrame
unsigned long Timestamp() const;
void SetData(const ArrayBuffer& aData);
void GetData(JSContext* aCx, JS::Rooted<JSObject*>* aObj);
uint64_t GetCounter() const;
virtual bool CheckOwner(RTCRtpScriptTransformer* aOwner) const = 0;
std::unique_ptr<webrtc::TransformableFrameInterface> TakeFrame();
virtual bool IsVideo() const = 0;
protected:
virtual ~RTCEncodedFrameBase();
RefPtr<nsIGlobalObject> mGlobal;
std::unique_ptr<webrtc::TransformableFrameInterface> mFrame;
const uint64_t mCounter = 0;
const unsigned long mTimestamp = 0;
JS::Heap<JSObject*> mData;
};
} // namespace mozilla::dom
#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDFRAMEBASE_H_

117
dom/media/webrtc/jsapi/RTCEncodedVideoFrame.cpp Normal file
View File

@ -0,0 +1,117 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "jsapi/RTCEncodedVideoFrame.h"
#include <stdint.h>
#include <memory>
#include <string>
#include <utility>
#include "api/frame_transformer_interface.h"
#include "jsapi/RTCEncodedFrameBase.h"
#include "mozilla/dom/RTCEncodedVideoFrameBinding.h"
#include "mozilla/dom/RTCRtpScriptTransformer.h"
#include "nsWrapperCache.h"
#include "nsISupports.h"
#include "nsCycleCollectionParticipant.h"
#include "nsIGlobalObject.h"
#include "nsContentUtils.h"
#include "mozilla/RefPtr.h"
#include "mozilla/Unused.h"
#include "mozilla/fallible.h"
#include "mozilla/Maybe.h"
#include "js/RootingAPI.h"
namespace mozilla::dom {
NS_IMPL_CYCLE_COLLECTION_INHERITED(RTCEncodedVideoFrame, RTCEncodedFrameBase,
mOwner)
NS_IMPL_ADDREF_INHERITED(RTCEncodedVideoFrame, RTCEncodedFrameBase)
NS_IMPL_RELEASE_INHERITED(RTCEncodedVideoFrame, RTCEncodedFrameBase)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCEncodedVideoFrame)
NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
NS_INTERFACE_MAP_END_INHERITING(RTCEncodedFrameBase)
RTCEncodedVideoFrame::RTCEncodedVideoFrame(
nsIGlobalObject* aGlobal,
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
uint64_t aCounter, RTCRtpScriptTransformer* aOwner)
: RTCEncodedFrameBase(aGlobal, std::move(aFrame), aCounter),
mOwner(aOwner) {
const auto& videoFrame(
static_cast<webrtc::TransformableVideoFrameInterface&>(*mFrame));
mType = videoFrame.IsKeyFrame() ? RTCEncodedVideoFrameType::Key
: RTCEncodedVideoFrameType::Delta;
if (videoFrame.GetMetadata().GetFrameId().has_value()) {
mMetadata.mFrameId.Construct(*videoFrame.GetMetadata().GetFrameId());
}
mMetadata.mDependencies.Construct();
for (const auto dep : videoFrame.GetMetadata().GetFrameDependencies()) {
Unused << mMetadata.mDependencies.Value().AppendElement(
static_cast<unsigned long long>(dep), fallible);
}
mMetadata.mWidth.Construct(videoFrame.GetMetadata().GetWidth());
mMetadata.mHeight.Construct(videoFrame.GetMetadata().GetHeight());
if (videoFrame.GetMetadata().GetSpatialIndex() >= 0) {
mMetadata.mSpatialIndex.Construct(
videoFrame.GetMetadata().GetSpatialIndex());
}
if (videoFrame.GetMetadata().GetTemporalIndex() >= 0) {
mMetadata.mTemporalIndex.Construct(
videoFrame.GetMetadata().GetTemporalIndex());
}
mMetadata.mSynchronizationSource.Construct(videoFrame.GetSsrc());
mMetadata.mPayloadType.Construct(videoFrame.GetPayloadType());
mMetadata.mContributingSources.Construct();
for (const auto csrc : videoFrame.GetMetadata().GetCsrcs()) {
Unused << mMetadata.mContributingSources.Value().AppendElement(csrc,
fallible);
}
// The metadata timestamp is different, and not presently present in the
// libwebrtc types
if (!videoFrame.GetRid().empty()) {
mRid = Some(videoFrame.GetRid());
}
// Base class needs this, but can't do it itself because of an assertion in
// the cycle-collector.
mozilla::HoldJSObjects(this);
}
RTCEncodedVideoFrame::~RTCEncodedVideoFrame() {
// Base class needs this, but can't do it itself because of an assertion in
// the cycle-collector.
mozilla::DropJSObjects(this);
}
JSObject* RTCEncodedVideoFrame::WrapObject(JSContext* aCx,
JS::Handle<JSObject*> aGivenProto) {
return RTCEncodedVideoFrame_Binding::Wrap(aCx, this, aGivenProto);
}
nsIGlobalObject* RTCEncodedVideoFrame::GetParentObject() const {
return mGlobal;
}
RTCEncodedVideoFrameType RTCEncodedVideoFrame::Type() const { return mType; }
void RTCEncodedVideoFrame::GetMetadata(
RTCEncodedVideoFrameMetadata& aMetadata) {
aMetadata = mMetadata;
}
bool RTCEncodedVideoFrame::CheckOwner(RTCRtpScriptTransformer* aOwner) const {
return aOwner == mOwner;
}
Maybe<std::string> RTCEncodedVideoFrame::Rid() const { return mRid; }
} // namespace mozilla::dom

61
dom/media/webrtc/jsapi/RTCEncodedVideoFrame.h Normal file
View File

@ -0,0 +1,61 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDVIDEOFRAME_H_
#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDVIDEOFRAME_H_
#include "mozilla/RefPtr.h"
#include "nsIGlobalObject.h"
#include "jsapi/RTCEncodedFrameBase.h"
#include "mozilla/dom/RTCEncodedVideoFrameBinding.h"
namespace mozilla::dom {
class RTCRtpScriptTransformer;
// Wraps a libwebrtc frame, allowing the frame buffer to be modified, and
// providing read-only access to various metadata. After the libwebrtc frame is
// extracted (with RTCEncodedFrameBase::TakeFrame), the frame buffer is
// detached, but the metadata remains accessible.
class RTCEncodedVideoFrame final : public RTCEncodedFrameBase {
public:
explicit RTCEncodedVideoFrame(
nsIGlobalObject* aGlobal,
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame,
uint64_t aCounter, RTCRtpScriptTransformer* aOwner);
// nsISupports
NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(RTCEncodedVideoFrame,
RTCEncodedFrameBase)
// webidl (timestamp and data accessors live in base class)
JSObject* WrapObject(JSContext* aCx,
JS::Handle<JSObject*> aGivenProto) override;
nsIGlobalObject* GetParentObject() const;
RTCEncodedVideoFrameType Type() const;
void GetMetadata(RTCEncodedVideoFrameMetadata& aMetadata);
bool CheckOwner(RTCRtpScriptTransformer* aOwner) const override;
bool IsVideo() const override { return true; }
// Not in webidl right now. Might change.
// https://github.com/w3c/webrtc-encoded-transform/issues/147
Maybe<std::string> Rid() const;
private:
virtual ~RTCEncodedVideoFrame();
RefPtr<RTCRtpScriptTransformer> mOwner;
RTCEncodedVideoFrameType mType;
RTCEncodedVideoFrameMetadata mMetadata;
Maybe<std::string> mRid;
};
} // namespace mozilla::dom
#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCENCODEDVIDEOFRAME_H_

136
dom/media/webrtc/jsapi/RTCRtpReceiver.cpp
View File

@ -3,32 +3,82 @@
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "RTCRtpReceiver.h"
#include <stdint.h>
#include <vector>
#include <string>
#include <set>
#include "call/call.h"
#include "call/audio_receive_stream.h"
#include "call/video_receive_stream.h"
#include "api/rtp_parameters.h"
#include "api/units/timestamp.h"
#include "api/units/time_delta.h"
#include "system_wrappers/include/clock.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "RTCRtpTransceiver.h"
#include "PeerConnectionImpl.h"
#include "RTCStatsReport.h"
#include "mozilla/dom/RTCRtpReceiverBinding.h"
#include "mozilla/dom/RTCRtpSourcesBinding.h"
#include "mozilla/dom/RTCStatsReportBinding.h"
#include "jsep/JsepTransceiver.h"
#include "libwebrtcglue/MediaConduitControl.h"
#include "libwebrtcglue/MediaConduitInterface.h"
#include "transportbridge/MediaPipeline.h"
#include "sdp/SdpEnum.h"
#include "sdp/SdpAttribute.h"
#include "MediaTransportHandler.h"
#include "RemoteTrackSource.h"
#include "mozilla/dom/RTCRtpCapabilitiesBinding.h"
#include "transport/logging.h"
#include "mozilla/dom/MediaStreamTrack.h"
#include "mozilla/dom/Promise.h"
#include "mozilla/dom/Nullable.h"
#include "mozilla/dom/Document.h"
#include "mozilla/dom/AudioStreamTrack.h"
#include "mozilla/dom/VideoStreamTrack.h"
#include "mozilla/dom/RTCRtpScriptTransform.h"
#include "nsPIDOMWindow.h"
#include "PrincipalHandle.h"
#include "nsIPrincipal.h"
#include "mozilla/dom/Document.h"
#include "mozilla/NullPrincipal.h"
#include "MediaTrackGraph.h"
#include "RemoteTrackSource.h"
#include "libwebrtcglue/RtpRtcpConfig.h"
#include "nsString.h"
#include "mozilla/dom/AudioStreamTrack.h"
#include "mozilla/dom/VideoStreamTrack.h"
#include "MediaTransportHandler.h"
#include "jsep/JsepTransceiver.h"
#include "mozilla/dom/RTCRtpReceiverBinding.h"
#include "mozilla/dom/RTCRtpSourcesBinding.h"
#include "RTCStatsReport.h"
#include "nsStringFwd.h"
#include "MediaSegment.h"
#include "nsLiteralString.h"
#include "nsTArray.h"
#include "nsDOMNavigationTiming.h"
#include "MainThreadUtils.h"
#include "ErrorList.h"
#include "nsWrapperCache.h"
#include "nsISupports.h"
#include "nsCOMPtr.h"
#include "nsIScriptObjectPrincipal.h"
#include "nsCycleCollectionParticipant.h"
#include "nsDebug.h"
#include "nsThreadUtils.h"
#include "PerformanceRecorder.h"
#include "mozilla/NullPrincipal.h"
#include "mozilla/Preferences.h"
#include "PeerConnectionCtx.h"
#include "RTCRtpTransceiver.h"
#include "libwebrtcglue/AudioConduit.h"
#include "call/call.h"
#include "mozilla/StateMirroring.h"
#include "mozilla/Logging.h"
#include "mozilla/RefPtr.h"
#include "mozilla/AbstractThread.h"
#include "mozilla/StateWatching.h"
#include "mozilla/Maybe.h"
#include "mozilla/Assertions.h"
#include "mozilla/AlreadyAddRefed.h"
#include "mozilla/MozPromise.h"
#include "mozilla/UniquePtr.h"
#include "mozilla/fallible.h"
#include "mozilla/mozalloc_oom.h"
#include "mozilla/ErrorResult.h"
#include "js/RootingAPI.h"
namespace mozilla::dom {
@ -40,8 +90,8 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(RTCRtpReceiver)
NS_IMPL_CYCLE_COLLECTION_UNLINK_PRESERVED_WRAPPER
NS_IMPL_CYCLE_COLLECTION_UNLINK_END
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(RTCRtpReceiver)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mWindow, mPc, mTransceiver, mTrack,
mTrackSource)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mWindow, mPc, mTransceiver, mTransform,
mTrack, mTrackSource)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
NS_IMPL_CYCLE_COLLECTING_ADDREF(RTCRtpReceiver)
@ -89,7 +139,8 @@ RTCRtpReceiver::RTCRtpReceiver(
INIT_CANONICAL(mAudioCodecs, std::vector<AudioCodecConfig>()),
INIT_CANONICAL(mVideoCodecs, std::vector<VideoCodecConfig>()),
INIT_CANONICAL(mVideoRtpRtcpConfig, Nothing()),
INIT_CANONICAL(mReceiving, false) {
INIT_CANONICAL(mReceiving, false),
INIT_CANONICAL(mFrameTransformerProxy, nullptr) {
PrincipalHandle principalHandle = GetPrincipalHandle(aWindow, aPrivacy);
const bool isAudio = aConduit->type() == MediaSessionConduit::AUDIO;
@ -605,6 +656,9 @@ void RTCRtpReceiver::Shutdown() {
mRtcpByeListener.DisconnectIfExists();
mRtcpTimeoutListener.DisconnectIfExists();
mUnmuteListener.DisconnectIfExists();
if (mTransform) {
mTransform->GetProxy().SetReceiver(nullptr);
}
}
void RTCRtpReceiver::BreakCycles() {
@ -937,6 +991,48 @@ const JsepTransceiver& RTCRtpReceiver::GetJsepTransceiver() const {
return mTransceiver->GetJsepTransceiver();
}
void RTCRtpReceiver::SetTransform(RTCRtpScriptTransform* aTransform,
ErrorResult& aError) {
if (aTransform == mTransform.get()) {
// Ok... smile and nod
// TODO: Depending on spec, this might throw
// https://github.com/w3c/webrtc-encoded-transform/issues/189
return;
}
if (aTransform && aTransform->IsClaimed()) {
aError.ThrowInvalidStateError("transform has already been used elsewhere");
return;
}
if (aTransform) {
mFrameTransformerProxy = &aTransform->GetProxy();
} else {
mFrameTransformerProxy = nullptr;
}
if (mTransform) {
mTransform->GetProxy().SetReceiver(nullptr);
}
mTransform = const_cast<RTCRtpScriptTransform*>(aTransform);
if (mTransform) {
mTransform->GetProxy().SetReceiver(this);
mTransform->SetClaimed();
}
}
void RTCRtpReceiver::RequestKeyFrame() {
if (!mTransform || !mPipeline) {
return;
}
mPipeline->mConduit->AsVideoSessionConduit().apply([&](const auto& conduit) {
conduit->RequestKeyFrame(&mTransform->GetProxy());
});
}
} // namespace mozilla::dom
#undef LOGTAG

16
dom/media/webrtc/jsapi/RTCRtpReceiver.h
View File

@ -38,6 +38,7 @@ struct RTCRtpCapabilities;
struct RTCRtpContributingSource;
struct RTCRtpSynchronizationSource;
class RTCRtpTransceiver;
class RTCRtpScriptTransform;
class RTCRtpReceiver : public nsISupports,
public nsWrapperCache,
@ -72,6 +73,10 @@ class RTCRtpReceiver : public nsISupports,
const uint32_t aSource, const DOMHighResTimeStamp aTimestamp,
const uint32_t aRtpTimestamp, const bool aHasLevel, const uint8_t aLevel);
RTCRtpScriptTransform* GetTransform() const { return mTransform; }
void SetTransform(RTCRtpScriptTransform* aTransform, ErrorResult& aError);
nsPIDOMWindowInner* GetParentObject() const;
nsTArray<RefPtr<RTCStatsPromise>> GetStatsInternal(
bool aSkipIceStats = false);
@ -120,6 +125,9 @@ class RTCRtpReceiver : public nsISupports,
// ALPN negotiation.
void UpdatePrincipalPrivacy(PrincipalPrivacy aPrivacy);
// Called by FrameTransformerProxy
void RequestKeyFrame();
void OnRtcpBye();
void OnRtcpTimeout();
@ -141,11 +149,17 @@ class RTCRtpReceiver : public nsISupports,
Canonical<std::vector<VideoCodecConfig>>& CanonicalVideoCodecs() {
return mVideoCodecs;
}
Canonical<Maybe<RtpRtcpConfig>>& CanonicalVideoRtpRtcpConfig() {
return mVideoRtpRtcpConfig;
}
Canonical<bool>& CanonicalReceiving() override { return mReceiving; }
Canonical<RefPtr<FrameTransformerProxy>>& CanonicalFrameTransformerProxy() {
return mFrameTransformerProxy;
}
private:
virtual ~RTCRtpReceiver();
@ -168,6 +182,7 @@ class RTCRtpReceiver : public nsISupports,
RefPtr<MediaPipelineReceive> mPipeline;
RefPtr<MediaTransportHandler> mTransportHandler;
RefPtr<RTCRtpTransceiver> mTransceiver;
RefPtr<RTCRtpScriptTransform> mTransform;
// This is [[AssociatedRemoteMediaStreams]], basically. We do not keep the
// streams themselves here, because that would require this object to know
// where the stream list for the whole RTCPeerConnection lives..
@ -191,6 +206,7 @@ class RTCRtpReceiver : public nsISupports,
Canonical<std::vector<VideoCodecConfig>> mVideoCodecs;
Canonical<Maybe<RtpRtcpConfig>> mVideoRtpRtcpConfig;
Canonical<bool> mReceiving;
Canonical<RefPtr<FrameTransformerProxy>> mFrameTransformerProxy;
};
} // namespace dom

84
dom/media/webrtc/jsapi/RTCRtpScriptTransform.cpp Normal file
View File

@ -0,0 +1,84 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "RTCRtpScriptTransform.h"
#include "libwebrtcglue/FrameTransformerProxy.h"
#include "jsapi/RTCTransformEventRunnable.h"
#include "mozilla/dom/Promise.h"
#include "mozilla/dom/Worker.h"
#include "mozilla/dom/RTCRtpScriptTransformBinding.h"
#include "mozilla/dom/MessagePortBinding.h"
#include "mozilla/Logging.h"
#include "mozilla/AlreadyAddRefed.h"
#include "mozilla/RefPtr.h"
#include "nsPIDOMWindow.h"
#include "nsContentUtils.h"
#include "nsCOMPtr.h"
#include "nsDebug.h"
#include "ErrorList.h"
#include "nsWrapperCache.h"
#include "nsISupports.h"
#include "nsCycleCollectionParticipant.h"
#include "js/RootingAPI.h"
namespace mozilla::dom {
LazyLogModule gScriptTransformLog("RTCRtpScriptTransform");
NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(RTCRtpScriptTransform, mWindow)
NS_IMPL_CYCLE_COLLECTING_ADDREF(RTCRtpScriptTransform)
NS_IMPL_CYCLE_COLLECTING_RELEASE(RTCRtpScriptTransform)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCRtpScriptTransform)
NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
NS_INTERFACE_MAP_ENTRY(nsISupports)
NS_INTERFACE_MAP_END
already_AddRefed<RTCRtpScriptTransform> RTCRtpScriptTransform::Constructor(
const GlobalObject& aGlobal, Worker& aWorker,
JS::Handle<JS::Value> aOptions,
const Optional<Sequence<JSObject*>>& aTransfer, ErrorResult& aRv) {
nsCOMPtr<nsPIDOMWindowInner> ownerWindow =
do_QueryInterface(aGlobal.GetAsSupports());
if (NS_WARN_IF(!ownerWindow)) {
aRv.Throw(NS_ERROR_FAILURE);
return nullptr;
}
auto newTransform = MakeRefPtr<RTCRtpScriptTransform>(ownerWindow);
RefPtr<RTCTransformEventRunnable> runnable =
new RTCTransformEventRunnable(aWorker, &newTransform->GetProxy());
if (aTransfer.WasPassed()) {
aWorker.PostEventWithOptions(aGlobal.Context(), aOptions, aTransfer.Value(),
runnable, aRv);
} else {
StructuredSerializeOptions transferOptions;
aWorker.PostEventWithOptions(aGlobal.Context(), aOptions,
transferOptions.mTransfer, runnable, aRv);
}
if (NS_WARN_IF(aRv.Failed())) {
return nullptr;
}
return newTransform.forget();
}
RTCRtpScriptTransform::RTCRtpScriptTransform(nsPIDOMWindowInner* aWindow)
: mWindow(aWindow), mProxy(new FrameTransformerProxy) {}
RTCRtpScriptTransform::~RTCRtpScriptTransform() {
mProxy->ReleaseScriptTransformer();
}
JSObject* RTCRtpScriptTransform::WrapObject(JSContext* aCx,
JS::Handle<JSObject*> aGivenProto) {
return RTCRtpScriptTransform_Binding::Wrap(aCx, this, aGivenProto);
}
} // namespace mozilla::dom
#undef LOGTAG

63
dom/media/webrtc/jsapi/RTCRtpScriptTransform.h Normal file
View File

@ -0,0 +1,63 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCRTPSCRIPTTRANSFORM_H_
#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCRTPSCRIPTTRANSFORM_H_
#include "nsISupports.h"
#include "nsWrapperCache.h"
#include "mozilla/RefPtr.h"
#include "mozilla/Maybe.h"
#include "js/RootingAPI.h"
#include "nsTArray.h"
class nsPIDOMWindowInner;
namespace mozilla {
class FrameTransformerProxy;
class ErrorResult;
namespace dom {
class Worker;
class GlobalObject;
template <typename T>
class Sequence;
template <typename T>
class Optional;
class RTCRtpScriptTransform : public nsISupports, public nsWrapperCache {
public:
static already_AddRefed<RTCRtpScriptTransform> Constructor(
const GlobalObject& aGlobal, Worker& aWorker,
JS::Handle<JS::Value> aOptions,
const Optional<Sequence<JSObject*>>& aTransfer, ErrorResult& aRv);
explicit RTCRtpScriptTransform(nsPIDOMWindowInner* aWindow);
// nsISupports
NS_DECL_CYCLE_COLLECTING_ISUPPORTS
NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(RTCRtpScriptTransform)
JSObject* WrapObject(JSContext* aCx,
JS::Handle<JSObject*> aGivenProto) override;
nsPIDOMWindowInner* GetParentObject() const { return mWindow; }
FrameTransformerProxy& GetProxy() { return *mProxy; }
bool IsClaimed() const { return mClaimed; }
void SetClaimed() { mClaimed = true; }
private:
virtual ~RTCRtpScriptTransform();
RefPtr<nsPIDOMWindowInner> mWindow;
RefPtr<FrameTransformerProxy> mProxy;
bool mClaimed = false;
};
} // namespace dom
} // namespace mozilla
#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCRTPSCRIPTTRANSFORM_H_

449
dom/media/webrtc/jsapi/RTCRtpScriptTransformer.cpp Normal file
View File

@ -0,0 +1,449 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "RTCRtpScriptTransformer.h"
#include <stdint.h>
#include <utility>
#include <memory>
#include <string>
#include "api/frame_transformer_interface.h"
#include "nsString.h"
#include "nsCycleCollectionParticipant.h"
#include "nsISupports.h"
#include "ErrorList.h"
#include "nsDebug.h"
#include "nsCycleCollectionTraversalCallback.h"
#include "nsTArray.h"
#include "nsWrapperCache.h"
#include "nsIGlobalObject.h"
#include "nsCOMPtr.h"
#include "nsStringFwd.h"
#include "nsLiteralString.h"
#include "nsContentUtils.h"
#include "mozilla/RefPtr.h"
#include "mozilla/AlreadyAddRefed.h"
#include "mozilla/ErrorResult.h"
#include "mozilla/Result.h"
#include "mozilla/HoldDropJSObjects.h"
#include "mozilla/Maybe.h"
#include "mozilla/Assertions.h"
#include "mozilla/Logging.h"
#include "mozilla/ErrorResult.h"
#include "mozilla/Likely.h"
#include "mozilla/dom/RTCRtpScriptTransformerBinding.h"
#include "mozilla/dom/BindingUtils.h"
#include "mozilla/dom/BindingDeclarations.h"
#include "mozilla/dom/PrototypeList.h"
#include "mozilla/dom/WorkerRef.h"
#include "mozilla/dom/RTCEncodedAudioFrame.h"
#include "mozilla/dom/RTCEncodedVideoFrame.h"
#include "mozilla/dom/UnderlyingSourceCallbackHelpers.h"
#include "mozilla/dom/ToJSValue.h"
#include "mozilla/dom/ReadableStream.h"
#include "mozilla/dom/WritableStream.h"
#include "mozilla/dom/UnderlyingSinkCallbackHelpers.h"
#include "mozilla/dom/WritableStreamDefaultController.h"
#include "mozilla/dom/ReadableStreamController.h"
#include "mozilla/dom/Promise.h"
#include "mozilla/dom/Promise-inl.h"
#include "js/RootingAPI.h"
#include "js/Value.h"
#include "js/CallArgs.h"
#include "libwebrtcglue/FrameTransformerProxy.h"
#include "sdp/SdpAttribute.h" // CheckRidValidity
namespace mozilla::dom {
LazyLogModule gScriptTransformerLog("RTCRtpScriptTransformer");
NS_IMPL_CYCLE_COLLECTION_INHERITED(nsISupportsStreamSource,
UnderlyingSourceAlgorithmsWrapper, mStream,
mThingQueuedPromise, mQueue)
NS_IMPL_ADDREF_INHERITED(nsISupportsStreamSource,
UnderlyingSourceAlgorithmsWrapper)
NS_IMPL_RELEASE_INHERITED(nsISupportsStreamSource,
UnderlyingSourceAlgorithmsWrapper)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(nsISupportsStreamSource)
NS_INTERFACE_MAP_END_INHERITING(UnderlyingSourceAlgorithmsWrapper)
nsISupportsStreamSource::nsISupportsStreamSource() = default;
nsISupportsStreamSource::~nsISupportsStreamSource() = default;
void nsISupportsStreamSource::Init(ReadableStream* aStream) {
mStream = aStream;
}
void nsISupportsStreamSource::Enqueue(nsISupports* aThing) {
if (!mThingQueuedPromise) {
mQueue.AppendElement(aThing);
return;
}
// Maybe put a limit here? Or at least some sort of logging if this gets
// unreasonably long?
AutoJSAPI jsapi;
if (NS_WARN_IF(!jsapi.Init(mStream->GetParentObject()))) {
return;
}
EnqueueToStream(jsapi.cx(), aThing);
mThingQueuedPromise->MaybeResolveWithUndefined();
mThingQueuedPromise = nullptr;
}
already_AddRefed<Promise> nsISupportsStreamSource::PullCallbackImpl(
JSContext* aCx, ReadableStreamController& aController, ErrorResult& aRv) {
if (!mQueue.IsEmpty()) {
EnqueueOneThingFromQueue(aCx);
return nullptr;
}
RefPtr<nsISupportsStreamSource> self(this);
mThingQueuedPromise = Promise::CreateInfallible(mStream->GetParentObject());
return do_AddRef(mThingQueuedPromise);
}
void nsISupportsStreamSource::EnqueueToStream(JSContext* aCx,
nsISupports* aThing) {
JS::Rooted<JS::Value> jsThing(aCx);
if (NS_WARN_IF(MOZ_UNLIKELY(!ToJSValue(aCx, *aThing, &jsThing)))) {
// Do we want to add error handling for this?
return;
}
IgnoredErrorResult rv;
// EnqueueNative is CAN_RUN_SCRIPT. Need a strong-ref temporary.
auto stream = mStream;
stream->EnqueueNative(aCx, jsThing, rv);
}
void nsISupportsStreamSource::EnqueueOneThingFromQueue(JSContext* aCx) {
if (!mQueue.IsEmpty()) {
RefPtr<nsISupports> thing = mQueue[0];
mQueue.RemoveElementAt(0);
EnqueueToStream(aCx, thing);
}
}
NS_IMPL_CYCLE_COLLECTION_INHERITED(WritableStreamRTCFrameSink,
UnderlyingSinkAlgorithmsWrapper,
mTransformer)
NS_IMPL_ADDREF_INHERITED(WritableStreamRTCFrameSink,
UnderlyingSinkAlgorithmsWrapper)
NS_IMPL_RELEASE_INHERITED(WritableStreamRTCFrameSink,
UnderlyingSinkAlgorithmsWrapper)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(WritableStreamRTCFrameSink)
NS_INTERFACE_MAP_END_INHERITING(UnderlyingSinkAlgorithmsWrapper)
WritableStreamRTCFrameSink::WritableStreamRTCFrameSink(
RTCRtpScriptTransformer* aTransformer)
: mTransformer(aTransformer) {}
WritableStreamRTCFrameSink::~WritableStreamRTCFrameSink() = default;
already_AddRefed<Promise> WritableStreamRTCFrameSink::WriteCallback(
JSContext* aCx, JS::Handle<JS::Value> aChunk,
WritableStreamDefaultController& aController, ErrorResult& aError) {
// Spec does not say to do this right now. Might be a spec bug, needs
// clarification.
// https://github.com/w3c/webrtc-encoded-transform/issues/191
if (NS_WARN_IF(!aChunk.isObject())) {
aError.ThrowTypeError(
"Wrong type for RTCRtpScriptTransformer.[[writeable]]: Not an object");
return nullptr;
}
// Lame. But, without a webidl base class, this is the only way.
RefPtr<RTCEncodedVideoFrame> video;
UNWRAP_OBJECT(RTCEncodedVideoFrame, &aChunk.toObject(), video);
RefPtr<RTCEncodedAudioFrame> audio;
UNWRAP_OBJECT(RTCEncodedAudioFrame, &aChunk.toObject(), audio);
RefPtr<RTCEncodedFrameBase> frame;
if (video) {
frame = video;
} else if (audio) {
frame = audio;
}
if (NS_WARN_IF(!frame)) {
aError.ThrowTypeError(
"Wrong type for RTCRtpScriptTransformer.[[writeable]]: Not an "
"RTCEncodedAudioFrame or RTCEncodedVideoFrame");
return nullptr;
}
return mTransformer->OnTransformedFrame(frame, aError);
}
// There is not presently an implementation of these for nsTHashMap :(
inline void ImplCycleCollectionUnlink(
RTCRtpScriptTransformer::GenerateKeyFramePromises& aMap) {
for (auto& tableEntry : aMap) {
ImplCycleCollectionUnlink(*tableEntry.GetModifiableData());
}
aMap.Clear();
}
inline void ImplCycleCollectionTraverse(
nsCycleCollectionTraversalCallback& aCallback,
RTCRtpScriptTransformer::GenerateKeyFramePromises& aMap, const char* aName,
uint32_t aFlags = 0) {
for (auto& tableEntry : aMap) {
ImplCycleCollectionTraverse(aCallback, *tableEntry.GetModifiableData(),
aName, aFlags);
}
}
NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_WITH_JS_MEMBERS(
RTCRtpScriptTransformer,
(mGlobal, mReadableSource, mReadable, mWritable, mWritableSink,
mKeyFrameRequestPromises, mGenerateKeyFramePromises),
(mOptions))
NS_IMPL_CYCLE_COLLECTING_ADDREF(RTCRtpScriptTransformer)
NS_IMPL_CYCLE_COLLECTING_RELEASE(RTCRtpScriptTransformer)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(RTCRtpScriptTransformer)
NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
NS_INTERFACE_MAP_ENTRY(nsISupports)
NS_INTERFACE_MAP_END
RTCRtpScriptTransformer::RTCRtpScriptTransformer(nsIGlobalObject* aGlobal)
: mGlobal(aGlobal),
mReadableSource(new nsISupportsStreamSource),
mWritableSink(new WritableStreamRTCFrameSink(this)),
mOptions(JS::UndefinedHandleValue) {
mozilla::HoldJSObjects(this);
}
RTCRtpScriptTransformer::~RTCRtpScriptTransformer() {
mozilla::DropJSObjects(this);
}
nsresult RTCRtpScriptTransformer::Init(JSContext* aCx,
JS::Handle<JS::Value> aOptions,
WorkerPrivate* aWorkerPrivate,
FrameTransformerProxy* aProxy) {
ErrorResult rv;
RefPtr<nsIGlobalObject> global(mGlobal);
auto source = mReadableSource;
auto sink = mWritableSink;
// NOTE: We do not transfer these streams from mainthread, as the spec says,
// because there's no JS observable reason to. The spec is likely to change
// here, because it is overspecifying implementation details.
mReadable = ReadableStream::CreateNative(aCx, global, *source, Some(1.0),
nullptr, rv);
if (rv.Failed()) {
return rv.StealNSResult();
}
mReadableSource->Init(mReadable);
// WritableStream::CreateNative takes a nsIGlobalObject&, but
// ReadableStream::CreateNative takes a nsIGlobalObject*?
mWritable =
WritableStream::CreateNative(aCx, *global, *sink, Nothing(), nullptr, rv);
if (rv.Failed()) {
return rv.StealNSResult();
}
mOptions = aOptions;
mProxy = aProxy;
// This will return null if the worker is already shutting down.
// A call to ReleaseScriptTransformer will eventually result in a call to
// NotifyReleased.
mWorkerRef = StrongWorkerRef::Create(
aWorkerPrivate, "RTCRtpScriptTransformer",
[this, self = RefPtr(this)]() { mProxy->ReleaseScriptTransformer(); });
if (mWorkerRef) {
mProxy->SetScriptTransformer(*this);
}
return NS_OK;
}
void RTCRtpScriptTransformer::NotifyReleased() {
RejectPendingPromises();
mWorkerRef = nullptr;
mProxy = nullptr;
}
void RTCRtpScriptTransformer::RejectPendingPromises() {
for (const auto& promise : mKeyFrameRequestPromises) {
ErrorResult rv;
rv.ThrowInvalidStateError(
"RTCRtpScriptTransformer is not associated with a receiver");
promise->MaybeReject(std::move(rv));
}
mKeyFrameRequestPromises.Clear();
// GenerateKeyFrame promises are indexed by rid
for (auto& ridAndPromises : mGenerateKeyFramePromises) {
for (const auto& promise : ridAndPromises.GetData()) {
ErrorResult rv;
rv.ThrowInvalidStateError(
"RTCRtpScriptTransformer is not associated with a sender");
promise->MaybeReject(std::move(rv));
}
}
mGenerateKeyFramePromises.Clear();
}
void RTCRtpScriptTransformer::TransformFrame(
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame) {
if (!mVideo.isSome()) {
// First frame. mProxy will know whether it's video or not by now.
mVideo = mProxy->IsVideo();
MOZ_ASSERT(mVideo.isSome());
}
RefPtr<RTCEncodedFrameBase> domFrame;
if (*mVideo) {
// If this is a send video keyframe, resolve any pending GenerateKeyFrame
// promises for its rid.
if (aFrame->GetDirection() ==
webrtc::TransformableFrameInterface::Direction::kSender) {
auto* videoFrame =
static_cast<webrtc::TransformableVideoFrameInterface*>(aFrame.get());
if (videoFrame->IsKeyFrame()) {
ResolveGenerateKeyFramePromises(videoFrame->GetRid(),
videoFrame->GetTimestamp());
if (!videoFrame->GetRid().empty() &&
videoFrame->GetMetadata().GetSimulcastIdx() == 0) {
ResolveGenerateKeyFramePromises("", videoFrame->GetTimestamp());
}
}
}
domFrame = new RTCEncodedVideoFrame(mGlobal, std::move(aFrame),
++mLastEnqueuedFrameCounter, this);
} else {
domFrame = new RTCEncodedAudioFrame(mGlobal, std::move(aFrame),
++mLastEnqueuedFrameCounter, this);
}
mReadableSource->Enqueue(domFrame);
}
void RTCRtpScriptTransformer::GetOptions(JSContext* aCx,
JS::MutableHandle<JS::Value> aVal,
ErrorResult& aError) {
if (!ToJSValue(aCx, mOptions, aVal)) {
aError.NoteJSContextException(aCx);
}
}
already_AddRefed<Promise> RTCRtpScriptTransformer::GenerateKeyFrame(
const Optional<nsAString>& aRid) {
Maybe<std::string> utf8Rid;
if (aRid.WasPassed()) {
utf8Rid = Some(NS_ConvertUTF16toUTF8(aRid.Value()).get());
std::string error;
if (!SdpRidAttributeList::CheckRidValidity(*utf8Rid, &error)) {
ErrorResult rv;
nsCString nsError(error.c_str());
rv.ThrowNotAllowedError(nsError);
return Promise::CreateRejectedWithErrorResult(GetParentObject(), rv);
}
}
nsCString key;
if (utf8Rid.isSome()) {
key.Assign(utf8Rid->data(), utf8Rid->size());
}
nsTArray<RefPtr<Promise>>& promises =
mGenerateKeyFramePromises.LookupOrInsert(key);
if (!promises.Length()) {
// No pending keyframe generation request for this rid. Make one.
if (!mProxy || !mProxy->GenerateKeyFrame(utf8Rid)) {
ErrorResult rv;
rv.ThrowInvalidStateError(
"RTCRtpScriptTransformer is not associated with a video sender");
return Promise::CreateRejectedWithErrorResult(GetParentObject(), rv);
}
}
RefPtr<Promise> promise = Promise::CreateInfallible(GetParentObject());
promises.AppendElement(promise);
return promise.forget();
}
void RTCRtpScriptTransformer::ResolveGenerateKeyFramePromises(
const std::string& aRid, uint64_t aTimestamp) {
nsCString key(aRid.data(), aRid.size());
nsTArray<RefPtr<Promise>> promises;
mGenerateKeyFramePromises.Remove(key, &promises);
for (auto& promise : promises) {
promise->MaybeResolve(aTimestamp);
}
}
void RTCRtpScriptTransformer::GenerateKeyFrameError(
const Maybe<std::string>& aRid, const CopyableErrorResult& aResult) {
nsCString key;
if (aRid.isSome()) {
key.Assign(aRid->data(), aRid->size());
}
nsTArray<RefPtr<Promise>> promises;
mGenerateKeyFramePromises.Remove(key, &promises);
for (auto& promise : promises) {
CopyableErrorResult rv(aResult);
promise->MaybeReject(std::move(rv));
}
}
already_AddRefed<Promise> RTCRtpScriptTransformer::SendKeyFrameRequest() {
if (!mKeyFrameRequestPromises.Length()) {
if (!mProxy || !mProxy->RequestKeyFrame()) {
ErrorResult rv;
rv.ThrowInvalidStateError(
"RTCRtpScriptTransformer is not associated with a video receiver");
return Promise::CreateRejectedWithErrorResult(GetParentObject(), rv);
}
}
RefPtr<Promise> promise = Promise::CreateInfallible(GetParentObject());
mKeyFrameRequestPromises.AppendElement(promise);
return promise.forget();
}
void RTCRtpScriptTransformer::KeyFrameRequestDone(bool aSuccess) {
auto promises = std::move(mKeyFrameRequestPromises);
if (aSuccess) {
for (const auto& promise : promises) {
promise->MaybeResolveWithUndefined();
}
} else {
for (const auto& promise : promises) {
ErrorResult rv;
rv.ThrowInvalidStateError(
"Depacketizer is not defined, or not processing");
promise->MaybeReject(std::move(rv));
}
}
}
JSObject* RTCRtpScriptTransformer::WrapObject(
JSContext* aCx, JS::Handle<JSObject*> aGivenProto) {
return RTCRtpScriptTransformer_Binding::Wrap(aCx, this, aGivenProto);
}
already_AddRefed<Promise> RTCRtpScriptTransformer::OnTransformedFrame(
RTCEncodedFrameBase* aFrame, ErrorResult& aError) {
// Spec says to skip frames that are out of order or have wrong owner
if (aFrame->GetCounter() > mLastReceivedFrameCounter &&
aFrame->CheckOwner(this) && mProxy) {
mLastReceivedFrameCounter = aFrame->GetCounter();
mProxy->OnTransformedFrame(aFrame->TakeFrame());
}
return Promise::CreateResolvedWithUndefined(GetParentObject(), aError);
}
} // namespace mozilla::dom
#undef LOGTAG

197
dom/media/webrtc/jsapi/RTCRtpScriptTransformer.h Normal file
View File

@ -0,0 +1,197 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCRTPSCRIPTTRANSFORMER_H_
#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCRTPSCRIPTTRANSFORMER_H_
#include "nsISupports.h"
#include "nsWrapperCache.h"
#include "mozilla/RefPtr.h"
#include "mozilla/dom/ReadableStream.h"
#include "mozilla/dom/WritableStream.h"
#include "mozilla/Maybe.h"
#include "js/RootingAPI.h"
#include "nsTArray.h"
#include "nsCOMArray.h"
#include <memory>
#include "nsTHashSet.h"
#include "nsCycleCollectionParticipant.h"
class nsPIDOMWindowInner;
namespace webrtc {
class TransformableFrameInterface;
}
namespace mozilla {
class FrameTransformerProxy;
namespace dom {
class Worker;
class WorkerPrivate;
// Dirt simple source for ReadableStream that accepts nsISupports
// Might be suitable to move someplace else, with some polish.
class nsISupportsStreamSource final : public UnderlyingSourceAlgorithmsWrapper {
public:
nsISupportsStreamSource();
nsISupportsStreamSource(const nsISupportsStreamSource&) = delete;
nsISupportsStreamSource(nsISupportsStreamSource&&) = delete;
nsISupportsStreamSource& operator=(const nsISupportsStreamSource&) = delete;
nsISupportsStreamSource& operator=(nsISupportsStreamSource&&) = delete;
NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(nsISupportsStreamSource,
UnderlyingSourceAlgorithmsWrapper)
void Init(ReadableStream* aStream);
void Enqueue(nsISupports* aThing);
// From UnderlyingSourceAlgorithmsWrapper
already_AddRefed<Promise> PullCallbackImpl(
JSContext* aCx, ReadableStreamController& aController,
ErrorResult& aRv) override;
void EnqueueOneThingFromQueue(JSContext* aCx);
private:
virtual ~nsISupportsStreamSource();
// Calls ReadableStream::EnqueueNative, which is MOZ_CAN_RUN_SCRIPT.
MOZ_CAN_RUN_SCRIPT_BOUNDARY void EnqueueToStream(JSContext* aCx,
nsISupports* aThing);
RefPtr<ReadableStream> mStream;
RefPtr<Promise> mThingQueuedPromise;
// mozilla::Queue is not cycle-collector friendly :(
nsCOMArray<nsISupports> mQueue;
};
class RTCRtpScriptTransformer;
class WritableStreamRTCFrameSink final
: public UnderlyingSinkAlgorithmsWrapper {
public:
explicit WritableStreamRTCFrameSink(RTCRtpScriptTransformer* aTransformer);
WritableStreamRTCFrameSink(const WritableStreamRTCFrameSink&) = delete;
WritableStreamRTCFrameSink(WritableStreamRTCFrameSink&&) = delete;
WritableStreamRTCFrameSink& operator=(const WritableStreamRTCFrameSink&) =
delete;
WritableStreamRTCFrameSink& operator=(WritableStreamRTCFrameSink&&) = delete;
NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(WritableStreamRTCFrameSink,
UnderlyingSinkAlgorithmsWrapper)
already_AddRefed<Promise> WriteCallback(
JSContext* aCx, JS::Handle<JS::Value> aChunk,
WritableStreamDefaultController& aController,
ErrorResult& aError) override;
private:
virtual ~WritableStreamRTCFrameSink();
RefPtr<RTCRtpScriptTransformer> mTransformer;
};
class RTCEncodedFrameBase;
// Here's the basic flow. All of this happens on the worker thread.
// 0. We register with a FrameTransformerProxy.
// 1. That FrameTransformerProxy dispatches webrtc::TransformableFrameInterface
// to us (from either the encoder/depacketizer thread), via our
// TransformFrame method.
// 2. We wrap these frames in RTCEncodedAudio/VideoFrame, and feed them to
// mReadableSource, which queues them.
// 3. mReadableSource.PullCallbackImpl consumes that queue, and feeds the
// frames to mReadable.
// 4. JS worker code consumes from mReadable, does whatever transformation it
// wants, then writes the frames to mWritable.
// 5. mWritableSink.WriteCallback passes those frames to us.
// 6. We unwrap the webrtc::TransformableFrameInterface from these frames.
// 7. We pass these unwrapped frames back to the FrameTransformerProxy.
// (FrameTransformerProxy handles any dispatching/synchronization necessary)
// 8. Eventually, that FrameTransformerProxy calls NotifyReleased (possibly at
// our prompting).
class RTCRtpScriptTransformer final : public nsISupports,
public nsWrapperCache {
public:
explicit RTCRtpScriptTransformer(nsIGlobalObject* aGlobal);
MOZ_CAN_RUN_SCRIPT_BOUNDARY nsresult Init(JSContext* aCx,
JS::Handle<JS::Value> aOptions,
WorkerPrivate* aWorkerPrivate,
FrameTransformerProxy* aProxy);
void NotifyReleased();
void TransformFrame(
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame);
already_AddRefed<Promise> OnTransformedFrame(RTCEncodedFrameBase* aFrame,
ErrorResult& aError);
// nsISupports
NS_DECL_CYCLE_COLLECTING_ISUPPORTS
NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_CLASS(RTCRtpScriptTransformer)
JSObject* WrapObject(JSContext* aCx,
JS::Handle<JSObject*> aGivenProto) override;
nsIGlobalObject* GetParentObject() const { return mGlobal; }
// WebIDL Interface
already_AddRefed<mozilla::dom::ReadableStream> Readable() const {
return do_AddRef(mReadable);
}
already_AddRefed<mozilla::dom::WritableStream> Writable() const {
return do_AddRef(mWritable);
}
void GetOptions(JSContext* aCx, JS::MutableHandle<JS::Value> aVal,
ErrorResult& aError);
already_AddRefed<Promise> GenerateKeyFrame(const Optional<nsAString>& aRid);
void GenerateKeyFrameError(const Maybe<std::string>& aRid,
const CopyableErrorResult& aResult);
already_AddRefed<Promise> SendKeyFrameRequest();
void KeyFrameRequestDone(bool aSuccess);
// Public to ease implementation of cycle collection functions
using GenerateKeyFramePromises =
nsTHashMap<nsCStringHashKey, nsTArray<RefPtr<Promise>>>;
private:
virtual ~RTCRtpScriptTransformer();
void RejectPendingPromises();
// empty string means no rid
void ResolveGenerateKeyFramePromises(const std::string& aRid,
uint64_t aTimestamp);
nsCOMPtr<nsIGlobalObject> mGlobal;
RefPtr<FrameTransformerProxy> mProxy;
RefPtr<nsISupportsStreamSource> mReadableSource;
RefPtr<ReadableStream> mReadable;
RefPtr<WritableStream> mWritable;
RefPtr<WritableStreamRTCFrameSink> mWritableSink;
JS::Heap<JS::Value> mOptions;
uint64_t mLastEnqueuedFrameCounter = 0;
uint64_t mLastReceivedFrameCounter = 0;
nsTArray<RefPtr<Promise>> mKeyFrameRequestPromises;
// Contains the promise returned for each call to GenerateKeyFrame(rid), in
// the order in which it was called, keyed by the rid (empty string if not
// passed). If there is already a promise in here for a given rid, we do not
// ask the FrameTransformerProxy again, and just bulk resolve/reject.
GenerateKeyFramePromises mGenerateKeyFramePromises;
Maybe<bool> mVideo;
RefPtr<StrongWorkerRef> mWorkerRef;
};
} // namespace dom
} // namespace mozilla
#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCRTPSCRIPTTRANSFORMER_H_

125
dom/media/webrtc/jsapi/RTCRtpSender.cpp
View File

@ -3,22 +3,77 @@
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "RTCRtpSender.h"
#include "transport/logging.h"
#include "mozilla/dom/MediaStreamTrack.h"
#include "mozilla/dom/Promise.h"
#include "mozilla/glean/GleanMetrics.h"
#include <stdint.h>
#include <vector>
#include <string>
#include <algorithm>
#include <utility>
#include <iterator>
#include <set>
#include <sstream>
#include "system_wrappers/include/clock.h"
#include "call/call.h"
#include "api/rtp_parameters.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "api/video_codecs/video_codec.h"
#include "api/video/video_codec_constants.h"
#include "call/audio_send_stream.h"
#include "call/video_send_stream.h"
#include "modules/rtp_rtcp/include/report_block_data.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "nsPIDOMWindow.h"
#include "nsString.h"
#include "MainThreadUtils.h"
#include "nsCOMPtr.h"
#include "nsContentUtils.h"
#include "nsCycleCollectionParticipant.h"
#include "nsDebug.h"
#include "nsISupports.h"
#include "nsLiteralString.h"
#include "nsStringFwd.h"
#include "nsTArray.h"
#include "nsThreadUtils.h"
#include "nsWrapperCache.h"
#include "mozilla/AbstractThread.h"
#include "mozilla/AlreadyAddRefed.h"
#include "mozilla/Assertions.h"
#include "mozilla/Attributes.h"
#include "mozilla/fallible.h"
#include "mozilla/Logging.h"
#include "mozilla/mozalloc_oom.h"
#include "mozilla/MozPromise.h"
#include "mozilla/OwningNonNull.h"
#include "mozilla/RefPtr.h"
#include "mozilla/StateWatching.h"
#include "mozilla/UniquePtr.h"
#include "mozilla/Unused.h"
#include "mozilla/StateMirroring.h"
#include "mozilla/Maybe.h"
#include "mozilla/dom/MediaStreamTrack.h"
#include "mozilla/dom/Promise.h"
#include "mozilla/dom/RTCRtpScriptTransform.h"
#include "mozilla/dom/VideoStreamTrack.h"
#include "jsep/JsepTransceiver.h"
#include "mozilla/dom/RTCRtpSenderBinding.h"
#include "mozilla/dom/MediaStreamTrackBinding.h"
#include "mozilla/dom/Nullable.h"
#include "mozilla/dom/RTCRtpParametersBinding.h"
#include "mozilla/dom/RTCStatsReportBinding.h"
#include "mozilla/glean/GleanMetrics.h"
#include "js/RootingAPI.h"
#include "jsep/JsepTransceiver.h"
#include "RTCStatsReport.h"
#include "mozilla/Preferences.h"
#include "RTCRtpTransceiver.h"
#include "PeerConnectionImpl.h"
#include "libwebrtcglue/AudioConduit.h"
#include <vector>
#include "call/call.h"
#include "libwebrtcglue/CodecConfig.h"
#include "libwebrtcglue/MediaConduitControl.h"
#include "libwebrtcglue/MediaConduitInterface.h"
#include "sdp/SdpAttribute.h"
#include "sdp/SdpEnum.h"
namespace mozilla::dom {
@ -31,7 +86,7 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(RTCRtpSender)
NS_IMPL_CYCLE_COLLECTION_UNLINK_END
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(RTCRtpSender)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mWindow, mPc, mSenderTrack, mTransceiver,
mStreams, mDtmf)
mStreams, mTransform, mDtmf)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
NS_IMPL_CYCLE_COLLECTING_ADDREF(RTCRtpSender)
@ -66,7 +121,8 @@ RTCRtpSender::RTCRtpSender(nsPIDOMWindowInner* aWindow, PeerConnectionImpl* aPc,
INIT_CANONICAL(mVideoRtpRtcpConfig, Nothing()),
INIT_CANONICAL(mVideoCodecMode, webrtc::VideoCodecMode::kRealtimeVideo),
INIT_CANONICAL(mCname, std::string()),
INIT_CANONICAL(mTransmitting, false) {
INIT_CANONICAL(mTransmitting, false),
INIT_CANONICAL(mFrameTransformerProxy, nullptr) {
mPipeline = MediaPipelineTransmit::Create(
mPc->GetHandle(), aTransportHandler, aCallThread, aStsThread,
aConduit->type() == MediaSessionConduit::VIDEO, aConduit);
@ -1260,6 +1316,9 @@ void RTCRtpSender::Shutdown() {
mWatchManager.Shutdown();
mPipeline->Shutdown();
mPipeline = nullptr;
if (mTransform) {
mTransform->GetProxy().SetSender(nullptr);
}
}
void RTCRtpSender::BreakCycles() {
@ -1671,6 +1730,50 @@ void RTCRtpSender::UpdateDtmfSender() {
mDtmf->StopPlayout();
}
void RTCRtpSender::SetTransform(RTCRtpScriptTransform* aTransform,
ErrorResult& aError) {
if (aTransform == mTransform.get()) {
// Ok... smile and nod
// TODO: Depending on spec, this might throw
// https://github.com/w3c/webrtc-encoded-transform/issues/189
return;
}
if (aTransform && aTransform->IsClaimed()) {
aError.ThrowInvalidStateError("transform has already been used elsewhere");
return;
}
// Seamless switch for frames
if (aTransform) {
mFrameTransformerProxy = &aTransform->GetProxy();
} else {
mFrameTransformerProxy = nullptr;
}
if (mTransform) {
mTransform->GetProxy().SetSender(nullptr);
}
mTransform = const_cast<RTCRtpScriptTransform*>(aTransform);
if (mTransform) {
mTransform->GetProxy().SetSender(this);
mTransform->SetClaimed();
}
}
bool RTCRtpSender::GenerateKeyFrame(const Maybe<std::string>& aRid) {
if (!mTransform || !mPipeline || !mSenderTrack) {
return false;
}
mPipeline->mConduit->AsVideoSessionConduit().apply([&](const auto& conduit) {
conduit->GenerateKeyFrame(aRid, &mTransform->GetProxy());
});
return true;
}
} // namespace mozilla::dom
#undef LOGTAG

12
dom/media/webrtc/jsapi/RTCRtpSender.h
View File

@ -36,6 +36,7 @@ class RTCDtlsTransport;
class RTCDTMFSender;
struct RTCRtpCapabilities;
class RTCRtpTransceiver;
class RTCRtpScriptTransform;
class RTCRtpSender : public nsISupports,
public nsWrapperCache,
@ -75,6 +76,11 @@ class RTCRtpSender : public nsISupports,
Sequence<RTCRtpEncodingParameters>& aEncodings, bool aVideo,
ErrorResult& aRv);
RTCRtpScriptTransform* GetTransform() const { return mTransform; }
void SetTransform(RTCRtpScriptTransform* aTransform, ErrorResult& aError);
bool GenerateKeyFrame(const Maybe<std::string>& aRid);
nsPIDOMWindowInner* GetParentObject() const;
nsTArray<RefPtr<RTCStatsPromise>> GetStatsInternal(
bool aSkipIceStats = false);
@ -126,6 +132,10 @@ class RTCRtpSender : public nsISupports,
Canonical<std::string>& CanonicalCname() { return mCname; }
Canonical<bool>& CanonicalTransmitting() override { return mTransmitting; }
Canonical<RefPtr<FrameTransformerProxy>>& CanonicalFrameTransformerProxy() {
return mFrameTransformerProxy;
}
bool HasPendingSetParameters() const { return mPendingParameters.isSome(); }
void InvalidateLastReturnedParameters() {
mLastReturnedParameters = Nothing();
@ -171,6 +181,7 @@ class RTCRtpSender : public nsISupports,
RefPtr<MediaTransportHandler> mTransportHandler;
RefPtr<RTCRtpTransceiver> mTransceiver;
nsTArray<RefPtr<DOMMediaStream>> mStreams;
RefPtr<RTCRtpScriptTransform> mTransform;
bool mHaveSetupTransport = false;
// TODO(bug 1803388): Remove this stuff once it is no longer needed.
bool mAllowOldSetParameters = false;
@ -251,6 +262,7 @@ class RTCRtpSender : public nsISupports,
Canonical<webrtc::VideoCodecMode> mVideoCodecMode;
Canonical<std::string> mCname;
Canonical<bool> mTransmitting;
Canonical<RefPtr<FrameTransformerProxy>> mFrameTransformerProxy;
};
} // namespace dom

94
dom/media/webrtc/jsapi/RTCRtpTransceiver.cpp
View File

@ -3,37 +3,84 @@
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "jsapi/RTCRtpTransceiver.h"
#include "mozilla/UniquePtr.h"
#include <stdint.h>
#include <algorithm>
#include <string>
#include <vector>
#include "libwebrtcglue/AudioConduit.h"
#include "libwebrtcglue/VideoConduit.h"
#include "MediaTrackGraph.h"
#include "transportbridge/MediaPipeline.h"
#include "transportbridge/MediaPipelineFilter.h"
#include "jsep/JsepTrack.h"
#include "sdp/SdpHelper.h"
#include "MediaTrackGraphImpl.h"
#include "transport/logging.h"
#include "MediaEngine.h"
#include "nsIPrincipal.h"
#include "MediaSegment.h"
#include "RemoteTrackSource.h"
#include "libwebrtcglue/RtpRtcpConfig.h"
#include "MediaTransportHandler.h"
#include <utility>
#include <set>
#include <string>
#include <tuple>
#include "api/video_codecs/video_codec.h"
#include "nsCOMPtr.h"
#include "nsContentUtils.h"
#include "nsCycleCollectionParticipant.h"
#include "nsDebug.h"
#include "nsISerialEventTarget.h"
#include "nsISupports.h"
#include "nsProxyRelease.h"
#include "nsStringFwd.h"
#include "nsString.h"
#include "nsTArray.h"
#include "nsThreadUtils.h"
#include "nsWrapperCache.h"
#include "PrincipalHandle.h"
#include "ErrorList.h"
#include "MainThreadUtils.h"
#include "MediaEventSource.h"
#include "mozilla/AbstractThread.h"
#include "mozilla/Assertions.h"
#include "mozilla/ErrorResult.h"
#include "mozilla/fallible.h"
#include "mozilla/Maybe.h"
#include "mozilla/mozalloc_oom.h"
#include "mozilla/MozPromise.h"
#include "mozilla/Preferences.h"
#include "mozilla/UniquePtr.h"
#include "mozilla/StateMirroring.h"
#include "mozilla/RefPtr.h"
#include "mozilla/dom/Nullable.h"
#include "mozilla/dom/RTCStatsReportBinding.h"
#include "mozilla/dom/RTCRtpReceiverBinding.h"
#include "mozilla/dom/RTCRtpSenderBinding.h"
#include "mozilla/dom/RTCRtpTransceiverBinding.h"
#include "mozilla/dom/Promise.h"
#include "utils/PerformanceRecorder.h"
#include "systemservices/MediaUtils.h"
#include "MediaTrackGraph.h"
#include "js/RootingAPI.h"
#include "libwebrtcglue/AudioConduit.h"
#include "libwebrtcglue/VideoConduit.h"
#include "transportbridge/MediaPipeline.h"
#include "jsep/JsepTrack.h"
#include "sdp/SdpHelper.h"
#include "transport/logging.h"
#include "RemoteTrackSource.h"
#include "libwebrtcglue/RtpRtcpConfig.h"
#include "MediaTransportHandler.h"
#include "RTCDtlsTransport.h"
#include "RTCRtpReceiver.h"
#include "RTCRtpSender.h"
#include "RTCDTMFSender.h"
#include "systemservices/MediaUtils.h"
#include "PeerConnectionImpl.h"
#include "RTCStatsIdGenerator.h"
#include "libwebrtcglue/WebrtcCallWrapper.h"
#include "libwebrtcglue/WebrtcGmpVideoCodec.h"
#include "utils/PerformanceRecorder.h"
#include "libwebrtcglue/FrameTransformerProxy.h"
#include "jsep/JsepCodecDescription.h"
#include "jsep/JsepSession.h"
#include "jsep/JsepTrackEncoding.h"
#include "libwebrtcglue/CodecConfig.h"
#include "libwebrtcglue/MediaConduitControl.h"
#include "libwebrtcglue/MediaConduitInterface.h"
#include "RTCStatsReport.h"
#include "sdp/SdpAttribute.h"
#include "sdp/SdpEnum.h"
#include "sdp/SdpMediaSection.h"
#include "transport/transportlayer.h"
namespace mozilla {
@ -119,6 +166,15 @@ struct ConduitControlState : public AudioConduitControlInterface,
Canonical<webrtc::VideoCodecMode>& CanonicalVideoCodecMode() override {
return mSender->CanonicalVideoCodecMode();
}
Canonical<RefPtr<FrameTransformerProxy>>& CanonicalFrameTransformerProxySend()
override {
return mSender->CanonicalFrameTransformerProxy();
}
Canonical<RefPtr<FrameTransformerProxy>>& CanonicalFrameTransformerProxyRecv()
override {
return mReceiver->CanonicalFrameTransformerProxy();
}
};
} // namespace

79
dom/media/webrtc/jsapi/RTCTransformEventRunnable.cpp Normal file
View File

@ -0,0 +1,79 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "RTCTransformEventRunnable.h"
#include "nsIGlobalObject.h"
#include "ErrorList.h"
#include "nsError.h"
#include "nsDebug.h"
#include "nsLiteralString.h"
#include "mozilla/RefPtr.h"
#include "mozilla/AlreadyAddRefed.h"
// This needs to come before RTCTransformEvent.h, since webidl codegen doesn't
// include-what-you-use or forward declare.
#include "mozilla/dom/RTCRtpScriptTransformer.h"
#include "mozilla/dom/RTCTransformEvent.h"
#include "mozilla/dom/RTCTransformEventBinding.h"
#include "mozilla/dom/EventWithOptionsRunnable.h"
#include "mozilla/dom/Event.h"
#include "mozilla/dom/EventTarget.h"
#include "mozilla/dom/RootedDictionary.h"
#include "js/RootingAPI.h"
#include "js/Value.h"
#include "libwebrtcglue/FrameTransformerProxy.h"
namespace mozilla::dom {
RTCTransformEventRunnable::RTCTransformEventRunnable(
Worker& aWorker, FrameTransformerProxy* aProxy)
: EventWithOptionsRunnable(aWorker), mProxy(aProxy) {}
RTCTransformEventRunnable::~RTCTransformEventRunnable() = default;
already_AddRefed<Event> RTCTransformEventRunnable::BuildEvent(
JSContext* aCx, nsIGlobalObject* aGlobal, EventTarget* aTarget,
JS::Handle<JS::Value> aTransformerOptions) {
// Let transformerOptions be the result of
// StructuredDeserialize(serializedOptions, the current Realm).
// NOTE: We do not do this streams stuff. Spec will likely change here.
// The gist here is that we hook [[readable]] and [[writable]] up to the frame
// source/sink, which in out case is FrameTransformerProxy.
// Let readable be the result of StructuredDeserialize(serializedReadable, the
// current Realm). Let writable be the result of
// StructuredDeserialize(serializedWritable, the current Realm).
// Let transformer be a new RTCRtpScriptTransformer.
// Set transformer.[[options]] to transformerOptions.
// Set transformer.[[readable]] to readable.
// Set transformer.[[writable]] to writable.
RefPtr<RTCRtpScriptTransformer> transformer =
new RTCRtpScriptTransformer(aGlobal);
nsresult nrv =
transformer->Init(aCx, aTransformerOptions, mWorkerPrivate, mProxy);
if (NS_WARN_IF(NS_FAILED(nrv))) {
// TODO: Error handling. Currently unspecified.
return nullptr;
}
// Fire an event named rtctransform using RTCTransformEvent with transformer
// set to transformer on worker’s global scope.
RootedDictionary<RTCTransformEventInit> init(aCx);
init.mBubbles = false;
init.mCancelable = false;
init.mTransformer = transformer;
RefPtr<RTCTransformEvent> event =
RTCTransformEvent::Constructor(aTarget, u"rtctransform"_ns, init);
event->SetTrusted(true);
return event.forget();
}
} // namespace mozilla::dom

38
dom/media/webrtc/jsapi/RTCTransformEventRunnable.h Normal file
View File

@ -0,0 +1,38 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCTRANSFORMEVENTRUNNABLE_H_
#define MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCTRANSFORMEVENTRUNNABLE_H_
#include "mozilla/dom/EventWithOptionsRunnable.h"
namespace mozilla {
class FrameTransformerProxy;
namespace dom {
// Cargo-culted from MesssageEventRunnable.
// TODO: Maybe could subclass WorkerRunnable instead? Comments on
// WorkerDebuggeeRunnable indicate that firing an event at JS means we need that
// class.
class RTCTransformEventRunnable final : public EventWithOptionsRunnable {
public:
RTCTransformEventRunnable(Worker& aWorker, FrameTransformerProxy* aProxy);
already_AddRefed<Event> BuildEvent(
JSContext* aCx, nsIGlobalObject* aGlobal, EventTarget* aTarget,
JS::Handle<JS::Value> aTransformerOptions) override;
private:
virtual ~RTCTransformEventRunnable();
RefPtr<FrameTransformerProxy> mProxy;
};
} // namespace dom
} // namespace mozilla
#endif // MOZILLA_DOM_MEDIA_WEBRTC_JSAPI_RTCTRANSFORMEVENTRUNNABLE_H_

11
dom/media/webrtc/jsapi/moz.build
View File

@ -28,12 +28,18 @@ UNIFIED_SOURCES += [
"RemoteTrackSource.cpp",
"RTCDtlsTransport.cpp",
"RTCDTMFSender.cpp",
"RTCEncodedAudioFrame.cpp",
"RTCEncodedFrameBase.cpp",
"RTCEncodedVideoFrame.cpp",
"RTCRtpReceiver.cpp",
"RTCRtpScriptTransform.cpp",
"RTCRtpScriptTransformer.cpp",
"RTCRtpSender.cpp",
"RTCRtpTransceiver.cpp",
"RTCSctpTransport.cpp",
"RTCStatsIdGenerator.cpp",
"RTCStatsReport.cpp",
"RTCTransformEventRunnable.cpp",
"WebrtcGlobalInformation.cpp",
"WebrtcGlobalStatsHistory.cpp",
]
@ -41,7 +47,12 @@ UNIFIED_SOURCES += [
EXPORTS.mozilla.dom += [
"RTCDtlsTransport.h",
"RTCDTMFSender.h",
"RTCEncodedAudioFrame.h",
"RTCEncodedFrameBase.h",
"RTCEncodedVideoFrame.h",
"RTCRtpReceiver.h",
"RTCRtpScriptTransform.h",
"RTCRtpScriptTransformer.h",
"RTCRtpSender.h",
"RTCRtpTransceiver.h",
"RTCSctpTransport.h",

123
dom/media/webrtc/libwebrtcglue/AudioConduit.cpp
View File

@ -6,16 +6,57 @@
#include "common/browser_logging/CSFLog.h"
#include "MediaConduitControl.h"
#include "mozilla/media/MediaUtils.h"
#include "mozilla/Telemetry.h"
#include "transport/runnable_utils.h"
#include "transport/SrtpFlow.h" // For SRTP_MAX_EXPANSION
#include "WebrtcCallWrapper.h"
#include "libwebrtcglue/FrameTransformer.h"
#include <vector>
#include "CodecConfig.h"
#include "mozilla/StateMirroring.h"
#include <vector>
#include "mozilla/MozPromise.h"
#include "mozilla/RefPtr.h"
#include "mozilla/RWLock.h"
// libwebrtc includes
#include "api/audio_codecs/builtin_audio_encoder_factory.h"
#include "audio/audio_receive_stream.h"
#include "media/base/media_constants.h"
#include "rtc_base/ref_counted_object.h"
#include "api/audio/audio_frame.h"
#include "api/audio/audio_mixer.h"
#include "api/audio_codecs/audio_format.h"
#include "api/call/transport.h"
#include "api/media_types.h"
#include "api/rtp_headers.h"
#include "api/rtp_parameters.h"
#include "api/transport/rtp/rtp_source.h"
#include <utility>
#include "call/audio_receive_stream.h"
#include "call/audio_send_stream.h"
#include "call/call_basic_stats.h"
#include "domstubs.h"
#include "jsapi/RTCStatsReport.h"
#include <limits>
#include "MainThreadUtils.h"
#include <map>
#include "MediaConduitErrors.h"
#include "MediaConduitInterface.h"
#include <memory>
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "mozilla/Assertions.h"
#include "mozilla/Atomics.h"
#include "mozilla/Maybe.h"
#include "mozilla/StateWatching.h"
#include "nsCOMPtr.h"
#include "nsError.h"
#include "nsISerialEventTarget.h"
#include "nsThreadUtils.h"
#include "rtc_base/copy_on_write_buffer.h"
#include "rtc_base/network/sent_packet.h"
#include <stdint.h>
#include <string>
#include "transport/mediapacket.h"
// for ntohs
#ifdef HAVE_NETINET_IN_H
@ -71,7 +112,9 @@ WebrtcAudioConduit::Control::Control(const RefPtr<AbstractThread>& aCallThread)
INIT_MIRROR(mLocalRecvRtpExtensions, RtpExtList()),
INIT_MIRROR(mLocalSendRtpExtensions, RtpExtList()),
INIT_MIRROR(mSendCodec, Nothing()),
INIT_MIRROR(mRecvCodecs, std::vector<AudioCodecConfig>()) {}
INIT_MIRROR(mRecvCodecs, std::vector<AudioCodecConfig>()),
INIT_MIRROR(mFrameTransformerProxySend, nullptr),
INIT_MIRROR(mFrameTransformerProxyRecv, nullptr) {}
#undef INIT_MIRROR
RefPtr<GenericPromise> WebrtcAudioConduit::Shutdown() {
@ -81,28 +124,30 @@ RefPtr<GenericPromise> WebrtcAudioConduit::Shutdown() {
return InvokeAsync(mCallThread, "WebrtcAudioConduit::Shutdown (main thread)",
[this, self = RefPtr<WebrtcAudioConduit>(this)] {
mControl.mReceiving.DisconnectIfConnected();
mControl.mTransmitting.DisconnectIfConnected();
mControl.mLocalSsrcs.DisconnectIfConnected();
mControl.mLocalCname.DisconnectIfConnected();
mControl.mMid.DisconnectIfConnected();
mControl.mRemoteSsrc.DisconnectIfConnected();
mControl.mSyncGroup.DisconnectIfConnected();
mControl.mLocalRecvRtpExtensions.DisconnectIfConnected();
mControl.mLocalSendRtpExtensions.DisconnectIfConnected();
mControl.mSendCodec.DisconnectIfConnected();
mControl.mRecvCodecs.DisconnectIfConnected();
mWatchManager.Shutdown();
mControl.mReceiving.DisconnectIfConnected();
mControl.mTransmitting.DisconnectIfConnected();
mControl.mLocalSsrcs.DisconnectIfConnected();
mControl.mLocalCname.DisconnectIfConnected();
mControl.mMid.DisconnectIfConnected();
mControl.mRemoteSsrc.DisconnectIfConnected();
mControl.mSyncGroup.DisconnectIfConnected();
mControl.mLocalRecvRtpExtensions.DisconnectIfConnected();
mControl.mLocalSendRtpExtensions.DisconnectIfConnected();
mControl.mSendCodec.DisconnectIfConnected();
mControl.mRecvCodecs.DisconnectIfConnected();
mControl.mFrameTransformerProxySend.DisconnectIfConnected();
mControl.mFrameTransformerProxyRecv.DisconnectIfConnected();
mWatchManager.Shutdown();
{
AutoWriteLock lock(mLock);
DeleteSendStream();
DeleteRecvStream();
}
{
AutoWriteLock lock(mLock);
DeleteSendStream();
DeleteRecvStream();
}
return GenericPromise::CreateAndResolve(
true, "WebrtcAudioConduit::Shutdown (call thread)");
});
return GenericPromise::CreateAndResolve(
true, "WebrtcAudioConduit::Shutdown (call thread)");
});
}
WebrtcAudioConduit::WebrtcAudioConduit(
@ -163,6 +208,10 @@ void WebrtcAudioConduit::InitControl(AudioConduitControlInterface* aControl) {
mControl.mLocalSendRtpExtensions);
CONNECT(aControl->CanonicalAudioSendCodec(), mControl.mSendCodec);
CONNECT(aControl->CanonicalAudioRecvCodecs(), mControl.mRecvCodecs);
CONNECT(aControl->CanonicalFrameTransformerProxySend(),
mControl.mFrameTransformerProxySend);
CONNECT(aControl->CanonicalFrameTransformerProxyRecv(),
mControl.mFrameTransformerProxyRecv);
mControl.mOnDtmfEventListener = aControl->OnDtmfEvent().Connect(
mCall->mCallThread, this, &WebrtcAudioConduit::OnDtmfEvent);
}
@ -288,6 +337,32 @@ void WebrtcAudioConduit::OnControlConfigChange() {
recvStreamReconfigureNeeded = true;
}
if (mControl.mConfiguredFrameTransformerProxySend.get() !=
mControl.mFrameTransformerProxySend.Ref().get()) {
mControl.mConfiguredFrameTransformerProxySend =
mControl.mFrameTransformerProxySend.Ref();
if (!mSendStreamConfig.frame_transformer) {
mSendStreamConfig.frame_transformer =
new rtc::RefCountedObject<FrameTransformer>(false);
sendStreamRecreationNeeded = true;
}
static_cast<FrameTransformer*>(mSendStreamConfig.frame_transformer.get())
->SetProxy(mControl.mConfiguredFrameTransformerProxySend);
}
if (mControl.mConfiguredFrameTransformerProxyRecv.get() !=
mControl.mFrameTransformerProxyRecv.Ref().get()) {
mControl.mConfiguredFrameTransformerProxyRecv =
mControl.mFrameTransformerProxyRecv.Ref();
if (!mRecvStreamConfig.frame_transformer) {
mRecvStreamConfig.frame_transformer =
new rtc::RefCountedObject<FrameTransformer>(false);
recvStreamRecreationNeeded = true;
}
static_cast<FrameTransformer*>(mRecvStreamConfig.frame_transformer.get())
->SetProxy(mControl.mConfiguredFrameTransformerProxyRecv);
}
if (!recvStreamReconfigureNeeded && !sendStreamReconfigureNeeded &&
!recvStreamRecreationNeeded && !sendStreamRecreationNeeded &&
mControl.mReceiving == mRecvStreamRunning &&

6
dom/media/webrtc/libwebrtcglue/AudioConduit.h
View File

@ -256,6 +256,8 @@ class WebrtcAudioConduit : public AudioSessionConduit,
Mirror<RtpExtList> mLocalSendRtpExtensions;
Mirror<Maybe<AudioCodecConfig>> mSendCodec;
Mirror<std::vector<AudioCodecConfig>> mRecvCodecs;
Mirror<RefPtr<FrameTransformerProxy>> mFrameTransformerProxySend;
Mirror<RefPtr<FrameTransformerProxy>> mFrameTransformerProxyRecv;
MediaEventListener mOnDtmfEventListener;
// For caching mRemoteSsrc, since another caller may change the remote ssrc
@ -266,6 +268,10 @@ class WebrtcAudioConduit : public AudioSessionConduit,
// For tracking changes to mRecvCodecs.
std::vector<AudioCodecConfig> mConfiguredRecvCodecs;
// For change tracking. Callthread only.
RefPtr<FrameTransformerProxy> mConfiguredFrameTransformerProxySend;
RefPtr<FrameTransformerProxy> mConfiguredFrameTransformerProxyRecv;
Control() = delete;
explicit Control(const RefPtr<AbstractThread>& aCallThread);
} mControl;

87
dom/media/webrtc/libwebrtcglue/FrameTransformer.cpp Normal file
View File

@ -0,0 +1,87 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "libwebrtcglue/FrameTransformer.h"
#include "api/frame_transformer_interface.h"
#include "mozilla/Mutex.h"
#include <memory>
#include <utility>
#include "api/scoped_refptr.h"
#include <stdint.h>
#include "libwebrtcglue/FrameTransformerProxy.h"
namespace mozilla {
FrameTransformer::FrameTransformer(bool aVideo)
: webrtc::FrameTransformerInterface(),
mVideo(aVideo),
mCallbacksMutex("FrameTransformer::mCallbacksMutex"),
mProxyMutex("FrameTransformer::mProxyMutex") {}
FrameTransformer::~FrameTransformer() {
if (mProxy) {
mProxy->SetLibwebrtcTransformer(nullptr);
}
}
void FrameTransformer::Transform(
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame) {
MutexAutoLock lock(mProxyMutex);
if (mProxy) {
mProxy->Transform(std::move(aFrame));
return;
}
// No transformer, just passthrough
OnTransformedFrame(std::move(aFrame));
}
void FrameTransformer::RegisterTransformedFrameCallback(
rtc::scoped_refptr<webrtc::TransformedFrameCallback> aCallback) {
MutexAutoLock lock(mCallbacksMutex);
mCallback = aCallback;
}
void FrameTransformer::UnregisterTransformedFrameCallback() {
MutexAutoLock lock(mCallbacksMutex);
mCallback = nullptr;
}
void FrameTransformer::RegisterTransformedFrameSinkCallback(
rtc::scoped_refptr<webrtc::TransformedFrameCallback> aCallback,
uint32_t aSsrc) {
MutexAutoLock lock(mCallbacksMutex);
mCallbacksBySsrc[aSsrc] = aCallback;
}
void FrameTransformer::UnregisterTransformedFrameSinkCallback(uint32_t aSsrc) {
MutexAutoLock lock(mCallbacksMutex);
mCallbacksBySsrc.erase(aSsrc);
}
void FrameTransformer::OnTransformedFrame(
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame) {
MutexAutoLock lock(mCallbacksMutex);
if (mCallback) {
mCallback->OnTransformedFrame(std::move(aFrame));
} else if (auto it = mCallbacksBySsrc.find(aFrame->GetSsrc());
it != mCallbacksBySsrc.end()) {
it->second->OnTransformedFrame(std::move(aFrame));
}
}
void FrameTransformer::SetProxy(FrameTransformerProxy* aProxy) {
MutexAutoLock lock(mProxyMutex);
if (mProxy) {
mProxy->SetLibwebrtcTransformer(nullptr);
}
mProxy = aProxy;
if (mProxy) {
mProxy->SetLibwebrtcTransformer(this);
}
}
} // namespace mozilla

79
dom/media/webrtc/libwebrtcglue/FrameTransformer.h Normal file
View File

@ -0,0 +1,79 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef MOZILLA_DOM_MEDIA_WEBRTC_LIBWEBRTCGLUE_FRAMETRANSFORMER_H_
#define MOZILLA_DOM_MEDIA_WEBRTC_LIBWEBRTCGLUE_FRAMETRANSFORMER_H_
#include "api/frame_transformer_interface.h"
#include "libwebrtcglue/FrameTransformerProxy.h"
#include "nsISupportsImpl.h"
#include "mozilla/Mutex.h"
#include "jsapi/RTCRtpScriptTransformer.h"
namespace mozilla {
// There is one of these per RTCRtpSender and RTCRtpReceiver, for its entire
// lifetime. SetProxy is used to activate/deactivate it. In the inactive state
// (the default), this is just a synchronous passthrough.
class FrameTransformer : public webrtc::FrameTransformerInterface {
public:
explicit FrameTransformer(bool aVideo);
virtual ~FrameTransformer();
// This is set when RTCRtpSender/Receiver.transform is set, and unset when
// RTCRtpSender/Receiver.transform is unset.
void SetProxy(FrameTransformerProxy* aProxy);
// If no proxy is set (ie; RTCRtpSender/Receiver.transform is not set), this
// synchronously calls OnTransformedFrame with no modifcation. If a proxy is
// set, we send the frame to it, and eventually that frame should come back
// to OnTransformedFrame.
void Transform(
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame) override;
void OnTransformedFrame(
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame);
// When libwebrtc uses the same callback for all ssrcs
// (right now, this is used for audio, but we do not care in this class)
void RegisterTransformedFrameCallback(
rtc::scoped_refptr<webrtc::TransformedFrameCallback> aCallback) override;
void UnregisterTransformedFrameCallback() override;
// When libwebrtc uses a different callback for each ssrc
// (right now, this is used for video, but we do not care in this class)
void RegisterTransformedFrameSinkCallback(
rtc::scoped_refptr<webrtc::TransformedFrameCallback> aCallback,
uint32_t aSsrc) override;
void UnregisterTransformedFrameSinkCallback(uint32_t aSsrc) override;
bool IsVideo() const { return mVideo; }
private:
const bool mVideo;
Mutex mCallbacksMutex;
// Written on a libwebrtc thread, read on the worker thread.
rtc::scoped_refptr<webrtc::TransformedFrameCallback> mCallback
MOZ_GUARDED_BY(mCallbacksMutex);
std::map<uint32_t, rtc::scoped_refptr<webrtc::TransformedFrameCallback>>
mCallbacksBySsrc MOZ_GUARDED_BY(mCallbacksMutex);
Mutex mProxyMutex;
// Written on the call thread, read on a libwebrtc/gmp/mediadataencoder/call
// thread (which one depends on the media type and direction). Right now,
// these are:
// Send video: VideoStreamEncoder::encoder_queue_,
// WebrtcMediaDataEncoder::mTaskQueue, or GMP encoder thread.
// Recv video: Call::worker_thread_
// Send audio: ChannelSend::encoder_queue_
// Recv audio: ChannelReceive::worker_thread_
// This should have little to no lock contention
// This corresponds to the RTCRtpScriptTransform/RTCRtpScriptTransformer.
RefPtr<FrameTransformerProxy> mProxy MOZ_GUARDED_BY(mProxyMutex);
}; // FrameTransformer
} // namespace mozilla
#endif // MOZILLA_DOM_MEDIA_WEBRTC_LIBWEBRTCGLUE_FRAMETRANSFORMER_H_

258
dom/media/webrtc/libwebrtcglue/FrameTransformerProxy.cpp Normal file
View File

@ -0,0 +1,258 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "libwebrtcglue/FrameTransformerProxy.h"
#include "libwebrtcglue/FrameTransformer.h"
#include "mozilla/dom/RTCRtpSender.h"
#include "mozilla/dom/RTCRtpReceiver.h"
#include "mozilla/Logging.h"
#include "mozilla/Mutex.h"
#include "jsapi/RTCRtpScriptTransformer.h"
#include "nsThreadUtils.h"
#include "mozilla/Assertions.h"
#include <utility>
#include "mozilla/Maybe.h"
#include "mozilla/RefPtr.h"
#include "nscore.h"
#include "ErrorList.h"
#include "nsIRunnable.h"
#include "nsIEventTarget.h"
#include "api/frame_transformer_interface.h"
#include <memory>
#include "nsDebug.h"
#include "nsISupports.h"
#include <string>
namespace mozilla {
LazyLogModule gFrameTransformerProxyLog("FrameTransformerProxy");
FrameTransformerProxy::FrameTransformerProxy()
: mMutex("FrameTransformerProxy::mMutex") {}
FrameTransformerProxy::~FrameTransformerProxy() = default;
void FrameTransformerProxy::SetScriptTransformer(
dom::RTCRtpScriptTransformer& aTransformer) {
MutexAutoLock lock(mMutex);
if (mReleaseScriptTransformerCalled) {
MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Warning,
("RTCRtpScriptTransformer is ready, but ReleaseScriptTransformer "
"has already been called."));
// The mainthread side has torn down while the worker init was pending.
// Don't grab a reference to the worker thread, or the script transformer.
// Also, let the script transformer know that we do not need it after all.
aTransformer.NotifyReleased();
return;
}
MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Info,
("RTCRtpScriptTransformer is ready!"));
mWorkerThread = GetCurrentSerialEventTarget();
MOZ_ASSERT(mWorkerThread);
MOZ_ASSERT(!mScriptTransformer);
mScriptTransformer = &aTransformer;
while (!mQueue.empty()) {
mScriptTransformer->TransformFrame(std::move(mQueue.front()));
mQueue.pop_front();
}
}
Maybe<bool> FrameTransformerProxy::IsVideo() const {
MutexAutoLock lock(mMutex);
return mVideo;
}
void FrameTransformerProxy::ReleaseScriptTransformer() {
MutexAutoLock lock(mMutex);
MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Debug, ("In %s", __FUNCTION__));
if (mReleaseScriptTransformerCalled) {
return;
}
mReleaseScriptTransformerCalled = true;
if (mWorkerThread) {
mWorkerThread->Dispatch(NS_NewRunnableFunction(
__func__, [this, self = RefPtr<FrameTransformerProxy>(this)] {
if (mScriptTransformer) {
mScriptTransformer->NotifyReleased();
mScriptTransformer = nullptr;
}
// Make sure cycles are broken; this unset might have been caused by
// something other than the sender/receiver being unset.
GetMainThreadSerialEventTarget()->Dispatch(
NS_NewRunnableFunction(__func__, [this, self] {
MutexAutoLock lock(mMutex);
mSender = nullptr;
mReceiver = nullptr;
}));
}));
mWorkerThread = nullptr;
}
}
void FrameTransformerProxy::SetLibwebrtcTransformer(
FrameTransformer* aLibwebrtcTransformer) {
MutexAutoLock lock(mMutex);
mLibwebrtcTransformer = aLibwebrtcTransformer;
if (mLibwebrtcTransformer) {
MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Info,
("mLibwebrtcTransformer is now set!"));
mVideo = Some(mLibwebrtcTransformer->IsVideo());
}
}
void FrameTransformerProxy::Transform(
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame) {
MutexAutoLock lock(mMutex);
MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Debug, ("In %s", __FUNCTION__));
if (!mWorkerThread && !mReleaseScriptTransformerCalled) {
MOZ_LOG(
gFrameTransformerProxyLog, LogLevel::Info,
("In %s, queueing frame because RTCRtpScriptTransformer is not ready",
__FUNCTION__));
// We are still waiting for the script transformer to be created on the
// worker thread.
mQueue.push_back(std::move(aFrame));
return;
}
if (mWorkerThread) {
MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Debug,
("Queueing call to RTCRtpScriptTransformer::TransformFrame"));
mWorkerThread->Dispatch(NS_NewRunnableFunction(
__func__, [this, self = RefPtr<FrameTransformerProxy>(this),
frame = std::move(aFrame)]() mutable {
if (NS_WARN_IF(!mScriptTransformer)) {
// Could happen due to errors. Is there some
// other processing we ought to do?
return;
}
mScriptTransformer->TransformFrame(std::move(frame));
}));
}
}
void FrameTransformerProxy::OnTransformedFrame(
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame) {
MutexAutoLock lock(mMutex);
// If the worker thread has changed, we drop the frame, to avoid frames
// arriving out of order.
if (mLibwebrtcTransformer) {
// This will lock, lock order is mMutex, FrameTransformer::mLibwebrtcMutex
mLibwebrtcTransformer->OnTransformedFrame(std::move(aFrame));
}
}
void FrameTransformerProxy::SetSender(dom::RTCRtpSender* aSender) {
{
MutexAutoLock lock(mMutex);
MOZ_ASSERT(!mReceiver);
mSender = aSender;
}
if (!aSender) {
MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Info, ("Sender set to null"));
ReleaseScriptTransformer();
}
}
void FrameTransformerProxy::SetReceiver(dom::RTCRtpReceiver* aReceiver) {
{
MutexAutoLock lock(mMutex);
MOZ_ASSERT(!mSender);
mReceiver = aReceiver;
}
if (!aReceiver) {
MOZ_LOG(gFrameTransformerProxyLog, LogLevel::Info,
("Receiver set to null"));
ReleaseScriptTransformer();
}
}
bool FrameTransformerProxy::RequestKeyFrame() {
{
// Spec wants this to reject synchronously if the RTCRtpScriptTransformer
// is not associated with a video receiver. This may change to an async
// check?
MutexAutoLock lock(mMutex);
if (!mReceiver || !mVideo.isSome() || !*mVideo) {
return false;
}
}
// Thread hop to main, and then the conduit thread-hops to the call thread.
GetMainThreadSerialEventTarget()->Dispatch(NS_NewRunnableFunction(
__func__, [this, self = RefPtr<FrameTransformerProxy>(this)] {
MutexAutoLock lock(mMutex);
if (mReceiver && mVideo.isSome() && *mVideo) {
mReceiver->RequestKeyFrame();
}
}));
return true;
}
void FrameTransformerProxy::KeyFrameRequestDone(bool aSuccess) {
MutexAutoLock lock(mMutex);
if (mWorkerThread) {
mWorkerThread->Dispatch(NS_NewRunnableFunction(
__func__, [this, self = RefPtr<FrameTransformerProxy>(this), aSuccess] {
if (mScriptTransformer) {
mScriptTransformer->KeyFrameRequestDone(aSuccess);
}
}));
}
}
bool FrameTransformerProxy::GenerateKeyFrame(const Maybe<std::string>& aRid) {
{
// Spec wants this to reject synchronously if the RTCRtpScriptTransformer
// is not associated with a video sender. This may change to an async
// check?
MutexAutoLock lock(mMutex);
if (!mSender || !mVideo.isSome() || !*mVideo) {
return false;
}
}
// Thread hop to main, and then the conduit thread-hops to the call thread.
GetMainThreadSerialEventTarget()->Dispatch(NS_NewRunnableFunction(
__func__, [this, self = RefPtr<FrameTransformerProxy>(this), aRid] {
MutexAutoLock lock(mMutex);
if (!mSender || !mVideo.isSome() || !*mVideo ||
!mSender->GenerateKeyFrame(aRid)) {
CopyableErrorResult rv;
rv.ThrowInvalidStateError("Not sending video");
if (mWorkerThread) {
mWorkerThread->Dispatch(NS_NewRunnableFunction(
__func__,
[this, self = RefPtr<FrameTransformerProxy>(this), aRid, rv] {
if (mScriptTransformer) {
mScriptTransformer->GenerateKeyFrameError(aRid, rv);
}
}));
}
}
}));
return true;
}
void FrameTransformerProxy::GenerateKeyFrameError(
const Maybe<std::string>& aRid, const CopyableErrorResult& aResult) {
MutexAutoLock lock(mMutex);
if (mWorkerThread) {
mWorkerThread->Dispatch(NS_NewRunnableFunction(
__func__,
[this, self = RefPtr<FrameTransformerProxy>(this), aRid, aResult] {
if (mScriptTransformer) {
mScriptTransformer->GenerateKeyFrameError(aRid, aResult);
}
}));
}
}
} // namespace mozilla

124
dom/media/webrtc/libwebrtcglue/FrameTransformerProxy.h Normal file
View File

@ -0,0 +1,124 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef MOZILLA_DOM_MEDIA_WEBRTC_LIBWEBRTCGLUE_FRAMETRANSFORMERPROXY_H_
#define MOZILLA_DOM_MEDIA_WEBRTC_LIBWEBRTCGLUE_FRAMETRANSFORMERPROXY_H_
#include "nsISupportsImpl.h"
#include "mozilla/Mutex.h"
#include "mozilla/Maybe.h"
#include <list>
#include <memory>
class nsIEventTarget;
namespace webrtc {
class TransformableFrameInterface;
class VideoReceiveStreamInterface;
} // namespace webrtc
namespace mozilla {
class FrameTransformer;
class WebrtcVideoConduit;
class CopyableErrorResult;
namespace dom {
class RTCRtpScriptTransformer;
class RTCRtpSender;
class RTCRtpReceiver;
} // namespace dom
// This corresponds to a single RTCRtpScriptTransform (and its
// RTCRtpScriptTransformer, once that is created on the worker thread). This
// is intended to decouple threading/lifecycle/include-dependencies between
// FrameTransformer (on the libwebrtc side of things), RTCRtpScriptTransformer
// (on the worker side of things), RTCRtpScriptTransform and
// RTCRtpSender/Receiver (on the main thread), and prevents frames from being
// lost while we're setting things up on the worker. In other words, this
// handles the inconvenient stuff.
class FrameTransformerProxy {
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(FrameTransformerProxy);
FrameTransformerProxy();
FrameTransformerProxy(const FrameTransformerProxy& aRhs) = delete;
FrameTransformerProxy(FrameTransformerProxy&& aRhs) = delete;
FrameTransformerProxy& operator=(const FrameTransformerProxy& aRhs) = delete;
FrameTransformerProxy& operator=(FrameTransformerProxy&& aRhs) = delete;
// Called at most once (might not be called if the worker is shutting down),
// on the worker thread.
void SetScriptTransformer(dom::RTCRtpScriptTransformer& aTransformer);
// Can be called from the worker thread (if the worker is shutting down), or
// main (if RTCRtpSender/RTCRtpReceiver is done with us).
void ReleaseScriptTransformer();
// RTCRtpScriptTransformer calls this when it is done transforming a frame.
void OnTransformedFrame(
std::unique_ptr<webrtc::TransformableFrameInterface> aFrame);
Maybe<bool> IsVideo() const;
// Called by FrameTransformer, on main. Only one FrameTransformer will ever
// be registered over the lifetime of this object. This is where we route
// transformed frames. If this is set, we can also expect to receive calls to
// Transform.
void SetLibwebrtcTransformer(FrameTransformer* aLibwebrtcTransformer);
// FrameTransformer calls this while we're registered with it (by
// SetLibwebrtcTransformer)
void Transform(std::unique_ptr<webrtc::TransformableFrameInterface> aFrame);
void SetSender(dom::RTCRtpSender* aSender);
void SetReceiver(dom::RTCRtpReceiver* aReceiver);
// Called on worker thread
bool RequestKeyFrame();
// Called on call thread
void KeyFrameRequestDone(bool aSuccess);
bool GenerateKeyFrame(const Maybe<std::string>& aRid);
void GenerateKeyFrameError(const Maybe<std::string>& aRid,
const CopyableErrorResult& aResult);
private:
virtual ~FrameTransformerProxy();
// Worker thread only. Set at most once.
// Does not need any mutex protection.
RefPtr<dom::RTCRtpScriptTransformer> mScriptTransformer;
mutable Mutex mMutex;
// Written on the worker thread. Read on libwebrtc threads, mainthread, and
// the worker thread.
RefPtr<nsIEventTarget> mWorkerThread MOZ_GUARDED_BY(mMutex);
// We need a flag for this in case the ReleaseScriptTransformer call comes
// _before_ the script transformer is set, to disable SetScriptTransformer.
// Could be written on main or the worker thread. Read on main, worker, and
// libwebrtc threads.
bool mReleaseScriptTransformerCalled MOZ_GUARDED_BY(mMutex) = false;
// Used when frames arrive before the script transformer is created, which
// should be pretty rare. Accessed on worker and libwebrtc threads.
std::list<std::unique_ptr<webrtc::TransformableFrameInterface>> mQueue
MOZ_GUARDED_BY(mMutex);
// Written on main, read on the worker thread.
FrameTransformer* mLibwebrtcTransformer MOZ_GUARDED_BY(mMutex) = nullptr;
// TODO: Will be used to route GenerateKeyFrame. Details TBD.
RefPtr<dom::RTCRtpSender> mSender MOZ_GUARDED_BY(mMutex);
// Set on mainthread. This is where we route RequestKeyFrame calls from the
// worker thread. Mutex protected because spec wants sync errors if the
// receiver is not set (or the right type). If spec drops this requirement,
// this could be mainthread only and non-mutex-protected.
RefPtr<dom::RTCRtpReceiver> mReceiver MOZ_GUARDED_BY(mMutex);
Maybe<bool> mVideo MOZ_GUARDED_BY(mMutex);
};
} // namespace mozilla
#endif // MOZILLA_DOM_MEDIA_WEBRTC_LIBWEBRTCGLUE_FRAMETRANSFORMERPROXY_H_

5
dom/media/webrtc/libwebrtcglue/MediaConduitControl.h
View File

@ -16,6 +16,7 @@
#include "CodecConfig.h" // For Audio/VideoCodecConfig
#include "api/rtp_parameters.h" // For webrtc::RtpExtension
#include "api/video_codecs/video_codec.h" // For webrtc::VideoCodecMode
#include "FrameTransformerProxy.h"
namespace mozilla {
@ -45,6 +46,10 @@ class MediaConduitControlInterface {
virtual Canonical<std::string>& CanonicalSyncGroup() = 0;
virtual Canonical<RtpExtList>& CanonicalLocalRecvRtpExtensions() = 0;
virtual Canonical<RtpExtList>& CanonicalLocalSendRtpExtensions() = 0;
virtual Canonical<RefPtr<FrameTransformerProxy>>&
CanonicalFrameTransformerProxySend() = 0;
virtual Canonical<RefPtr<FrameTransformerProxy>>&
CanonicalFrameTransformerProxyRecv() = 0;
};
class AudioConduitControlInterface : public MediaConduitControlInterface {

5
dom/media/webrtc/libwebrtcglue/MediaConduitInterface.h
View File

@ -55,6 +55,7 @@ enum class MediaSessionConduitLocalDirection : int { kSend, kRecv };
class VideoSessionConduit;
class AudioSessionConduit;
class WebrtcCallWrapper;
class FrameTransformerProxy;
/**
* 1. Abstract renderer for video data
@ -413,6 +414,10 @@ class VideoSessionConduit : public MediaSessionConduit {
};
virtual Maybe<Resolution> GetLastResolution() const = 0;
virtual void RequestKeyFrame(FrameTransformerProxy* aProxy) = 0;
virtual void GenerateKeyFrame(const Maybe<std::string>& aRid,
FrameTransformerProxy* aProxy) = 0;
protected:
/* RTCP feedback settings, for unit testing purposes */
FrameRequestType mFrameRequestMethod;

209
dom/media/webrtc/libwebrtcglue/VideoConduit.cpp
View File

@ -5,28 +5,28 @@
#include "VideoConduit.h"
#include <algorithm>
#include <cinttypes>
#include <cmath>
#include "common/browser_logging/CSFLog.h"
#include "common/YuvStamper.h"
#include "GmpVideoCodec.h"
#include "MediaConduitControl.h"
#include "MediaDataCodec.h"
#include "mozilla/dom/RTCRtpSourcesBinding.h"
#include "mozilla/media/MediaUtils.h"
#include "mozilla/StaticPrefs_media.h"
#include "mozilla/TemplateLib.h"
#include "nsIGfxInfo.h"
#include "nsIPrefBranch.h"
#include "nsIPrefService.h"
#include "nsServiceManagerUtils.h"
#include "RtpRtcpConfig.h"
#include "transport/SrtpFlow.h" // For SRTP_MAX_EXPANSION
#include "Tracing.h"
#include "VideoStreamFactory.h"
#include "WebrtcCallWrapper.h"
#include "WebrtcGmpVideoCodec.h"
#include "libwebrtcglue/FrameTransformer.h"
#include "libwebrtcglue/FrameTransformerProxy.h"
#include "mozilla/StateMirroring.h"
#include "mozilla/RefPtr.h"
#include "nsThreadUtils.h"
#include "mozilla/Maybe.h"
#include "mozilla/ErrorResult.h"
#include <string>
#include <utility>
#include <vector>
// libwebrtc includes
#include "api/transport/bitrate_settings.h"
@ -36,8 +36,70 @@
#include "media/base/media_constants.h"
#include "media/engine/simulcast_encoder_adapter.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "rtc_base/ref_counted_object.h"
#include "api/call/transport.h"
#include "api/media_types.h"
#include "api/rtp_headers.h"
#include "api/rtp_parameters.h"
#include "api/scoped_refptr.h"
#include "api/transport/rtp/rtp_source.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video/video_codec_constants.h"
#include "api/video/video_codec_type.h"
#include "api/video/video_frame_buffer.h"
#include "api/video/video_sink_interface.h"
#include "api/video/video_source_interface.h"
#include <utility>
#include "call/call.h"
#include "call/rtp_config.h"
#include "call/video_receive_stream.h"
#include "call/video_send_stream.h"
#include "CodecConfig.h"
#include "common_video/include/video_frame_buffer_pool.h"
#include "domstubs.h"
#include <iomanip>
#include <ios>
#include "jsapi/RTCStatsReport.h"
#include <limits>
#include "MainThreadUtils.h"
#include <map>
#include "MediaConduitErrors.h"
#include "MediaConduitInterface.h"
#include "MediaEventSource.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "mozilla/Assertions.h"
#include "mozilla/Atomics.h"
#include "mozilla/DataMutex.h"
#include "mozilla/dom/BindingDeclarations.h"
#include "mozilla/dom/RTCStatsReportBinding.h"
#include "mozilla/fallible.h"
#include "mozilla/mozalloc_oom.h"
#include "mozilla/MozPromise.h"
#include "mozilla/Mutex.h"
#include "mozilla/ProfilerState.h"
#include "mozilla/ReentrantMonitor.h"
#include "mozilla/ReverseIterator.h"
#include "mozilla/StateWatching.h"
#include "mozilla/Telemetry.h"
#include "mozilla/TelemetryHistogramEnums.h"
#include "mozilla/TelemetryScalarEnums.h"
#include "mozilla/Types.h"
#include "mozilla/UniquePtr.h"
#include "nsCOMPtr.h"
#include "nsDebug.h"
#include "nsError.h"
#include "nsIDirectTaskDispatcher.h"
#include "nsISerialEventTarget.h"
#include "nsStringFwd.h"
#include "PerformanceRecorder.h"
#include "rtc_base/copy_on_write_buffer.h"
#include "rtc_base/network/sent_packet.h"
#include <sstream>
#include <stdint.h>
#include "transport/mediapacket.h"
#include "video/config/video_encoder_config.h"
#include "WebrtcVideoCodecFactory.h"
#ifdef MOZ_WIDGET_ANDROID
# include "VideoEngine.h"
@ -329,7 +391,9 @@ WebrtcVideoConduit::Control::Control(const RefPtr<AbstractThread>& aCallThread)
INIT_MIRROR(mSendRtpRtcpConfig, Nothing()),
INIT_MIRROR(mRecvCodecs, std::vector<VideoCodecConfig>()),
INIT_MIRROR(mRecvRtpRtcpConfig, Nothing()),
INIT_MIRROR(mCodecMode, webrtc::VideoCodecMode::kRealtimeVideo) {}
INIT_MIRROR(mCodecMode, webrtc::VideoCodecMode::kRealtimeVideo),
INIT_MIRROR(mFrameTransformerProxySend, nullptr),
INIT_MIRROR(mFrameTransformerProxyRecv, nullptr) {}
#undef INIT_MIRROR
WebrtcVideoConduit::WebrtcVideoConduit(
@ -368,7 +432,6 @@ WebrtcVideoConduit::WebrtcVideoConduit(
WebrtcVideoConduit::~WebrtcVideoConduit() {
CSFLogDebug(LOGTAG, "%s ", __FUNCTION__);
MOZ_ASSERT(!mSendStream && !mRecvStream,
"Call DeleteStreams prior to ~WebrtcVideoConduit.");
}
@ -408,6 +471,10 @@ void WebrtcVideoConduit::InitControl(VideoConduitControlInterface* aControl) {
CONNECT(aControl->CanonicalVideoRecvRtpRtcpConfig(),
mControl.mRecvRtpRtcpConfig);
CONNECT(aControl->CanonicalVideoCodecMode(), mControl.mCodecMode);
CONNECT(aControl->CanonicalFrameTransformerProxySend(),
mControl.mFrameTransformerProxySend);
CONNECT(aControl->CanonicalFrameTransformerProxyRecv(),
mControl.mFrameTransformerProxyRecv);
}
#undef CONNECT
@ -765,6 +832,32 @@ void WebrtcVideoConduit::OnControlConfigChange() {
}
}
if (mControl.mConfiguredFrameTransformerProxySend.get() !=
mControl.mFrameTransformerProxySend.Ref().get()) {
mControl.mConfiguredFrameTransformerProxySend =
mControl.mFrameTransformerProxySend.Ref();
if (!mSendStreamConfig.frame_transformer) {
mSendStreamConfig.frame_transformer =
new rtc::RefCountedObject<FrameTransformer>(true);
sendStreamRecreationNeeded = true;
}
static_cast<FrameTransformer*>(mSendStreamConfig.frame_transformer.get())
->SetProxy(mControl.mConfiguredFrameTransformerProxySend);
}
if (mControl.mConfiguredFrameTransformerProxyRecv.get() !=
mControl.mFrameTransformerProxyRecv.Ref().get()) {
mControl.mConfiguredFrameTransformerProxyRecv =
mControl.mFrameTransformerProxyRecv.Ref();
if (!mRecvStreamConfig.frame_transformer) {
mRecvStreamConfig.frame_transformer =
new rtc::RefCountedObject<FrameTransformer>(true);
}
static_cast<FrameTransformer*>(mRecvStreamConfig.frame_transformer.get())
->SetProxy(mControl.mConfiguredFrameTransformerProxyRecv);
// No flag to set, we always recreate recv streams
}
if (remoteSsrcUpdateNeeded) {
SetRemoteSSRCConfig(mControl.mConfiguredRemoteSsrc,
mControl.mConfiguredRemoteRtxSsrc);
@ -1219,6 +1312,8 @@ RefPtr<GenericPromise> WebrtcVideoConduit::Shutdown() {
mControl.mRecvCodecs.DisconnectIfConnected();
mControl.mRecvRtpRtcpConfig.DisconnectIfConnected();
mControl.mCodecMode.DisconnectIfConnected();
mControl.mFrameTransformerProxySend.DisconnectIfConnected();
mControl.mFrameTransformerProxyRecv.DisconnectIfConnected();
mWatchManager.Shutdown();
mCall->UnregisterConduit(this);
@ -1860,6 +1955,92 @@ std::vector<webrtc::RtpSource> WebrtcVideoConduit::GetUpstreamRtpSources()
return sources;
}
void WebrtcVideoConduit::RequestKeyFrame(FrameTransformerProxy* aProxy) {
mCallThread->Dispatch(NS_NewRunnableFunction(
__func__, [this, self = RefPtr<WebrtcVideoConduit>(this),
proxy = RefPtr<FrameTransformerProxy>(aProxy)] {
bool success = false;
if (mRecvStream && mEngineReceiving) {
// This is a misnomer. This requests a keyframe from the other side.
mRecvStream->GenerateKeyFrame();
success = true;
}
proxy->KeyFrameRequestDone(success);
}));
}
void WebrtcVideoConduit::GenerateKeyFrame(const Maybe<std::string>& aRid,
FrameTransformerProxy* aProxy) {
// libwebrtc does not implement error handling in the way that
// webrtc-encoded-transform specifies. So, we'll need to do that here.
// Also, spec wants us to synchronously check whether there's an encoder, but
// that's not something that can be checked synchronously.
mCallThread->Dispatch(NS_NewRunnableFunction(
__func__, [this, self = RefPtr<WebrtcVideoConduit>(this),
proxy = RefPtr<FrameTransformerProxy>(aProxy), aRid] {
// If encoder is undefined, reject promise with InvalidStateError,
// abort these steps.
// If encoder is not processing video frames, reject promise with
// InvalidStateError, abort these steps.
if (!mSendStream || !mCurSendCodecConfig || !mEngineTransmitting) {
CopyableErrorResult result;
result.ThrowInvalidStateError("No encoders");
proxy->GenerateKeyFrameError(aRid, result);
return;
}
// Gather a list of video encoders, named videoEncoders from encoder,
// ordered according negotiated RIDs if any.
// NOTE: This is represented by mCurSendCodecConfig->mEncodings
// If rid is defined, remove from videoEncoders any video encoder that
// does not match rid.
// If rid is undefined, remove from videoEncoders all video encoders
// except the first one.
bool found = false;
std::vector<std::string> rids;
if (!aRid.isSome()) {
// If rid is undefined, set rid to the RID value corresponding to
// videoEncoder.
if (!mCurSendCodecConfig->mEncodings.empty()) {
if (!mCurSendCodecConfig->mEncodings[0].rid.empty()) {
rids.push_back(mCurSendCodecConfig->mEncodings[0].rid);
}
found = true;
}
} else {
for (const auto& encoding : mCurSendCodecConfig->mEncodings) {
if (encoding.rid == *aRid) {
found = true;
rids.push_back(encoding.rid);
break;
}
}
}
// If videoEncoders is empty, reject promise with NotFoundError and
// abort these steps. videoEncoders is expected to be empty if the
// corresponding RTCRtpSender is not active, or the corresponding
// RTCRtpSender track is ended.
if (!found) {
CopyableErrorResult result;
result.ThrowNotFoundError("Rid not in use");
proxy->GenerateKeyFrameError(aRid, result);
}
// NOTE: We don't do this stuff, because libwebrtc's interface is
// rid-based.
// Let videoEncoder be the first encoder in videoEncoders.
// If rid is undefined, set rid to the RID value corresponding to
// videoEncoder.
mSendStream->GenerateKeyFrame(rids);
}));
}
bool WebrtcVideoConduit::HasCodecPluginID(uint64_t aPluginID) const {
MOZ_ASSERT(NS_IsMainThread());

10
dom/media/webrtc/libwebrtcglue/VideoConduit.h
View File

@ -231,6 +231,10 @@ class WebrtcVideoConduit
std::vector<webrtc::RtpSource> GetUpstreamRtpSources() const override;
void RequestKeyFrame(FrameTransformerProxy* aProxy) override;
void GenerateKeyFrame(const Maybe<std::string>& aRid,
FrameTransformerProxy* aProxy) override;
private:
// Don't allow copying/assigning.
WebrtcVideoConduit(const WebrtcVideoConduit&) = delete;
@ -298,6 +302,8 @@ class WebrtcVideoConduit
Mirror<std::vector<VideoCodecConfig>> mRecvCodecs;
Mirror<Maybe<RtpRtcpConfig>> mRecvRtpRtcpConfig;
Mirror<webrtc::VideoCodecMode> mCodecMode;
Mirror<RefPtr<FrameTransformerProxy>> mFrameTransformerProxySend;
Mirror<RefPtr<FrameTransformerProxy>> mFrameTransformerProxyRecv;
// For caching mRemoteSsrc and mRemoteRtxSsrc, since another caller may
// change the remote ssrc in the stream config directly.
@ -310,6 +316,10 @@ class WebrtcVideoConduit
std::vector<VideoCodecConfig> mConfiguredRecvCodecs;
Maybe<RtpRtcpConfig> mConfiguredRecvRtpRtcpConfig;
// For change tracking. Callthread only.
RefPtr<FrameTransformerProxy> mConfiguredFrameTransformerProxySend;
RefPtr<FrameTransformerProxy> mConfiguredFrameTransformerProxyRecv;
Control() = delete;
explicit Control(const RefPtr<AbstractThread>& aCallThread);
} mControl;

2
dom/media/webrtc/libwebrtcglue/moz.build
View File

@ -19,6 +19,8 @@ LOCAL_INCLUDES += [
UNIFIED_SOURCES += [
"AudioConduit.cpp",
"FrameTransformer.cpp",
"FrameTransformerProxy.cpp",
"GmpVideoCodec.cpp",
"MediaConduitInterface.cpp",
"MediaDataCodec.cpp",

6
dom/tests/mochitest/general/test_interfaces.js
View File

@ -1064,6 +1064,10 @@ let interfaceNamesInGlobalScope = [
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCDTMFToneChangeEvent", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCEncodedAudioFrame", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCEncodedVideoFrame", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCIceCandidate", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCPeerConnection", insecureContext: true },
@ -1072,6 +1076,8 @@ let interfaceNamesInGlobalScope = [
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCRtpReceiver", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCRtpScriptTransform", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCRtpSender", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCRtpTransceiver", insecureContext: true },

6
dom/webidl/DedicatedWorkerGlobalScope.webidl
View File

@ -37,3 +37,9 @@ interface DedicatedWorkerGlobalScope : WorkerGlobalScope {
[Pref="dom.workers.requestAnimationFrame", Throws]
undefined cancelAnimationFrame(long handle);
};
// https://w3c.github.io/webrtc-encoded-transform/#RTCEncodedAudioFrame-methods
partial interface DedicatedWorkerGlobalScope {
[Pref="media.peerconnection.enabled",
Pref="media.peerconnection.scripttransform.enabled"] attribute EventHandler onrtctransform;
};

24
dom/webidl/RTCEncodedAudioFrame.webidl Normal file
View File

@ -0,0 +1,24 @@
/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/.
*
* The origin of this IDL file is
* https://w3c.github.io/webrtc-encoded-transform
*/
dictionary RTCEncodedAudioFrameMetadata {
unsigned long synchronizationSource;
octet payloadType;
sequence<unsigned long> contributingSources;
short sequenceNumber;
};
[Pref="media.peerconnection.enabled",
Pref="media.peerconnection.scripttransform.enabled",
Exposed=(Window,DedicatedWorker)]
interface RTCEncodedAudioFrame {
readonly attribute unsigned long timestamp;
attribute ArrayBuffer data;
RTCEncodedAudioFrameMetadata getMetadata();
};

41
dom/webidl/RTCEncodedVideoFrame.webidl Normal file
View File

@ -0,0 +1,41 @@
/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/.
*
* The origin of this IDL file is
* https://www.w3.org/TR/webrtc-encoded-transform
*/
// New enum for video frame types. Will eventually re-use the equivalent defined
// by WebCodecs.
enum RTCEncodedVideoFrameType {
"empty",
"key",
"delta",
};
dictionary RTCEncodedVideoFrameMetadata {
unsigned long long frameId;
sequence<unsigned long long> dependencies;
unsigned short width;
unsigned short height;
unsigned long spatialIndex;
unsigned long temporalIndex;
unsigned long synchronizationSource;
octet payloadType;
sequence<unsigned long> contributingSources;
long long timestamp; // microseconds
};
// New interfaces to define encoded video and audio frames. Will eventually
// re-use or extend the equivalent defined in WebCodecs.
[Pref="media.peerconnection.enabled",
Pref="media.peerconnection.scripttransform.enabled",
Exposed=(Window,DedicatedWorker)]
interface RTCEncodedVideoFrame {
readonly attribute RTCEncodedVideoFrameType type;
readonly attribute unsigned long timestamp;
attribute ArrayBuffer data;
RTCEncodedVideoFrameMetadata getMetadata();
};

6
dom/webidl/RTCRtpReceiver.webidl
View File

@ -32,3 +32,9 @@ partial interface RTCRtpReceiver {
[Throws]
attribute DOMHighResTimeStamp? jitterBufferTarget;
};
// https://w3c.github.io/webrtc-encoded-transform/#specification
partial interface RTCRtpReceiver {
[SetterThrows,
Pref="media.peerconnection.scripttransform.enabled"] attribute RTCRtpTransform? transform;
};

20
dom/webidl/RTCRtpScriptTransform.webidl Normal file
View File

@ -0,0 +1,20 @@
/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/.
*
* The origin of this IDL file is
* https://www.w3.org/TR/webrtc-encoded-transform
*/
// Spec version is commented out (uncomment if SFrameTransform is implemented)
// typedef (SFrameTransform or RTCRtpScriptTransform) RTCRtpTransform;
typedef RTCRtpScriptTransform RTCRtpTransform;
[Pref="media.peerconnection.enabled",
Pref="media.peerconnection.scripttransform.enabled",
Exposed=Window]
interface RTCRtpScriptTransform {
[Throws]
constructor(Worker worker, optional any options, optional sequence<object> transfer);
};

19
dom/webidl/RTCRtpScriptTransformer.webidl Normal file
View File

@ -0,0 +1,19 @@
/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/.
*
* The origin of this IDL file is
* https://www.w3.org/TR/webrtc-encoded-transform
*/
[Pref="media.peerconnection.enabled",
Pref="media.peerconnection.scripttransform.enabled",
Exposed=DedicatedWorker]
interface RTCRtpScriptTransformer {
readonly attribute ReadableStream readable;
readonly attribute WritableStream writable;
[Throws] readonly attribute any options;
Promise<unsigned long long> generateKeyFrame(optional DOMString rid);
Promise<undefined> sendKeyFrameRequest();
};

6
dom/webidl/RTCRtpSender.webidl
View File

@ -30,3 +30,9 @@ interface RTCRtpSender {
[ChromeOnly]
undefined setTrack(MediaStreamTrack? track);
};
// https://w3c.github.io/webrtc-encoded-transform/#specification
partial interface RTCRtpSender {
[SetterThrows,
Pref="media.peerconnection.scripttransform.enabled"] attribute RTCRtpTransform? transform;
};

20
dom/webidl/RTCTransformEvent.webidl Normal file
View File

@ -0,0 +1,20 @@
/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/.
*
* The origin of this IDL file is
* https://www.w3.org/TR/webrtc-encoded-transform
*/
[Pref="media.peerconnection.enabled",
Pref="media.peerconnection.scripttransform.enabled",
Exposed=DedicatedWorker]
interface RTCTransformEvent : Event {
constructor(DOMString type, RTCTransformEventInit eventInitDict);
readonly attribute RTCRtpScriptTransformer transformer;
};
dictionary RTCTransformEventInit : EventInit {
required RTCRtpScriptTransformer transformer;
};

6
dom/webidl/moz.build
View File

@ -1060,6 +1060,8 @@ if CONFIG["MOZ_WEBRTC"]:
"RTCDataChannel.webidl",
"RTCDtlsTransport.webidl",
"RTCDTMFSender.webidl",
"RTCEncodedAudioFrame.webidl",
"RTCEncodedVideoFrame.webidl",
"RTCIceCandidate.webidl",
"RTCIdentityAssertion.webidl",
"RTCIdentityProvider.webidl",
@ -1068,12 +1070,15 @@ if CONFIG["MOZ_WEBRTC"]:
"RTCRtpCapabilities.webidl",
"RTCRtpParameters.webidl",
"RTCRtpReceiver.webidl",
"RTCRtpScriptTransform.webidl",
"RTCRtpScriptTransformer.webidl",
"RTCRtpSender.webidl",
"RTCRtpSources.webidl",
"RTCRtpTransceiver.webidl",
"RTCSctpTransport.webidl",
"RTCSessionDescription.webidl",
"RTCStatsReport.webidl",
"RTCTransformEvent.webidl",
"WebrtcGlobalInformation.webidl",
]
@ -1181,6 +1186,7 @@ GENERATED_EVENTS_WEBIDL_FILES = [
"PositionStateEvent.webidl",
"ProgressEvent.webidl",
"PromiseRejectionEvent.webidl",
"RTCTransformEvent.webidl",
"ScrollViewChangeEvent.webidl",
"SecurityPolicyViolationEvent.webidl",
"StyleSheetApplicableStateChangeEvent.webidl",

164
dom/workers/EventWithOptionsRunnable.cpp Normal file
View File

@ -0,0 +1,164 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "EventWithOptionsRunnable.h"
#include "WorkerScope.h"
#include "mozilla/dom/WorkerRunnable.h"
#include "mozilla/dom/StructuredCloneHolder.h"
#include "js/StructuredClone.h"
#include "js/RootingAPI.h"
#include "js/Value.h"
#include "nsJSPrincipals.h"
#include "nsContentUtils.h"
#include "nsDebug.h"
#include "MainThreadUtils.h"
#include "mozilla/Assertions.h"
#include "nsGlobalWindowInner.h"
#include "mozilla/DOMEventTargetHelper.h"
#include "mozilla/ErrorResult.h"
#include "nsIGlobalObject.h"
#include "nsCOMPtr.h"
#include "js/GlobalObject.h"
#include "xpcpublic.h"
#include "mozilla/dom/MessagePortBinding.h"
#include "mozilla/dom/MessagePort.h"
#include "mozilla/OwningNonNull.h"
#include "mozilla/RefPtr.h"
#include "mozilla/dom/Event.h"
#include "mozilla/dom/WorkerCommon.h"
namespace mozilla::dom {
EventWithOptionsRunnable::EventWithOptionsRunnable(Worker& aWorker)
: WorkerDebuggeeRunnable(aWorker.mWorkerPrivate,
WorkerRunnable::WorkerThreadModifyBusyCount),
StructuredCloneHolder(CloningSupported, TransferringSupported,
StructuredCloneScope::SameProcess) {}
EventWithOptionsRunnable::~EventWithOptionsRunnable() = default;
void EventWithOptionsRunnable::InitOptions(
JSContext* aCx, JS::Handle<JS::Value> aOptions,
const Sequence<JSObject*>& aTransferable, ErrorResult& aRv) {
JS::Rooted<JS::Value> transferable(aCx, JS::UndefinedValue());
aRv = nsContentUtils::CreateJSValueFromSequenceOfObject(aCx, aTransferable,
&transferable);
if (NS_WARN_IF(aRv.Failed())) {
return;
}
JS::CloneDataPolicy clonePolicy;
// DedicatedWorkers are always part of the same agent cluster.
clonePolicy.allowIntraClusterClonableSharedObjects();
MOZ_ASSERT(NS_IsMainThread());
nsGlobalWindowInner* win = nsContentUtils::IncumbentInnerWindow();
if (win && win->IsSharedMemoryAllowed()) {
clonePolicy.allowSharedMemoryObjects();
}
Write(aCx, aOptions, transferable, clonePolicy, aRv);
}
// Cargo-culted from MesssageEventRunnable.
bool EventWithOptionsRunnable::BuildAndFireEvent(
JSContext* aCx, WorkerPrivate* aWorkerPrivate,
DOMEventTargetHelper* aTarget) {
IgnoredErrorResult rv;
nsCOMPtr<nsIGlobalObject> parent = aTarget->GetParentObject();
// For some workers without window, parent is null and we try to find it
// from the JS Context.
if (!parent) {
JS::Rooted<JSObject*> globalObject(aCx, JS::CurrentGlobalOrNull(aCx));
if (NS_WARN_IF(!globalObject)) {
rv.ThrowDataCloneError("failed to get global object");
OptionsDeserializeFailed(rv);
return false;
}
parent = xpc::NativeGlobal(globalObject);
if (NS_WARN_IF(!parent)) {
rv.ThrowDataCloneError("failed to get parent");
OptionsDeserializeFailed(rv);
return false;
}
}
MOZ_ASSERT(parent);
JS::Rooted<JS::Value> options(aCx);
JS::CloneDataPolicy cloneDataPolicy;
if (parent->GetClientInfo().isSome() &&
parent->GetClientInfo()->AgentClusterId().isSome() &&
parent->GetClientInfo()->AgentClusterId()->Equals(
aWorkerPrivate->AgentClusterId())) {
cloneDataPolicy.allowIntraClusterClonableSharedObjects();
}
if (aWorkerPrivate->IsSharedMemoryAllowed()) {
cloneDataPolicy.allowSharedMemoryObjects();
}
Read(parent, aCx, &options, cloneDataPolicy, rv);
if (NS_WARN_IF(rv.Failed())) {
OptionsDeserializeFailed(rv);
return false;
}
Sequence<OwningNonNull<MessagePort>> ports;
if (NS_WARN_IF(!TakeTransferredPortsAsSequence(ports))) {
// TODO: Is this an appropriate type? What does this actually do?
rv.ThrowDataCloneError("TakeTransferredPortsAsSequence failed");
OptionsDeserializeFailed(rv);
return false;
}
RefPtr<dom::Event> event = BuildEvent(aCx, parent, aTarget, options);
if (NS_WARN_IF(!event)) {
return false;
}
aTarget->DispatchEvent(*event);
return true;
}
bool EventWithOptionsRunnable::WorkerRun(JSContext* aCx,
WorkerPrivate* aWorkerPrivate) {
if (mBehavior == ParentThreadUnchangedBusyCount) {
// Don't fire this event if the JS object has been disconnected from the
// private object.
if (!aWorkerPrivate->IsAcceptingEvents()) {
return true;
}
// Once a window has frozen its workers, their
// mMainThreadDebuggeeEventTargets should be paused, and their
// WorkerDebuggeeRunnables should not be being executed. The same goes for
// WorkerDebuggeeRunnables sent from child to parent workers, but since a
// frozen parent worker runs only control runnables anyway, that is taken
// care of naturally.
MOZ_ASSERT(!aWorkerPrivate->IsFrozen());
// Similarly for paused windows; all its workers should have been informed.
// (Subworkers are unaffected by paused windows.)
MOZ_ASSERT(!aWorkerPrivate->IsParentWindowPaused());
aWorkerPrivate->AssertInnerWindowIsCorrect();
return BuildAndFireEvent(aCx, aWorkerPrivate,
aWorkerPrivate->ParentEventTargetRef());
}
MOZ_ASSERT(aWorkerPrivate == GetWorkerPrivateFromContext(aCx));
return BuildAndFireEvent(aCx, aWorkerPrivate, aWorkerPrivate->GlobalScope());
}
} // namespace mozilla::dom

55
dom/workers/EventWithOptionsRunnable.h Normal file
View File

@ -0,0 +1,55 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef MOZILLA_DOM_WORKERS_EVENTWITHOPTIONSRUNNABLE_H_
#define MOZILLA_DOM_WORKERS_EVENTWITHOPTIONSRUNNABLE_H_
#include "WorkerCommon.h"
#include "WorkerRunnable.h"
#include "mozilla/dom/StructuredCloneHolder.h"
namespace mozilla {
class DOMEventTargetHelper;
namespace dom {
class Event;
class EventTarget;
class Worker;
class WorkerPrivate;
// Cargo-culted from MesssageEventRunnable.
// Intended to be used for the idiom where arbitrary options are transferred to
// the worker thread (with optional transfer functions), which are then used to
// build an event, which is then fired on the global worker scope.
class EventWithOptionsRunnable : public WorkerDebuggeeRunnable,
public StructuredCloneHolder {
public:
explicit EventWithOptionsRunnable(Worker& aWorker);
void InitOptions(JSContext* aCx, JS::Handle<JS::Value> aOptions,
const Sequence<JSObject*>& aTransferable, ErrorResult& aRv);
// Called on the worker thread. The event returned will be fired on the
// worker's global scope. If a StrongWorkerRef needs to be retained, the
// implementation can do so with the WorkerPrivate.
virtual already_AddRefed<Event> BuildEvent(
JSContext* aCx, nsIGlobalObject* aGlobal, EventTarget* aTarget,
JS::Handle<JS::Value> aOptions) = 0;
// Called on the worker thread
virtual void OptionsDeserializeFailed(ErrorResult& aRv) {}
protected:
virtual ~EventWithOptionsRunnable();
private:
bool WorkerRun(JSContext* aCx, WorkerPrivate* aWorkerPrivate) override;
bool BuildAndFireEvent(JSContext* aCx, WorkerPrivate* aWorkerPrivate,
DOMEventTargetHelper* aTarget);
};
} // namespace dom
} // namespace mozilla
#endif // MOZILLA_DOM_WORKERS_EVENTWITHOPTIONSRUNNABLE_H_

35
dom/workers/Worker.cpp
View File

@ -16,6 +16,13 @@
#include "nsContentUtils.h"
#include "nsGlobalWindowOuter.h"
#include "WorkerPrivate.h"
#include "EventWithOptionsRunnable.h"
#include "js/RootingAPI.h"
#include "mozilla/dom/BindingDeclarations.h"
#include "nsISupports.h"
#include "nsDebug.h"
#include "mozilla/dom/WorkerStatus.h"
#include "mozilla/RefPtr.h"
#ifdef XP_WIN
# undef PostMessage
@ -177,6 +184,34 @@ void Worker::PostMessage(JSContext* aCx, JS::Handle<JS::Value> aMessage,
PostMessage(aCx, aMessage, aOptions.mTransfer, aRv);
}
void Worker::PostEventWithOptions(JSContext* aCx,
JS::Handle<JS::Value> aOptions,
const Sequence<JSObject*>& aTransferable,
EventWithOptionsRunnable* aRunnable,
ErrorResult& aRv) {
NS_ASSERT_OWNINGTHREAD(Worker);
if (NS_WARN_IF(!mWorkerPrivate ||
mWorkerPrivate->ParentStatusProtected() > Running)) {
return;
}
RefPtr<WorkerPrivate> workerPrivate = mWorkerPrivate;
Unused << workerPrivate;
aRunnable->InitOptions(aCx, aOptions, aTransferable, aRv);
if (NS_WARN_IF(!mWorkerPrivate ||
mWorkerPrivate->ParentStatusProtected() > Running)) {
return;
}
if (NS_WARN_IF(aRv.Failed())) {
return;
}
Unused << NS_WARN_IF(!aRunnable->Dispatch());
}
void Worker::Terminate() {
NS_ASSERT_OWNINGTHREAD(Worker);

7
dom/workers/Worker.h
View File

@ -19,6 +19,7 @@
namespace mozilla::dom {
class EventWithOptionsRunnable;
struct StructuredSerializeOptions;
struct WorkerOptions;
class WorkerPrivate;
@ -48,6 +49,11 @@ class Worker : public DOMEventTargetHelper, public SupportsWeakPtr {
const StructuredSerializeOptions& aOptions,
ErrorResult& aRv);
void PostEventWithOptions(JSContext* aCx, JS::Handle<JS::Value> aOptions,
const Sequence<JSObject*>& aTransferable,
EventWithOptionsRunnable* aRunnable,
ErrorResult& aRv);
void Terminate();
IMPL_EVENT_HANDLER(error)
@ -59,6 +65,7 @@ class Worker : public DOMEventTargetHelper, public SupportsWeakPtr {
already_AddRefed<WorkerPrivate> aWorkerPrivate);
~Worker();
friend class EventWithOptionsRunnable;
RefPtr<WorkerPrivate> mWorkerPrivate;
};

1
dom/workers/WorkerScope.h
View File

@ -424,6 +424,7 @@ class DedicatedWorkerGlobalScope final
IMPL_EVENT_HANDLER(message)
IMPL_EVENT_HANDLER(messageerror)
IMPL_EVENT_HANDLER(rtctransform)
private:
~DedicatedWorkerGlobalScope() = default;

2
dom/workers/moz.build
View File

@ -12,6 +12,7 @@ DIRS += ["remoteworkers", "sharedworkers", "loader"]
# Public stuff.
EXPORTS.mozilla.dom += [
"ChromeWorker.h",
"EventWithOptionsRunnable.h",
"JSExecutionManager.h",
"Worker.h",
"WorkerChannelInfo.h",
@ -51,6 +52,7 @@ XPIDL_SOURCES += [
UNIFIED_SOURCES += [
"ChromeWorker.cpp",
"ChromeWorkerScope.cpp",
"EventWithOptionsRunnable.cpp",
"JSExecutionManager.cpp",
"MessageEventRunnable.cpp",
"RegisterBindings.cpp",

10
dom/workers/test/test_worker_interfaces.js
View File

@ -307,6 +307,14 @@ let interfaceNamesInGlobalScope = [
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "Response", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCEncodedAudioFrame", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCEncodedVideoFrame", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCRtpScriptTransformer", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "RTCTransformEvent", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "Scheduler", insecureContext: true, nightly: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "StorageManager", fennec: false },
@ -426,6 +434,8 @@ let interfaceNamesInGlobalScope = [
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "onmessageerror", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "onrtctransform", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "postMessage", insecureContext: true },
// IMPORTANT: Do not change this list without review from a DOM peer!
{ name: "requestAnimationFrame", insecureContext: true },

12
media/webrtc/signaling/gtest/Canonicals.h
View File

@ -29,6 +29,8 @@ class ConcreteCanonicals {
INIT_CANONICAL(mLocalRecvRtpExtensions, RtpExtList()),
INIT_CANONICAL(mRemoteSsrc, 0),
INIT_CANONICAL(mRemoteVideoRtxSsrc, 0),
INIT_CANONICAL(mFrameTransformerProxySend, nullptr),
INIT_CANONICAL(mFrameTransformerProxyRecv, nullptr),
INIT_CANONICAL(mAudioRecvCodecs, std::vector<AudioCodecConfig>()),
INIT_CANONICAL(mAudioSendCodec, Nothing()),
INIT_CANONICAL(mVideoRecvCodecs, std::vector<VideoCodecConfig>()),
@ -49,6 +51,8 @@ class ConcreteCanonicals {
Canonical<RtpExtList> mLocalRecvRtpExtensions;
Canonical<Ssrc> mRemoteSsrc;
Canonical<Ssrc> mRemoteVideoRtxSsrc;
Canonical<RefPtr<FrameTransformerProxy>> mFrameTransformerProxySend;
Canonical<RefPtr<FrameTransformerProxy>> mFrameTransformerProxyRecv;
Canonical<std::vector<AudioCodecConfig>> mAudioRecvCodecs;
Canonical<Maybe<AudioCodecConfig>> mAudioSendCodec;
@ -103,6 +107,14 @@ class ConcreteControl : public AudioConduitControlInterface,
Canonical<RtpExtList>& CanonicalLocalSendRtpExtensions() override {
return mLocalSendRtpExtensions;
}
Canonical<RefPtr<FrameTransformerProxy>>& CanonicalFrameTransformerProxySend()
override {
return mFrameTransformerProxySend;
}
Canonical<RefPtr<FrameTransformerProxy>>& CanonicalFrameTransformerProxyRecv()
override {
return mFrameTransformerProxyRecv;
}
// AudioConduitControlInterface
Canonical<Maybe<AudioCodecConfig>>& CanonicalAudioSendCodec() override {

1
media/webrtc/signaling/gtest/MockConduit.h
View File

@ -9,6 +9,7 @@
#include "gmock/gmock.h"
#include "MediaConduitInterface.h"
#include "libwebrtcglue/FrameTransformer.h"
namespace webrtc {
std::ostream& operator<<(std::ostream& aStream,

7
modules/libpref/init/StaticPrefList.yaml
View File

@ -10750,7 +10750,12 @@
# navigator.mediaDevices and getUserMedia() support as well.
# See also media.navigator.enabled
- name: media.peerconnection.enabled
type: bool
type: RelaxedAtomicBool
value: true
mirror: always
- name: media.peerconnection.scripttransform.enabled
type: RelaxedAtomicBool
value: true
mirror: always

1
widget/EventMessageList.h
View File

@ -359,6 +359,7 @@ NS_EVENT_MESSAGE(eAfterPrint)
NS_EVENT_MESSAGE(eMessage)
NS_EVENT_MESSAGE(eMessageError)
NS_EVENT_MESSAGE(eRTCTransform)
// Menu open event
NS_EVENT_MESSAGE(eOpen)

2
xpcom/ds/StaticAtoms.py
View File

@ -1934,6 +1934,8 @@ STATIC_ATOMS = [
Atom("ondevicelight", "ondevicelight"),
# MediaDevices device change event
Atom("ondevicechange", "ondevicechange"),
# WebRTC events
Atom("onrtctransform", "onrtctransform"),
# Internal Visual Viewport events
Atom("onmozvisualresize", "onmozvisualresize"),
Atom("onmozvisualscroll", "onmozvisualscroll"),