diff --git a/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.h b/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.h index 4b0dd1668bc8..f5a45b453162 100644 --- a/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.h +++ b/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.h @@ -11,25 +11,17 @@ #include "api/scoped_refptr.h" #include "api/sequence_checker.h" -#include "MediaEventSource.h" #include "modules/video_capture/video_capture_impl.h" #include "mozilla/Maybe.h" #include "PerformanceRecorder.h" -@interface VideoCaptureAdapter : NSObject { - @public - mozilla::MediaEventProducerExc<__strong RTCVideoFrame* _Nullable> frameEvent; -} -@end +@class VideoCaptureAdapter; namespace webrtc::videocapturemodule { /** * VideoCaptureImpl implementation of the libwebrtc ios/mac sdk camera backend. - * Single threaded API. Callbacks run on the API thread. - * - * Internally callbacks come on a dedicated dispatch queue. We bounce them via a MediaEvent to the - * API thread. + * Single threaded except for OnFrame() that happens on a platform callback thread. */ class VideoCaptureAvFoundation : public VideoCaptureImpl { public: @@ -42,16 +34,19 @@ class VideoCaptureAvFoundation : public VideoCaptureImpl { // Starts capturing synchronously. Idempotent. If an existing capture is live and another // capability is requested we'll restart the underlying backend with the new capability. - int32_t StartCapture(const VideoCaptureCapability& aCapability) override; + int32_t StartCapture(const VideoCaptureCapability& aCapability) MOZ_EXCLUDES(api_lock_) override; // Stops capturing synchronously. Idempotent. - int32_t StopCapture() override; - bool CaptureStarted() override; + int32_t StopCapture() MOZ_EXCLUDES(api_lock_) override; + bool CaptureStarted() MOZ_EXCLUDES(api_lock_) override; int32_t CaptureSettings(VideoCaptureCapability& aSettings) override; - // Callback on the mChecker thread. - void OnFrame(__strong RTCVideoFrame* _Nonnull aFrame); + // Callback. This can be called on any thread. + int32_t OnFrame(__strong RTCVideoFrame* _Nonnull aFrame) MOZ_EXCLUDES(api_lock_); - void SetTrackingId(uint32_t aTrackingIdProcId) override; + void SetTrackingId(uint32_t aTrackingIdProcId) MOZ_EXCLUDES(api_lock_) override; + + // Registers the current thread with the profiler if not already registered. + void MaybeRegisterCallbackThread(); private: // Control thread checker. @@ -59,21 +54,26 @@ class VideoCaptureAvFoundation : public VideoCaptureImpl { AVCaptureDevice* _Nonnull const mDevice RTC_GUARDED_BY(mChecker); VideoCaptureAdapter* _Nonnull const mAdapter RTC_GUARDED_BY(mChecker); RTCCameraVideoCapturer* _Nonnull const mCapturer RTC_GUARDED_BY(mChecker); - // If capture has started, this is the capability it was started for. - mozilla::Maybe mCapability RTC_GUARDED_BY(mChecker); + // If capture has started, this is the capability it was started for. Written on the mChecker + // thread only. + mozilla::Maybe mCapability MOZ_GUARDED_BY(api_lock_); // The image type that mCapability maps to. Set in lockstep with mCapability. - mozilla::Maybe mImageType RTC_GUARDED_BY(mChecker); - // Id string uniquely identifying this capture source. - mozilla::Maybe mTrackingId RTC_GUARDED_BY(mChecker); - // Handle for the mAdapter frame event. mChecker thread only. - mozilla::MediaEventListener mFrameListener RTC_GUARDED_BY(mChecker); - // Adds frame specific markers to the profiler while mTrackingId is set. - mozilla::PerformanceRecorderMulti mCaptureRecorder - RTC_GUARDED_BY(mChecker); - mozilla::PerformanceRecorderMulti mConversionRecorder - RTC_GUARDED_BY(mChecker); + mozilla::Maybe mImageType MOZ_GUARDED_BY(api_lock_); + // Id string uniquely identifying this capture source. Written on the mChecker thread only. + mozilla::Maybe mTrackingId MOZ_GUARDED_BY(api_lock_); + // Adds frame specific markers to the profiler while mTrackingId is set. Callback thread only. + mozilla::PerformanceRecorderMulti mCaptureRecorder; + mozilla::PerformanceRecorderMulti mConversionRecorder; + std::atomic mCallbackThreadId; }; } // namespace webrtc::videocapturemodule +@interface VideoCaptureAdapter : NSObject { + webrtc::Mutex _mutex; + webrtc::videocapturemodule::VideoCaptureAvFoundation* _Nullable _capturer RTC_GUARDED_BY(_mutex); +} +- (void)setCapturer:(webrtc::videocapturemodule::VideoCaptureAvFoundation* _Nullable)capturer; +@end + #endif diff --git a/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm b/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm index eee88f0745b7..ae30671d8243 100644 --- a/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm +++ b/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm @@ -73,9 +73,20 @@ AVCaptureDeviceFormat* _Nullable FindFormat(AVCaptureDevice* _Nonnull aDevice, } // namespace @implementation VideoCaptureAdapter +- (void)setCapturer:(webrtc::videocapturemodule::VideoCaptureAvFoundation* _Nullable)capturer { + webrtc::MutexLock lock(&_mutex); + _capturer = capturer; +} + - (void)capturer:(RTCVideoCapturer* _Nonnull)capturer didCaptureVideoFrame:(RTCVideoFrame* _Nonnull)frame { - frameEvent.Notify(frame); + rtc::scoped_refptr cap; + { + webrtc::MutexLock lock(&_mutex); + cap = rtc::scoped_refptr(_capturer); + } + if (!cap) return; + cap->OnFrame(frame); } @end @@ -83,7 +94,8 @@ namespace webrtc::videocapturemodule { VideoCaptureAvFoundation::VideoCaptureAvFoundation(AVCaptureDevice* _Nonnull aDevice) : mDevice(aDevice), mAdapter([[VideoCaptureAdapter alloc] init]), - mCapturer([[RTC_OBJC_TYPE(RTCCameraVideoCapturer) alloc] initWithDelegate:mAdapter]) { + mCapturer([[RTC_OBJC_TYPE(RTCCameraVideoCapturer) alloc] initWithDelegate:mAdapter]), + mCallbackThreadId() { const char* uniqueId = [[aDevice uniqueID] UTF8String]; size_t len = strlen(uniqueId); _deviceUniqueId = new (std::nothrow) char[len + 1]; @@ -137,8 +149,7 @@ int32_t VideoCaptureAvFoundation::StartCapture(const VideoCaptureCapability& aCa } } - mFrameListener = mAdapter->frameEvent.Connect(GetCurrentSerialEventTarget(), this, - &VideoCaptureAvFoundation::OnFrame); + [mAdapter setCapturer:this]; { Monitor monitor("VideoCaptureAVFoundation::StartCapture"); @@ -227,13 +238,14 @@ int32_t VideoCaptureAvFoundation::StopCapture() { monitor.Wait(); } - mFrameListener.Disconnect(); + [mAdapter setCapturer:nil]; return 0; } bool VideoCaptureAvFoundation::CaptureStarted() { RTC_DCHECK_RUN_ON(&mChecker); + MutexLock lock(&api_lock_); return mCapability.isSome(); } @@ -242,9 +254,9 @@ int32_t VideoCaptureAvFoundation::CaptureSettings(VideoCaptureCapability& aSetti return -1; } -void VideoCaptureAvFoundation::OnFrame(__strong RTCVideoFrame* _Nonnull aFrame) { - RTC_DCHECK_RUN_ON(&mChecker); - if (MOZ_LIKELY(mTrackingId)) { +int32_t VideoCaptureAvFoundation::OnFrame(__strong RTCVideoFrame* _Nonnull aFrame) { + MaybeRegisterCallbackThread(); + if (MutexLock lock(&api_lock_); MOZ_LIKELY(mTrackingId)) { mCaptureRecorder.Start(0, "VideoCaptureAVFoundation"_ns, *mTrackingId, aFrame.width, aFrame.height, mImageType.valueOr(CaptureStage::ImageType::Unknown)); if (mCapability && mCapability->videoType != webrtc::VideoType::kI420) { @@ -266,12 +278,14 @@ void VideoCaptureAvFoundation::OnFrame(__strong RTCVideoFrame* _Nonnull aFrame) .build(); MutexLock lock(&api_lock_); - DeliverCapturedFrame(frame); + int32_t rv = DeliverCapturedFrame(frame); mCaptureRecorder.Record(0); + return rv; } void VideoCaptureAvFoundation::SetTrackingId(uint32_t aTrackingIdProcId) { RTC_DCHECK_RUN_ON(&mChecker); + MutexLock lock(&api_lock_); if (NS_WARN_IF(mTrackingId.isSome())) { // This capture instance must be shared across multiple camera requests. For now ignore other // requests than the first. @@ -279,4 +293,13 @@ void VideoCaptureAvFoundation::SetTrackingId(uint32_t aTrackingIdProcId) { } mTrackingId.emplace(TrackingId::Source::Camera, aTrackingIdProcId); } + +void VideoCaptureAvFoundation::MaybeRegisterCallbackThread() { + ProfilerThreadId id = profiler_current_thread_id(); + if (MOZ_LIKELY(id == mCallbackThreadId)) { + return; + } + mCallbackThreadId = id; + CallbackThreadRegistry::Get()->Register(mCallbackThreadId, "VideoCaptureAVFoundationCallback"); +} } // namespace webrtc::videocapturemodule