Bug 1388219 - add a nsTArray mTargetCapability to record each track target capability. r=jib

MozReview-Commit-ID: E8ZCmXEDxKs

--HG--
extra : rebase_source : 5cab9cdb5cc1a67d6cf4c0b5c5c7caef5cfe7ea0
This commit is contained in:
Munro Mengjue Chiang 2017-11-30 15:44:20 +08:00
parent 03a6e9d255
commit e782a0379f
9 changed files with 195 additions and 37 deletions

View File

@ -217,6 +217,7 @@ public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AllocationHandle);
protected:
~AllocationHandle() {}
static uint64_t sId;
public:
AllocationHandle(const dom::MediaTrackConstraints& aConstraints,
const mozilla::ipc::PrincipalInfo& aPrincipalInfo,
@ -226,11 +227,15 @@ public:
: mConstraints(aConstraints),
mPrincipalInfo(aPrincipalInfo),
mPrefs(aPrefs),
#ifdef MOZ_WEBRTC
mId(sId++),
#endif
mDeviceId(aDeviceId) {}
public:
NormalizedConstraints mConstraints;
mozilla::ipc::PrincipalInfo mPrincipalInfo;
MediaEnginePrefs mPrefs;
uint64_t mId;
nsString mDeviceId;
};
@ -366,6 +371,7 @@ protected:
virtual nsresult
UpdateSingleSource(const AllocationHandle* aHandle,
const NormalizedConstraints& aNetConstraints,
const NormalizedConstraints& aNewConstraint,
const MediaEnginePrefs& aPrefs,
const nsString& aDeviceId,
const char** aOutBadConstraint) {
@ -394,6 +400,7 @@ protected:
// aHandle and/or aConstraintsUpdate may be nullptr (see below)
AutoTArray<const NormalizedConstraints*, 10> allConstraints;
AutoTArray<const NormalizedConstraints*, 1> updatedConstraint;
for (auto& registered : mRegisteredHandles) {
if (aConstraintsUpdate && registered.get() == aHandle) {
continue; // Don't count old constraints
@ -402,9 +409,13 @@ protected:
}
if (aConstraintsUpdate) {
allConstraints.AppendElement(aConstraintsUpdate);
updatedConstraint.AppendElement(aConstraintsUpdate);
} else if (aHandle) {
// In the case of AddShareOfSingleSource, the handle isn't registered yet.
allConstraints.AppendElement(&aHandle->mConstraints);
updatedConstraint.AppendElement(&aHandle->mConstraints);
} else {
updatedConstraint.AppendElements(allConstraints);
}
NormalizedConstraints netConstraints(allConstraints);
@ -413,7 +424,8 @@ protected:
return NS_ERROR_FAILURE;
}
nsresult rv = UpdateSingleSource(aHandle, netConstraints, aPrefs, aDeviceId,
NormalizedConstraints newConstraint(updatedConstraint);
nsresult rv = UpdateSingleSource(aHandle, netConstraints, newConstraint, aPrefs, aDeviceId,
aOutBadConstraint);
if (NS_FAILED(rv)) {
return rv;

View File

@ -54,6 +54,19 @@ MediaEngineCameraVideoSource::GetCapability(size_t aIndex,
aOut = mHardcodedCapabilities.SafeElementAt(aIndex, webrtc::CaptureCapability());
}
uint32_t
MediaEngineCameraVideoSource::GetDistance(
const webrtc::CaptureCapability& aCandidate,
const NormalizedConstraintSet &aConstraints,
const nsString& aDeviceId,
const DistanceCalculation aCalculate) const
{
if (aCalculate == kFeasibility) {
return GetFeasibilityDistance(aCandidate, aConstraints, aDeviceId);
}
return GetFitnessDistance(aCandidate, aConstraints, aDeviceId);
}
uint32_t
MediaEngineCameraVideoSource::GetFitnessDistance(
const webrtc::CaptureCapability& aCandidate,
@ -75,6 +88,27 @@ MediaEngineCameraVideoSource::GetFitnessDistance(
return uint32_t(std::min(distance, uint64_t(UINT32_MAX)));
}
uint32_t
MediaEngineCameraVideoSource::GetFeasibilityDistance(
const webrtc::CaptureCapability& aCandidate,
const NormalizedConstraintSet &aConstraints,
const nsString& aDeviceId) const
{
// Treat width|height|frameRate == 0 on capability as "can do any".
// This allows for orthogonal capabilities that are not in discrete steps.
uint64_t distance =
uint64_t(FitnessDistance(aDeviceId, aConstraints.mDeviceId)) +
uint64_t(FitnessDistance(mFacingMode, aConstraints.mFacingMode)) +
uint64_t(aCandidate.width? FeasibilityDistance(int32_t(aCandidate.width),
aConstraints.mWidth) : 0) +
uint64_t(aCandidate.height? FeasibilityDistance(int32_t(aCandidate.height),
aConstraints.mHeight) : 0) +
uint64_t(aCandidate.maxFPS? FeasibilityDistance(double(aCandidate.maxFPS),
aConstraints.mFrameRate) : 0);
return uint32_t(std::min(distance, uint64_t(UINT32_MAX)));
}
// Find best capability by removing inferiors. May leave >1 of equal distance
/* static */ void
@ -218,7 +252,9 @@ bool
MediaEngineCameraVideoSource::ChooseCapability(
const NormalizedConstraints &aConstraints,
const MediaEnginePrefs &aPrefs,
const nsString& aDeviceId)
const nsString& aDeviceId,
webrtc::CaptureCapability& aCapability,
const DistanceCalculation aCalculate)
{
if (MOZ_LOG_TEST(GetMediaManagerLog(), LogLevel::Debug)) {
LOG(("ChooseCapability: prefs: %dx%d @%dfps",
@ -246,7 +282,7 @@ MediaEngineCameraVideoSource::ChooseCapability(
auto& candidate = candidateSet[i];
webrtc::CaptureCapability cap;
GetCapability(candidate.mIndex, cap);
candidate.mDistance = GetFitnessDistance(cap, aConstraints, aDeviceId);
candidate.mDistance = GetDistance(cap, aConstraints, aDeviceId, aCalculate);
LogCapability("Capability", cap, candidate.mDistance);
if (candidate.mDistance == UINT32_MAX) {
candidateSet.RemoveElementAt(i);
@ -268,7 +304,7 @@ MediaEngineCameraVideoSource::ChooseCapability(
auto& candidate = candidateSet[i];
webrtc::CaptureCapability cap;
GetCapability(candidate.mIndex, cap);
if (GetFitnessDistance(cap, cs, aDeviceId) == UINT32_MAX) {
if (GetDistance(cap, cs, aDeviceId, aCalculate) == UINT32_MAX) {
rejects.AppendElement(candidate);
candidateSet.RemoveElementAt(i);
} else {
@ -299,7 +335,7 @@ MediaEngineCameraVideoSource::ChooseCapability(
for (auto& candidate : candidateSet) {
webrtc::CaptureCapability cap;
GetCapability(candidate.mIndex, cap);
candidate.mDistance = GetFitnessDistance(cap, normPrefs, aDeviceId);
candidate.mDistance = GetDistance(cap, normPrefs, aDeviceId, aCalculate);
}
TrimLessFitCandidates(candidateSet);
}
@ -315,13 +351,13 @@ MediaEngineCameraVideoSource::ChooseCapability(
if (cap.rawType == webrtc::RawVideoType::kVideoI420 ||
cap.rawType == webrtc::RawVideoType::kVideoYUY2 ||
cap.rawType == webrtc::RawVideoType::kVideoYV12) {
mCapability = cap;
aCapability = cap;
found = true;
break;
}
}
if (!found) {
GetCapability(candidateSet[0].mIndex, mCapability);
GetCapability(candidateSet[0].mIndex, aCapability);
}
LogCapability("Chosen capability", mCapability, sameDistance);

View File

@ -24,6 +24,19 @@ namespace webrtc {
namespace mozilla {
// Fitness distance is defined in
// https://www.w3.org/TR/2017/CR-mediacapture-streams-20171003/#dfn-selectsettings
// The main difference of feasibility and fitness distance is that if the
// constraint is required ('max', or 'exact'), and the settings dictionary's value
// for the constraint does not satisfy the constraint, the fitness distance is
// positive infinity. Given a continuous space of settings dictionaries comprising
// all discrete combinations of dimension and frame-rate related properties,
// the feasibility distance is still in keeping with the constraints algorithm.
enum DistanceCalculation {
kFitness,
kFeasibility
};
class MediaEngineCameraVideoSource : public MediaEngineVideoSource
{
public:
@ -86,9 +99,16 @@ protected:
TrackID aID,
StreamTime delta,
const PrincipalHandle& aPrincipalHandle);
uint32_t GetDistance(const webrtc::CaptureCapability& aCandidate,
const NormalizedConstraintSet &aConstraints,
const nsString& aDeviceId,
const DistanceCalculation aCalculate) const;
uint32_t GetFitnessDistance(const webrtc::CaptureCapability& aCandidate,
const NormalizedConstraintSet &aConstraints,
const nsString& aDeviceId) const;
uint32_t GetFeasibilityDistance(const webrtc::CaptureCapability& aCandidate,
const NormalizedConstraintSet &aConstraints,
const nsString& aDeviceId) const;
static void TrimLessFitCandidates(CapabilitySet& set);
static void LogConstraints(const NormalizedConstraintSet& aConstraints);
static void LogCapability(const char* aHeader,
@ -96,9 +116,13 @@ protected:
uint32_t aDistance);
virtual size_t NumCapabilities() const;
virtual void GetCapability(size_t aIndex, webrtc::CaptureCapability& aOut) const;
virtual bool ChooseCapability(const NormalizedConstraints &aConstraints,
const MediaEnginePrefs &aPrefs,
const nsString& aDeviceId);
virtual bool ChooseCapability(
const NormalizedConstraints &aConstraints,
const MediaEnginePrefs &aPrefs,
const nsString& aDeviceId,
webrtc::CaptureCapability& aCapability,
const DistanceCalculation aCalculate
);
void SetName(nsString aName);
void SetUUID(const char* aUUID);
const nsCString& GetUUID() const; // protected access
@ -116,6 +140,8 @@ protected:
nsTArray<RefPtr<SourceMediaStream>> mSources; // When this goes empty, we shut down HW
nsTArray<PrincipalHandle> mPrincipalHandles; // Directly mapped to mSources.
RefPtr<layers::Image> mImage;
nsTArray<webrtc::CaptureCapability> mTargetCapabilities;
nsTArray<uint64_t> mHandleIds;
RefPtr<layers::ImageContainer> mImageContainer;
// end of data protected by mMonitor
@ -125,6 +151,8 @@ protected:
TrackID mTrackID;
webrtc::CaptureCapability mCapability;
webrtc::CaptureCapability mTargetCapability;
uint64_t mHandleId;
mutable nsTArray<webrtc::CaptureCapability> mHardcodedCapabilities;
private:

View File

@ -17,6 +17,8 @@ extern mozilla::LogModule* GetMediaManagerLog();
namespace mozilla {
uint64_t MediaEngineCameraVideoSource::AllocationHandle::sId = 0;
// These need a definition somewhere because template
// code is allowed to take their address, and they aren't
// guaranteed to have one without this.
@ -80,6 +82,8 @@ MediaEngineRemoteVideoSource::Shutdown()
empty = mSources.IsEmpty();
if (empty) {
MOZ_ASSERT(mPrincipalHandles.IsEmpty());
MOZ_ASSERT(mTargetCapabilities.IsEmpty());
MOZ_ASSERT(mHandleIds.IsEmpty());
break;
}
source = mSources[0];
@ -126,6 +130,8 @@ MediaEngineRemoteVideoSource::Allocate(
MonitorAutoLock lock(mMonitor);
if (mSources.IsEmpty()) {
MOZ_ASSERT(mPrincipalHandles.IsEmpty());
MOZ_ASSERT(mTargetCapabilities.IsEmpty());
MOZ_ASSERT(mHandleIds.IsEmpty());
LOG(("Video device %d reallocated", mCaptureIndex));
} else {
LOG(("Video device %d allocated shared", mCaptureIndex));
@ -172,7 +178,12 @@ MediaEngineRemoteVideoSource::Start(SourceMediaStream* aStream, TrackID aID,
MonitorAutoLock lock(mMonitor);
mSources.AppendElement(aStream);
mPrincipalHandles.AppendElement(aPrincipalHandle);
mTargetCapabilities.AppendElement(mTargetCapability);
mHandleIds.AppendElement(mHandleId);
MOZ_ASSERT(mSources.Length() == mPrincipalHandles.Length());
MOZ_ASSERT(mSources.Length() == mTargetCapabilities.Length());
MOZ_ASSERT(mSources.Length() == mHandleIds.Length());
}
aStream->AddTrack(aID, 0, new VideoSegment(), SourceMediaStream::ADDTRACK_QUEUED);
@ -218,8 +229,12 @@ MediaEngineRemoteVideoSource::Stop(mozilla::SourceMediaStream* aSource,
}
MOZ_ASSERT(mSources.Length() == mPrincipalHandles.Length());
MOZ_ASSERT(mSources.Length() == mTargetCapabilities.Length());
MOZ_ASSERT(mSources.Length() == mHandleIds.Length());
mSources.RemoveElementAt(i);
mPrincipalHandles.RemoveElementAt(i);
mTargetCapabilities.RemoveElementAt(i);
mHandleIds.RemoveElementAt(i);
aSource->EndTrack(aID);
@ -262,18 +277,21 @@ nsresult
MediaEngineRemoteVideoSource::UpdateSingleSource(
const AllocationHandle* aHandle,
const NormalizedConstraints& aNetConstraints,
const NormalizedConstraints& aNewConstraint,
const MediaEnginePrefs& aPrefs,
const nsString& aDeviceId,
const char** aOutBadConstraint)
{
if (!ChooseCapability(aNetConstraints, aPrefs, aDeviceId)) {
*aOutBadConstraint = FindBadConstraint(aNetConstraints, *this, aDeviceId);
return NS_ERROR_FAILURE;
}
switch (mState) {
case kReleased:
MOZ_ASSERT(aHandle);
mHandleId = aHandle->mId;
if (!ChooseCapability(aNetConstraints, aPrefs, aDeviceId, mCapability, kFitness)) {
*aOutBadConstraint = FindBadConstraint(aNetConstraints, *this, aDeviceId);
return NS_ERROR_FAILURE;
}
mTargetCapability = mCapability;
if (camera::GetChildAndCall(&camera::CamerasChild::AllocateCaptureDevice,
mCapEngine, GetUUID().get(),
kMaxUniqueIdLength, mCaptureIndex,
@ -286,18 +304,42 @@ MediaEngineRemoteVideoSource::UpdateSingleSource(
break;
case kStarted:
if (mCapability != mLastCapability) {
camera::GetChildAndCall(&camera::CamerasChild::StopCapture,
mCapEngine, mCaptureIndex);
if (camera::GetChildAndCall(&camera::CamerasChild::StartCapture,
mCapEngine, mCaptureIndex, mCapability,
this)) {
LOG(("StartCapture failed"));
{
size_t index = mHandleIds.NoIndex;
if (aHandle) {
mHandleId = aHandle->mId;
index = mHandleIds.IndexOf(mHandleId);
}
if (!ChooseCapability(aNewConstraint, aPrefs, aDeviceId, mTargetCapability,
kFitness)) {
*aOutBadConstraint = FindBadConstraint(aNewConstraint, *this, aDeviceId);
return NS_ERROR_FAILURE;
}
SetLastCapability(mCapability);
if (index != mHandleIds.NoIndex) {
mTargetCapabilities[index] = mTargetCapability;
}
if (!ChooseCapability(aNetConstraints, aPrefs, aDeviceId, mCapability,
kFeasibility)) {
*aOutBadConstraint = FindBadConstraint(aNetConstraints, *this, aDeviceId);
return NS_ERROR_FAILURE;
}
if (mCapability != mLastCapability) {
camera::GetChildAndCall(&camera::CamerasChild::StopCapture,
mCapEngine, mCaptureIndex);
if (camera::GetChildAndCall(&camera::CamerasChild::StartCapture,
mCapEngine, mCaptureIndex, mCapability,
this)) {
LOG(("StartCapture failed"));
return NS_ERROR_FAILURE;
}
SetLastCapability(mCapability);
}
break;
}
break;
default:
LOG(("Video device %d in ignored state %d", mCaptureIndex, mState));
@ -464,7 +506,9 @@ bool
MediaEngineRemoteVideoSource::ChooseCapability(
const NormalizedConstraints &aConstraints,
const MediaEnginePrefs &aPrefs,
const nsString& aDeviceId)
const nsString& aDeviceId,
webrtc::CaptureCapability& aCapability,
const DistanceCalculation aCalculate)
{
AssertIsOnOwningThread();
@ -477,15 +521,16 @@ MediaEngineRemoteVideoSource::ChooseCapability(
// time (and may in fact change over time), so as a hack, we push ideal
// and max constraints down to desktop_capture_impl.cc and finish the
// algorithm there.
mCapability.width = (c.mWidth.mIdeal.valueOr(0) & 0xffff) << 16 |
(c.mWidth.mMax & 0xffff);
mCapability.height = (c.mHeight.mIdeal.valueOr(0) & 0xffff) << 16 |
(c.mHeight.mMax & 0xffff);
mCapability.maxFPS = c.mFrameRate.Clamp(c.mFrameRate.mIdeal.valueOr(aPrefs.mFPS));
aCapability.width =
(c.mWidth.mIdeal.valueOr(0) & 0xffff) << 16 | (c.mWidth.mMax & 0xffff);
aCapability.height =
(c.mHeight.mIdeal.valueOr(0) & 0xffff) << 16 | (c.mHeight.mMax & 0xffff);
aCapability.maxFPS =
c.mFrameRate.Clamp(c.mFrameRate.mIdeal.valueOr(aPrefs.mFPS));
return true;
}
default:
return MediaEngineCameraVideoSource::ChooseCapability(aConstraints, aPrefs, aDeviceId);
return MediaEngineCameraVideoSource::ChooseCapability(aConstraints, aPrefs, aDeviceId, aCapability, aCalculate);
}
}

View File

@ -84,9 +84,12 @@ public:
return mMediaSource;
}
bool ChooseCapability(const NormalizedConstraints &aConstraints,
const MediaEnginePrefs &aPrefs,
const nsString& aDeviceId) override;
bool ChooseCapability(
const NormalizedConstraints &aConstraints,
const MediaEnginePrefs &aPrefs,
const nsString& aDeviceId,
webrtc::CaptureCapability& aCapability,
const DistanceCalculation aCalculate) override;
void Refresh(int aIndex);
@ -107,6 +110,7 @@ private:
nsresult
UpdateSingleSource(const AllocationHandle* aHandle,
const NormalizedConstraints& aNetConstraints,
const NormalizedConstraints& aNewConstraint,
const MediaEnginePrefs& aPrefs,
const nsString& aDeviceId,
const char** aOutBadConstraint) override;

View File

@ -566,6 +566,7 @@ private:
nsresult
UpdateSingleSource(const AllocationHandle* aHandle,
const NormalizedConstraints& aNetConstraints,
const NormalizedConstraints& aNewConstraint,
const MediaEnginePrefs& aPrefs,
const nsString& aDeviceId,
const char** aOutBadConstraint) override;

View File

@ -279,6 +279,7 @@ nsresult
MediaEngineWebRTCMicrophoneSource::UpdateSingleSource(
const AllocationHandle* aHandle,
const NormalizedConstraints& aNetConstraints,
const NormalizedConstraints& aNewConstraint, /* Ignored */
const MediaEnginePrefs& aPrefs,
const nsString& aDeviceId,
const char** aOutBadConstraint)

View File

@ -417,6 +417,28 @@ MediaConstraintsHelper::FitnessDistance(ValueType aN,
std::max(std::abs(aN), std::abs(aRange.mIdeal.value()))));
}
template<class ValueType, class NormalizedRange>
/* static */ uint32_t
MediaConstraintsHelper::FeasibilityDistance(ValueType aN,
const NormalizedRange& aRange)
{
if (aRange.mMin > aN) {
return UINT32_MAX;
}
// We prefer larger resolution because now we support downscaling
if (aN == aRange.mIdeal.valueOr(aN)) {
return 0;
}
if (aN > aRange.mIdeal.value()) {
return uint32_t(ValueType((std::abs(aN - aRange.mIdeal.value()) * 1000) /
std::max(std::abs(aN), std::abs(aRange.mIdeal.value()))));
}
return 10000 + uint32_t(ValueType((std::abs(aN - aRange.mIdeal.value()) * 1000) /
std::max(std::abs(aN), std::abs(aRange.mIdeal.value()))));
}
// Fitness distance returned as integer math * 1000. Infinity = UINT32_MAX
/* static */ uint32_t

View File

@ -85,12 +85,19 @@ public:
return mMax >= aOther.mMin && mMin <= aOther.mMax;
}
void Intersect(const Range& aOther) {
MOZ_ASSERT(Intersects(aOther));
mMin = std::max(mMin, aOther.mMin);
mMax = std::min(mMax, aOther.mMax);
if (Intersects(aOther)) {
mMax = std::min(mMax, aOther.mMax);
} else {
// If there is no intersection, we will down-scale or drop frame
mMax = std::max(mMax, aOther.mMax);
}
}
bool Merge(const Range& aOther) {
if (!Intersects(aOther)) {
if (strcmp(mName, "width") != 0 &&
strcmp(mName, "height") != 0 &&
strcmp(mName, "frameRate") != 0 &&
!Intersects(aOther)) {
return false;
}
Intersect(aOther);
@ -297,6 +304,8 @@ class MediaConstraintsHelper
protected:
template<class ValueType, class NormalizedRange>
static uint32_t FitnessDistance(ValueType aN, const NormalizedRange& aRange);
template<class ValueType, class NormalizedRange>
static uint32_t FeasibilityDistance(ValueType aN, const NormalizedRange& aRange);
static uint32_t FitnessDistance(nsString aN,
const NormalizedConstraintSet::StringRange& aConstraint);