Backed out changeset a7c322ebcfea (bug 1737682) for causing high frequency bc failures in browser_panelUINotifications_multiWindow. CLOSED TREE

This commit is contained in:
Sandor Molnar 2021-12-18 01:09:45 +02:00
parent 11fc69125a
commit c088d49147
3 changed files with 80 additions and 251 deletions

View File

@ -42,31 +42,6 @@ namespace layers {
class NativeLayerRootSnapshotterCA;
class SurfacePoolHandleCA;
enum class SpecializeType {
// Order here is important. Later enums will dominate earlier enums.
// These are ordered so that we can start with None, then progress to a
// soft failure state indicating that no further processing is needed.
// Active is followed by increasingly interesting failure states. Any
// state that starts with "Fail" is a failure state.
// These must be kept synchronized with the telemetry histogram enums.
None, // Never emitted as telemetry. A baseline value.
FailNotVideo, // Never emitted as telemetry. No video is visible. We
// treat this as a failure state to prevent further
// processing of the layers.
Active, // "Active" instead of "Success" to avoid name-collision
FailPref, // on Linux.
Fail10_13,
FailFullscreen,
FailIsoMouse, // "FailIso" series denote a failure of isolation.
FailIsoTopVideo, // Abbreviating these enums keeps them below the
FailIsoSize, // 20-character limit for telemetry enums, which we want
FailIsoCenter, // to match exactly.
FailIsoOneVideo,
FailSurface,
FailEnqueue,
};
// NativeLayerRootCA is the CoreAnimation implementation of the NativeLayerRoot
// interface. A NativeLayerRootCA is created by the widget around an existing
// CALayer with a call to CreateForCALayer - this CALayer is the root of the
@ -154,16 +129,14 @@ class NativeLayerRootCA : public NativeLayerRoot {
struct Representation {
explicit Representation(CALayer* aRootCALayer);
~Representation();
SpecializeType Commit(WhichRepresentation aRepresentation,
void Commit(WhichRepresentation aRepresentation,
const nsTArray<RefPtr<NativeLayerCA>>& aSublayers,
bool aWindowIsFullscreen, bool aMouseMovedRecently);
CALayer* FindVideoLayerToIsolate(
WhichRepresentation aRepresentation,
const nsTArray<RefPtr<NativeLayerCA>>& aSublayers,
SpecializeType& aSpecialize // out param
);
const nsTArray<RefPtr<NativeLayerCA>>& aSublayers);
CALayer* mRootCALayer = nullptr; // strong
SpecializeType mIsIsolatingVideo = SpecializeType::FailNotVideo;
bool mIsIsolatingVideo = false;
bool mMutatedLayerStructure = false;
bool mMutatedMouseMovedRecently = false;
};
@ -202,12 +175,6 @@ class NativeLayerRootCA : public NativeLayerRoot {
// Has the mouse recently moved?
bool mMouseMovedRecently = false;
// How many times have we committed since the last time we emitted
// telemetry?
unsigned int mTelemetryCommitCount = 0;
static const unsigned int TELEMETRY_COMMIT_PERIOD =
600; // 10 seconds at 60fps
};
class RenderSourceNLRS;
@ -316,8 +283,7 @@ class NativeLayerCA : public NativeLayer {
UpdateType HasUpdate(WhichRepresentation aRepresentation);
bool WillUpdateAffectLayers(WhichRepresentation aRepresentation);
SpecializeType ApplyChanges(WhichRepresentation aRepresentation,
UpdateType aUpdate);
bool ApplyChanges(WhichRepresentation aRepresentation, UpdateType aUpdate);
void SetBackingScale(float aBackingScale);
@ -352,8 +318,7 @@ class NativeLayerCA : public NativeLayer {
bool IsVideo();
bool IsVideoAndLocked(const MutexAutoLock& aProofOfLock);
SpecializeType CanSpecializeVideo(const MutexAutoLock& aProofOfLock);
bool ShouldSpecializeVideo(const MutexAutoLock& aProofOfLock);
// Wraps one CALayer representation of this NativeLayer.
struct Representation {
@ -373,14 +338,14 @@ class NativeLayerCA : public NativeLayer {
// a partial update, the return value will indicate if all the needed
// changes were able to be applied under these restrictions. A false return
// value indicates an All update is necessary.
SpecializeType ApplyChanges(UpdateType aUpdate, const gfx::IntSize& aSize,
bool ApplyChanges(UpdateType aUpdate, const gfx::IntSize& aSize,
bool aIsOpaque, const gfx::IntPoint& aPosition,
const gfx::Matrix4x4& aTransform,
const gfx::IntRect& aDisplayRect,
const Maybe<gfx::IntRect>& aClipRect,
float aBackingScale, bool aSurfaceIsFlipped,
const Maybe<gfx::IntRect>& aClipRect, float aBackingScale,
bool aSurfaceIsFlipped,
gfx::SamplingFilter aSamplingFilter,
SpecializeType aSpecializeVideo,
bool aSpecializeVideo,
CFTypeRefPtr<IOSurfaceRef> aFrontSurface);
// Return whether any aspects of this layer representation have been mutated
@ -484,7 +449,7 @@ class NativeLayerCA : public NativeLayer {
bool mSurfaceIsFlipped = false;
const bool mIsOpaque = false;
bool mRootWindowIsFullscreen = false;
SpecializeType mSpecializeVideo = SpecializeType::None;
bool mSpecializeVideo = false;
};
} // namespace layers

View File

@ -26,7 +26,6 @@
#include "mozilla/layers/ScreenshotGrabber.h"
#include "mozilla/layers/SurfacePoolCA.h"
#include "mozilla/StaticPrefs_gfx.h"
#include "mozilla/Telemetry.h"
#include "mozilla/webrender/RenderMacIOSurfaceTextureHost.h"
#include "nsCocoaFeatures.h"
#include "ScopedGLHelpers.h"
@ -48,58 +47,6 @@ using gfx::SurfaceFormat;
using gl::GLContext;
using gl::GLContextCGL;
/* static */ bool IsSpecializeFail(SpecializeType aSpecialize) {
return (aSpecialize != SpecializeType::None && aSpecialize != SpecializeType::Active);
}
/* static */ Maybe<Telemetry::LABELS_GFX_MACOS_VIDEO_LOW_POWER> SpecializeTypeToTelemetryType(
SpecializeType aSpecialize) {
switch (aSpecialize) {
case SpecializeType::Active:
return Some(Telemetry::LABELS_GFX_MACOS_VIDEO_LOW_POWER::Active);
case SpecializeType::FailPref:
return Some(Telemetry::LABELS_GFX_MACOS_VIDEO_LOW_POWER::FailPref);
case SpecializeType::Fail10_13:
return Some(Telemetry::LABELS_GFX_MACOS_VIDEO_LOW_POWER::Fail10_13);
case SpecializeType::FailFullscreen:
return Some(Telemetry::LABELS_GFX_MACOS_VIDEO_LOW_POWER::FailFullscreen);
case SpecializeType::FailIsoMouse:
return Some(Telemetry::LABELS_GFX_MACOS_VIDEO_LOW_POWER::FailIsoMouse);
case SpecializeType::FailIsoTopVideo:
return Some(Telemetry::LABELS_GFX_MACOS_VIDEO_LOW_POWER::FailIsoTopVideo);
case SpecializeType::FailIsoSize:
return Some(Telemetry::LABELS_GFX_MACOS_VIDEO_LOW_POWER::FailIsoSize);
case SpecializeType::FailIsoCenter:
return Some(Telemetry::LABELS_GFX_MACOS_VIDEO_LOW_POWER::FailIsoCenter);
case SpecializeType::FailIsoOneVideo:
return Some(Telemetry::LABELS_GFX_MACOS_VIDEO_LOW_POWER::FailIsoOneVideo);
case SpecializeType::FailSurface:
return Some(Telemetry::LABELS_GFX_MACOS_VIDEO_LOW_POWER::FailSurface);
case SpecializeType::FailEnqueue:
return Some(Telemetry::LABELS_GFX_MACOS_VIDEO_LOW_POWER::FailEnqueue);
default:
return Nothing();
}
}
/* static */ void EmitTelemetryForSpecialize(SpecializeType aSpecialize) {
auto telemetryValue = SpecializeTypeToTelemetryType(aSpecialize);
if (telemetryValue.isSome()) {
Telemetry::AccumulateCategorical(telemetryValue.value());
}
}
// Utility classes for NativeLayerRootSnapshotter (NLRS) profiler screenshots.
class RenderSourceNLRS : public profiler_screenshots::RenderSource {
@ -187,8 +134,9 @@ static CALayer* MakeOffscreenRootCALayer() {
NativeLayerRootCA::NativeLayerRootCA(CALayer* aLayer)
: mMutex("NativeLayerRootCA"),
mOnscreenRepresentation(aLayer),
mOffscreenRepresentation(MakeOffscreenRootCALayer()),
mLastMouseMoveTime(TimeStamp::NowLoRes()) {}
mOffscreenRepresentation(MakeOffscreenRootCALayer()) {
mLastMouseMoveTime = TimeStamp::NowLoRes();
}
NativeLayerRootCA::~NativeLayerRootCA() {
MOZ_RELEASE_ASSERT(mSublayers.IsEmpty(),
@ -293,14 +241,9 @@ bool NativeLayerRootCA::CommitToScreen() {
}
UpdateMouseMovedRecently(lock);
SpecializeType specialize = mOnscreenRepresentation.Commit(
WhichRepresentation::ONSCREEN, mSublayers, mWindowIsFullscreen, mMouseMovedRecently);
mOnscreenRepresentation.Commit(WhichRepresentation::ONSCREEN, mSublayers, mWindowIsFullscreen,
mMouseMovedRecently);
// Decide if we are going to emit telemetry about video specialization on this commit.
mTelemetryCommitCount = (mTelemetryCommitCount + 1) % TELEMETRY_COMMIT_PERIOD;
if (mTelemetryCommitCount == 0) {
EmitTelemetryForSpecialize(specialize);
}
mCommitPending = false;
}
@ -371,13 +314,11 @@ NativeLayerRootCA::Representation::~Representation() {
[mRootCALayer release];
}
SpecializeType NativeLayerRootCA::Representation::Commit(
WhichRepresentation aRepresentation, const nsTArray<RefPtr<NativeLayerCA>>& aSublayers,
void NativeLayerRootCA::Representation::Commit(WhichRepresentation aRepresentation,
const nsTArray<RefPtr<NativeLayerCA>>& aSublayers,
bool aWindowIsFullscreen, bool aMouseMovedRecently) {
SpecializeType specialize = SpecializeType::None;
bool mightIsolate = (aRepresentation == WhichRepresentation::ONSCREEN &&
StaticPrefs::gfx_core_animation_specialize_video() && aWindowIsFullscreen);
StaticPrefs::gfx_core_animation_specialize_video());
bool mustRebuild = (mMutatedLayerStructure || (mightIsolate && mMutatedMouseMovedRecently));
if (!mustRebuild) {
// Check which type of update we need to do, if any.
@ -393,23 +334,18 @@ SpecializeType NativeLayerRootCA::Representation::Commit(
if (updateRequired == NativeLayerCA::UpdateType::None) {
// Nothing more needed, so early exit.
return specialize;
return;
}
if (updateRequired == NativeLayerCA::UpdateType::OnlyVideo) {
for (auto layer : aSublayers) {
// Use the ordering of our SpecializeType enums to build a specialize
// state that succeeds as long as it doesn't fail.
specialize = std::max(
specialize, layer->ApplyChanges(aRepresentation, NativeLayerCA::UpdateType::OnlyVideo));
}
bool allUpdatesSucceeded = std::all_of(
aSublayers.begin(), aSublayers.end(), [=](const RefPtr<NativeLayerCA>& layer) {
return layer->ApplyChanges(aRepresentation, NativeLayerCA::UpdateType::OnlyVideo);
});
if (!IsSpecializeFail(specialize)) {
// Nothing more needed, so early exit.
// We've completed an OnlyVideo update, but we don't know if the layers have been
// isolated. We've stored the isolation state in mIsIsolatingVideo, so emit that as
// telemetry.
return mIsIsolatingVideo;
if (allUpdatesSucceeded) {
// Nothing more needed, so early exit;
return;
}
}
}
@ -418,8 +354,7 @@ SpecializeType NativeLayerRootCA::Representation::Commit(
AutoCATransaction transaction;
for (auto layer : aSublayers) {
mustRebuild |= layer->WillUpdateAffectLayers(aRepresentation);
specialize =
std::max(specialize, layer->ApplyChanges(aRepresentation, NativeLayerCA::UpdateType::All));
layer->ApplyChanges(aRepresentation, NativeLayerCA::UpdateType::All);
}
if (mustRebuild) {
@ -469,32 +404,15 @@ SpecializeType NativeLayerRootCA::Representation::Commit(
"The topmost layer must be a child of mRootCALayer.");
bool didIsolate = false;
// If we decided that there's no way we can isolate, then specialize should have
// been assigned a fail value. The logic is !mightIsolate => fail specialize. We
// assert that so the decision of whether or not we attempt isolation can be based
// purely on the specialize value.
MOZ_ASSERT(mightIsolate || IsSpecializeFail(specialize),
"If we know we can't isolate, specialize should be a failure.");
// As long as specialize isn't failing, we can attempt to isolate.
if (!IsSpecializeFail(specialize)) {
CALayer* isolatedLayer = FindVideoLayerToIsolate(aRepresentation, aSublayers, specialize);
if (mightIsolate && aWindowIsFullscreen && !aMouseMovedRecently) {
CALayer* isolatedLayer = FindVideoLayerToIsolate(aRepresentation, aSublayers);
if (isolatedLayer) {
MOZ_ASSERT(!IsSpecializeFail(specialize),
"We chose to isolate, so we shouldn't have a specialize fail.");
if (aMouseMovedRecently) {
// We aren't able to isolate this otherwise specializable video.
specialize = SpecializeType::FailIsoMouse;
} else {
// No matter what happens next, we did choose to isolate.
didIsolate = true;
// We only need to change our sublayers if we weren't already isolating, or
// if the isolatedLayer does not match our current top layer.
if (IsSpecializeFail(mIsIsolatingVideo) ||
isolatedLayer != mRootCALayer.sublayers.lastObject) {
if (!mIsIsolatingVideo || isolatedLayer != mRootCALayer.sublayers.lastObject) {
// Create a full coverage black layer behind the isolated layer.
CGFloat rootWidth = mRootCALayer.bounds.size.width;
CGFloat rootHeight = mRootCALayer.bounds.size.height;
@ -511,32 +429,22 @@ SpecializeType NativeLayerRootCA::Representation::Commit(
}
}
}
}
// If we didn't accept the sublayers earlier, and we decided we couldn't isolate,
// accept them now.
if (topLayerIsRooted && !didIsolate) {
acceptProvidedSublayers();
}
} else if (aMouseMovedRecently && specialize == SpecializeType::Active) {
// We aren't able to isolate this otherwise specializable video.
specialize = SpecializeType::FailIsoMouse;
mIsIsolatingVideo = didIsolate;
}
mMutatedLayerStructure = false;
mMutatedMouseMovedRecently = false;
// Store our specialize value to inform future Commits. We use this to determine if the
// layers were already successfully isolated during an OnlyVideo update, and to determine
// if we are already isolating when we identify an opportunity to do so.
mIsIsolatingVideo = specialize;
return specialize;
}
CALayer* NativeLayerRootCA::Representation::FindVideoLayerToIsolate(
WhichRepresentation aRepresentation, const nsTArray<RefPtr<NativeLayerCA>>& aSublayers,
SpecializeType& aSpecialize) {
WhichRepresentation aRepresentation, const nsTArray<RefPtr<NativeLayerCA>>& aSublayers) {
// Run a heuristic to determine if any one of aSublayers is a video layer that should be
// isolated. These layers are ordered back-to-front. This function will return a candidate
// CALayer if all of the following are true:
@ -553,7 +461,6 @@ CALayer* NativeLayerRootCA::Representation::FindVideoLayerToIsolate(
auto topLayer = aSublayers.LastElement();
if (!topLayer || !topLayer->IsVideo()) {
// FAIL Step 1: the topmost layer is not video.
aSpecialize = SpecializeType::FailIsoTopVideo;
return nil;
}
@ -573,7 +480,6 @@ CALayer* NativeLayerRootCA::Representation::FindVideoLayerToIsolate(
CGFloat candidateArea = candidateBoundsInRoot.size.width * candidateBoundsInRoot.size.height;
if (candidateArea < minimumRootArea) {
// FAIL Step 2: the candidate layer is not big enough.
aSpecialize = SpecializeType::FailIsoSize;
return nil;
}
@ -588,7 +494,6 @@ CALayer* NativeLayerRootCA::Representation::FindVideoLayerToIsolate(
candidateBoundsInRoot.origin.y + (candidateBoundsInRoot.size.height * 0.5));
if (!CGRectContainsPoint(centerZone, candidateCenterInRoot)) {
// FAIL Step 3: the candidate layer is off-center.
aSpecialize = SpecializeType::FailIsoCenter;
return nil;
}
@ -596,7 +501,6 @@ CALayer* NativeLayerRootCA::Representation::FindVideoLayerToIsolate(
for (auto layer : aSublayers) {
if (layer->IsVideo() && layer != topLayer) {
// FAIL Step 4: there are multiple video layers.
aSpecialize = SpecializeType::FailIsoOneVideo;
return nil;
}
}
@ -854,8 +758,8 @@ void NativeLayerCA::AttachExternalImage(wr::RenderTextureHost* aExternalImage) {
mDisplayRect = IntRect(IntPoint{}, mSize);
SpecializeType oldSpecializeVideo = mSpecializeVideo;
mSpecializeVideo = CanSpecializeVideo(lock);
bool oldSpecializeVideo = mSpecializeVideo;
mSpecializeVideo = ShouldSpecializeVideo(lock);
bool changedSpecializeVideo = (mSpecializeVideo != oldSpecializeVideo);
ForAllRepresentations([&](Representation& r) {
@ -876,24 +780,10 @@ bool NativeLayerCA::IsVideoAndLocked(const MutexAutoLock& aProofOfLock) {
return mTextureHost;
}
SpecializeType NativeLayerCA::CanSpecializeVideo(const MutexAutoLock& aProofOfLock) {
if (!IsVideoAndLocked(aProofOfLock)) {
return SpecializeType::FailNotVideo;
}
if (!StaticPrefs::gfx_core_animation_specialize_video()) {
return SpecializeType::FailPref;
}
if (!nsCocoaFeatures::OnHighSierraOrLater()) {
return SpecializeType::Fail10_13;
}
if (!mRootWindowIsFullscreen) {
return SpecializeType::FailFullscreen;
}
return SpecializeType::None;
bool NativeLayerCA::ShouldSpecializeVideo(const MutexAutoLock& aProofOfLock) {
return StaticPrefs::gfx_core_animation_specialize_video() &&
nsCocoaFeatures::OnHighSierraOrLater() && mRootWindowIsFullscreen &&
IsVideoAndLocked(aProofOfLock);
}
void NativeLayerCA::SetRootWindowIsFullscreen(bool aFullscreen) {
@ -901,8 +791,8 @@ void NativeLayerCA::SetRootWindowIsFullscreen(bool aFullscreen) {
mRootWindowIsFullscreen = aFullscreen;
SpecializeType oldSpecializeVideo = mSpecializeVideo;
mSpecializeVideo = CanSpecializeVideo(lock);
bool oldSpecializeVideo = mSpecializeVideo;
mSpecializeVideo = ShouldSpecializeVideo(lock);
if (mSpecializeVideo != oldSpecializeVideo) {
ForAllRepresentations([&](Representation& r) { r.mMutatedSpecializeVideo = true; });
@ -1314,7 +1204,7 @@ NativeLayerCA::UpdateType NativeLayerCA::HasUpdate(WhichRepresentation aRepresen
return GetRepresentation(aRepresentation).HasUpdate(IsVideoAndLocked(lock));
}
SpecializeType NativeLayerCA::ApplyChanges(WhichRepresentation aRepresentation,
bool NativeLayerCA::ApplyChanges(WhichRepresentation aRepresentation,
NativeLayerCA::UpdateType aUpdate) {
MutexAutoLock lock(mMutex);
CFTypeRefPtr<IOSurfaceRef> surface;
@ -1384,44 +1274,38 @@ bool NativeLayerCA::Representation::EnqueueSurface(IOSurfaceRef aSurfaceRef) {
return true;
}
SpecializeType NativeLayerCA::Representation::ApplyChanges(
bool NativeLayerCA::Representation::ApplyChanges(
NativeLayerCA::UpdateType aUpdate, const IntSize& aSize, bool aIsOpaque,
const IntPoint& aPosition, const Matrix4x4& aTransform, const IntRect& aDisplayRect,
const Maybe<IntRect>& aClipRect, float aBackingScale, bool aSurfaceIsFlipped,
gfx::SamplingFilter aSamplingFilter, SpecializeType aSpecializeVideo,
gfx::SamplingFilter aSamplingFilter, bool aSpecializeVideo,
CFTypeRefPtr<IOSurfaceRef> aFrontSurface) {
SpecializeType specialize = aSpecializeVideo;
bool tryToSpecialize = !IsSpecializeFail(aSpecializeVideo);
// If we have an OnlyVideo update, handle it and early exit.
if (aUpdate == UpdateType::OnlyVideo) {
// If we don't have any updates to do, exit early with success. This is
// important to do so that the overall OnlyVideo pass will succeed as long
// as the video layers are successful.
if (HasUpdate(true) == UpdateType::None) {
return specialize;
return true;
}
MOZ_ASSERT(!mMutatedSpecializeVideo && mMutatedFrontSurface,
"Shouldn't attempt a OnlyVideo update in this case.");
if (tryToSpecialize) {
bool updateSucceeded = false;
if (aSpecializeVideo) {
IOSurfaceRef surface = aFrontSurface.get();
if (CanSpecializeSurface(surface)) {
IOSurfaceRef surface = aFrontSurface.get();
bool isEnqueued = EnqueueSurface(surface);
if (isEnqueued) {
updateSucceeded = EnqueueSurface(surface);
if (updateSucceeded) {
mMutatedFrontSurface = false;
specialize = SpecializeType::Active;
} else {
specialize = SpecializeType::FailEnqueue;
}
} else {
specialize = SpecializeType::FailSurface;
}
}
return specialize;
return updateSucceeded;
}
MOZ_ASSERT(aUpdate == UpdateType::All);
@ -1445,7 +1329,7 @@ SpecializeType NativeLayerCA::Representation::ApplyChanges(
mWrappingCALayer.anchorPoint = NSZeroPoint;
mWrappingCALayer.contentsGravity = kCAGravityTopLeft;
mWrappingCALayer.edgeAntialiasingMask = 0;
if (tryToSpecialize) {
if (aSpecializeVideo) {
mContentCALayer = [[AVSampleBufferDisplayLayer layer] retain];
} else {
mContentCALayer = [[CALayer layer] retain];
@ -1566,18 +1450,9 @@ SpecializeType NativeLayerCA::Representation::ApplyChanges(
if (mMutatedFrontSurface) {
bool isEnqueued = false;
IOSurfaceRef surface = aFrontSurface.get();
if (tryToSpecialize) {
if (CanSpecializeSurface(surface)) {
if (aSpecializeVideo && CanSpecializeSurface(surface)) {
// Attempt to enqueue this as a video frame. If we fail, we'll fall back to image case.
isEnqueued = EnqueueSurface(surface);
if (isEnqueued) {
specialize = SpecializeType::Active;
} else {
specialize = SpecializeType::FailEnqueue;
}
} else {
specialize = SpecializeType::FailSurface;
}
}
if (!isEnqueued) {
@ -1606,7 +1481,7 @@ SpecializeType NativeLayerCA::Representation::ApplyChanges(
mMutatedSamplingFilter = false;
mMutatedSpecializeVideo = false;
return specialize;
return true;
}
NativeLayerCA::UpdateType NativeLayerCA::Representation::HasUpdate(bool aIsVideo) {

View File

@ -15965,17 +15965,6 @@
"description": "The amount of async paint tasks queued to the paint thread during a layer transaction.",
"bug_numbers": [1483245]
},
"GFX_MACOS_VIDEO_LOW_POWER": {
"record_in_processes": ["main"],
"products": ["firefox"],
"alert_emails": ["bwerth@mozilla.com"],
"expires_in_version": "never",
"releaseChannelCollection": "opt-out",
"bug_numbers": [1737682],
"description": "MacOS video low power state achieved when enqueueing a video frame.",
"kind": "categorical",
"labels": ["None", "FailNotVideo", "Active", "FailPref", "Fail10_13", "FailFullscreen", "FailIsoMouse", "FailIsoTopVideo", "FailIsoSize", "FailIsoCenter", "FailIsoOneVideo", "FailSurface", "FailEnqueue"]
},
"PERMISSION_REQUEST_ORIGIN_SCHEME": {
"record_in_processes": ["main"],
"products": ["firefox", "fennec"],