mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-27 14:52:16 +00:00
Bug 1799179 - Add multi-suite test categories to perf selector. r=perftest-reviewers,kshampur
This patch adds 4 new categories of tests to the perf selector: DAMP tests, Talos tests, Graphics, & Media Playback, and Resource Usage tests. Each of these use a different suite (talos), and one of them also makes use of AWSY. Some extra code needed to be added, and the existing setup that we use for categories had to change to handle the case of a category using multiple suites. The Resource Usage category has the most going on in it. A few new fields are added to allow categories to specify/restrict where they can run (platform, variants, and app). Differential Revision: https://phabricator.services.mozilla.com/D162871
This commit is contained in:
Greg Mierzwinski
parent
03c17445f3
commit
bae4637326
@ -2,6 +2,7 @@
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import copy
|
||||
import itertools
|
||||
import os
|
||||
import re
|
||||
@ -165,36 +166,137 @@ class PerfParser(CompareParser):
|
||||
"platforms": ["desktop", "android"],
|
||||
"apps": ["firefox", "geckoview", "fenix"],
|
||||
},
|
||||
"swr": {
|
||||
"query": "'swr",
|
||||
"negation": "!swr",
|
||||
"platforms": ["desktop"],
|
||||
"apps": ["firefox"],
|
||||
},
|
||||
}
|
||||
|
||||
# The tasks field can be used to hardcode tasks to run,
|
||||
# otherwise we run the query selector combined with the platform
|
||||
# and variants queries
|
||||
"""
|
||||
Here you can find the base categories that are defined for the perf
|
||||
selector. The following fields are available:
|
||||
* query: Set the queries to use for each suite you need.
|
||||
* suites: The suites that are needed for this category.
|
||||
* tasks: A hard-coded list of tasks to select.
|
||||
* platforms: The platforms that it can run on.
|
||||
* app-restrictions: A list of apps that the category can run.
|
||||
* variant-restrictions: A list of variants available for each suite.
|
||||
|
||||
Note that setting the App/Variant-Restriction fields should be used to
|
||||
restrict the available apps and variants, not expand them.
|
||||
"""
|
||||
categories = {
|
||||
"Pageload": {
|
||||
"query": "'browsertime 'tp6",
|
||||
"query": {
|
||||
"raptor": ["'browsertime 'tp6"],
|
||||
},
|
||||
"suites": ["raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Pageload (essential)": {
|
||||
"query": "'browsertime 'tp6 'essential",
|
||||
"query": {
|
||||
"raptor": ["'browsertime 'tp6 'essential"],
|
||||
},
|
||||
"suites": ["raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Pageload (live)": {
|
||||
"query": "'browsertime 'tp6 'live",
|
||||
"query": {
|
||||
"raptor": ["'browsertime 'tp6 'live"],
|
||||
},
|
||||
"suites": ["raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Bytecode Cached": {
|
||||
"query": "'browsertime 'bytecode",
|
||||
"query": {
|
||||
"raptor": ["'browsertime 'bytecode"],
|
||||
},
|
||||
"suites": ["raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Responsiveness": {
|
||||
"query": "'browsertime 'responsive",
|
||||
"query": {
|
||||
"raptor": ["'browsertime 'responsive"],
|
||||
},
|
||||
"suites": ["raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Benchmarks": {
|
||||
"query": "'browsertime 'benchmark",
|
||||
"query": {
|
||||
"raptor": ["'browsertime 'benchmark"],
|
||||
},
|
||||
"suites": ["raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
"DAMP (Devtools)": {
|
||||
"query": {
|
||||
"talos": ["'talos 'damp"],
|
||||
},
|
||||
"suites": ["talos"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Talos PerfTests": {
|
||||
"query": {
|
||||
"talos": ["'talos"],
|
||||
},
|
||||
"suites": ["talos"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Resource Usage": {
|
||||
"query": {
|
||||
"talos": ["'talos 'xperf | 'tp5"],
|
||||
"raptor": ["'power 'osx"],
|
||||
"awsy": ["'awsy"],
|
||||
},
|
||||
"suites": ["talos", "raptor", "awsy"],
|
||||
"platform-restrictions": ["desktop"],
|
||||
"variant-restrictions": {
|
||||
"raptor": [],
|
||||
"talos": [],
|
||||
},
|
||||
"app-restrictions": {
|
||||
"raptor": ["firefox"],
|
||||
"talos": ["firefox"],
|
||||
},
|
||||
"tasks": [],
|
||||
},
|
||||
"Graphics, & Media Playback": {
|
||||
"query": {
|
||||
# XXX This might not be an exhaustive list for talos atm
|
||||
"talos": ["'talos 'svgr | 'bcv | 'webgl"],
|
||||
"raptor": ["'browsertime 'youtube-playback"],
|
||||
},
|
||||
"suites": ["talos", "raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
}
|
||||
|
||||
suites = {
|
||||
"raptor": {
|
||||
"apps": list(apps.keys()),
|
||||
"platforms": list(platforms.keys()),
|
||||
"variants": [
|
||||
"no-fission",
|
||||
"live-sites",
|
||||
"profiling",
|
||||
"bytecode-cached",
|
||||
],
|
||||
},
|
||||
"talos": {
|
||||
"apps": ["firefox"],
|
||||
"platforms": ["desktop"],
|
||||
"variants": [
|
||||
"profiling",
|
||||
"swr",
|
||||
],
|
||||
},
|
||||
"awsy": {
|
||||
"apps": ["firefox"],
|
||||
"platforms": ["desktop"],
|
||||
"variants": [],
|
||||
},
|
||||
}
|
||||
|
||||
arguments = [
|
||||
@ -306,28 +408,38 @@ class PerfParser(CompareParser):
|
||||
continue
|
||||
print("Gathering tasks for %s category" % category)
|
||||
|
||||
# Either perform a query to get the tasks (recommended), or
|
||||
# use a hardcoded task list
|
||||
category_tasks = set()
|
||||
if category_info["queries"]:
|
||||
print("Executing queries: %s" % ", ".join(category_info["queries"]))
|
||||
for suite in PerfParser.suites:
|
||||
# Either perform a query to get the tasks (recommended), or
|
||||
# use a hardcoded task list
|
||||
suite_queries = category_info["queries"].get(suite)
|
||||
|
||||
for perf_query in category_info["queries"]:
|
||||
if not category_tasks:
|
||||
# Get all tasks selected with the first query
|
||||
category_tasks |= PerfParser.get_tasks(
|
||||
base_cmd, queries, perf_query, all_tg_tasks
|
||||
)
|
||||
else:
|
||||
# Keep only those tasks that matched in all previous queries
|
||||
category_tasks &= PerfParser.get_tasks(
|
||||
base_cmd, queries, perf_query, category_tasks
|
||||
)
|
||||
category_suite_tasks = set()
|
||||
if suite_queries:
|
||||
print(
|
||||
"Executing %s queries: %s" % (suite, ", ".join(suite_queries))
|
||||
)
|
||||
|
||||
if len(category_tasks) == 0:
|
||||
print("Failed to find any tasks for query: %s" % perf_query)
|
||||
break
|
||||
else:
|
||||
for perf_query in suite_queries:
|
||||
if not category_suite_tasks:
|
||||
# Get all tasks selected with the first query
|
||||
category_suite_tasks |= PerfParser.get_tasks(
|
||||
base_cmd, queries, perf_query, all_tg_tasks
|
||||
)
|
||||
else:
|
||||
# Keep only those tasks that matched in all previous queries
|
||||
category_suite_tasks &= PerfParser.get_tasks(
|
||||
base_cmd, queries, perf_query, category_suite_tasks
|
||||
)
|
||||
|
||||
if len(category_suite_tasks) == 0:
|
||||
print("Failed to find any tasks for query: %s" % perf_query)
|
||||
break
|
||||
|
||||
if category_suite_tasks:
|
||||
category_tasks |= category_suite_tasks
|
||||
|
||||
if category_info["tasks"]:
|
||||
category_tasks = set(category_info["tasks"]) & all_tg_tasks
|
||||
if category_tasks != set(category_info["tasks"]):
|
||||
print(
|
||||
@ -351,6 +463,82 @@ class PerfParser(CompareParser):
|
||||
|
||||
return selected_tasks, selected_categories, queries
|
||||
|
||||
def _accept_variant(suite, category_info, variant):
|
||||
"""Checks if the variant can run in the given suite."""
|
||||
variant_restrictions = PerfParser.suites[suite]["variants"]
|
||||
if (
|
||||
category_info.get("original", {})
|
||||
.get("variant-restrictions", {})
|
||||
.get(suite, None)
|
||||
is not None
|
||||
):
|
||||
# The category itself might have restricted variants
|
||||
variant_restrictions = category_info["original"]["variant-restrictions"][
|
||||
suite
|
||||
]
|
||||
if variant in variant_restrictions:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _accept_variants(suite, category_info, variants):
|
||||
"""Checks if some part of the variant combination can run in the suite."""
|
||||
for variant in variants:
|
||||
if PerfParser._accept_variant(suite, category_info, variant):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _maybe_add_variant_queries(
|
||||
suite,
|
||||
category_info,
|
||||
current_queries,
|
||||
variants,
|
||||
):
|
||||
"""Used to add variant queries to any suite that needs them."""
|
||||
modified_queries = copy.deepcopy(current_queries)
|
||||
if variants is not None:
|
||||
# We might not be able to add all of them to this suite,
|
||||
# so we go through them one by one
|
||||
for variant in variants:
|
||||
if PerfParser._accept_variant(suite, category_info, variant):
|
||||
modified_queries.append(PerfParser.variants[variant]["query"])
|
||||
return modified_queries
|
||||
|
||||
def _accept_category(
|
||||
suite,
|
||||
category_info,
|
||||
platforms=None,
|
||||
app=None,
|
||||
):
|
||||
"""Used for accepting categories based on the suite."""
|
||||
if platforms is not None and not any(
|
||||
platform in PerfParser.suites[suite]["platforms"] for platform in platforms
|
||||
):
|
||||
return False
|
||||
app_restrictions = (
|
||||
PerfParser.suites[suite]["apps"]
|
||||
if category_info.get("app-restrictions", None) is None
|
||||
else category_info["app-restrictions"]
|
||||
)
|
||||
if app is not None and app not in app_restrictions:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _maybe_add_queries(
|
||||
suite,
|
||||
category_info,
|
||||
current_queries,
|
||||
new_queries,
|
||||
platforms=None,
|
||||
app=None,
|
||||
):
|
||||
"""This is a helper method to apply suite-based restrictions."""
|
||||
modified_queries = copy.deepcopy(current_queries)
|
||||
if PerfParser._accept_category(
|
||||
suite, category_info, platforms=platforms, app=app
|
||||
):
|
||||
modified_queries.extend(new_queries)
|
||||
return modified_queries
|
||||
|
||||
def expand_categories(
|
||||
android=False,
|
||||
chrome=False,
|
||||
@ -384,21 +572,25 @@ class PerfParser(CompareParser):
|
||||
# These global queries get applied to all of the categories. They make it
|
||||
# simpler to prevent, for example, chrome tests running in the
|
||||
# "Pageload desktop" category
|
||||
global_queries = []
|
||||
global_queries = {suite: [] for suite in PerfParser.suites}
|
||||
|
||||
# Rather than dealing with these flags below, simply add the
|
||||
# variants related to them here
|
||||
if live_sites:
|
||||
requested_variants.append("live-sites")
|
||||
else:
|
||||
global_queries.append(PerfParser.variants["live-sites"]["negation"])
|
||||
global_queries["raptor"].append(
|
||||
PerfParser.variants["live-sites"]["negation"]
|
||||
)
|
||||
if profile:
|
||||
requested_variants.append("profiling")
|
||||
else:
|
||||
global_queries.append(PerfParser.variants["profiling"]["negation"])
|
||||
global_queries["raptor"].append(
|
||||
PerfParser.variants["profiling"]["negation"]
|
||||
)
|
||||
|
||||
if not chrome:
|
||||
global_queries.append("!chrom")
|
||||
global_queries["raptor"].append("!chrom")
|
||||
|
||||
# Start by expanding the variants the variants to include combinatorial
|
||||
# options, searching for these tasks is "best-effort" and we can't
|
||||
@ -427,17 +619,50 @@ class PerfParser(CompareParser):
|
||||
# Skip android if it wasn't requested
|
||||
continue
|
||||
|
||||
# Skip category if it can't use this platform
|
||||
if (
|
||||
category_info.get("platform-restrictions", None)
|
||||
and platform_type not in category_info["platform-restrictions"]
|
||||
):
|
||||
continue
|
||||
|
||||
if not any(
|
||||
PerfParser._accept_category(
|
||||
suite,
|
||||
category_info,
|
||||
platforms=(
|
||||
[platform_type]
|
||||
+ category_info.get("platforms", {}).get(suite, [])
|
||||
),
|
||||
)
|
||||
for suite in category_info["suites"]
|
||||
):
|
||||
continue
|
||||
|
||||
# The queries field will hold all the queries needed to run
|
||||
# (in any order). Combinations of queries are used to make the
|
||||
# selected tests increasingly more specific.
|
||||
new_category = category + " %s" % platform
|
||||
cur_cat = {
|
||||
"queries": [category_info["query"]]
|
||||
+ [platform_info["query"]]
|
||||
+ global_queries,
|
||||
"queries": {},
|
||||
"tasks": category_info["tasks"],
|
||||
"platform": platform_type,
|
||||
"suites": category_info["suites"],
|
||||
"original": category_info,
|
||||
}
|
||||
|
||||
for suite, query in category_info["query"].items():
|
||||
cur_cat["queries"][suite] = PerfParser._maybe_add_queries(
|
||||
suite,
|
||||
category_info,
|
||||
category_info["query"][suite],
|
||||
[platform_info["query"]] + global_queries[suite],
|
||||
platforms=(
|
||||
[platform_type]
|
||||
+ category_info.get("platforms", {}).get(suite, [])
|
||||
),
|
||||
)
|
||||
|
||||
# If we didn't request apps, add the global category
|
||||
if len(requested_apps) == 0:
|
||||
expanded_categories[new_category] = cur_cat
|
||||
@ -453,12 +678,28 @@ class PerfParser(CompareParser):
|
||||
if platform_type not in app_info["platforms"]:
|
||||
# Ensure this app can run on this platform
|
||||
continue
|
||||
if not any(
|
||||
PerfParser._accept_category(suite, category_info, app=app)
|
||||
for suite in category_info["suites"]
|
||||
):
|
||||
continue
|
||||
|
||||
new_app_category = new_category + " %s" % app
|
||||
expanded_categories[new_app_category] = {
|
||||
"queries": cur_cat["queries"] + [app_info["query"]],
|
||||
"queries": {
|
||||
suite: PerfParser._maybe_add_queries(
|
||||
suite,
|
||||
category_info,
|
||||
query,
|
||||
[app_info["query"]],
|
||||
app=app,
|
||||
)
|
||||
for suite, query in cur_cat["queries"].items()
|
||||
},
|
||||
"tasks": category_info["tasks"],
|
||||
"platform": platform_type,
|
||||
"suites": category_info["suites"],
|
||||
"original": category_info,
|
||||
}
|
||||
|
||||
# Finally, handle expanding the variants. This needs to be done
|
||||
@ -486,39 +727,65 @@ class PerfParser(CompareParser):
|
||||
):
|
||||
runnable = False
|
||||
break
|
||||
if (
|
||||
len(info["suites"]) == 1
|
||||
and variant
|
||||
not in PerfParser.suites[info["suites"][0]]["variants"]
|
||||
):
|
||||
# This section differs from below because we're handling
|
||||
# single-suite categories and a single variant being
|
||||
# non-existent prevents the entire category from existing but
|
||||
# this isn't the case with more than 1 suite.
|
||||
runnable = False
|
||||
break
|
||||
if not runnable:
|
||||
continue
|
||||
|
||||
# Prevent running this variant combination when none of the suites
|
||||
# specified can run it
|
||||
if len(info["suites"]) > 1 and not any(
|
||||
PerfParser._accept_variants(suite, info, variant_combination)
|
||||
for suite in info["suites"]
|
||||
):
|
||||
continue
|
||||
|
||||
# Build the category name, and setup the queries/tasks
|
||||
# that it would use/select
|
||||
new_variant_category = expanded_category + " %s" % "+".join(
|
||||
variant_combination
|
||||
)
|
||||
variant_queries = [
|
||||
v_info["query"]
|
||||
for v, v_info in PerfParser.variants.items()
|
||||
if v in variant_combination
|
||||
]
|
||||
new_categories[new_variant_category] = {
|
||||
"queries": info["queries"] + variant_queries,
|
||||
"queries": {
|
||||
suite: PerfParser._maybe_add_variant_queries(
|
||||
suite,
|
||||
info,
|
||||
suite_queries,
|
||||
variant_combination,
|
||||
)
|
||||
for suite, suite_queries in info["queries"].items()
|
||||
},
|
||||
"tasks": info["tasks"],
|
||||
}
|
||||
|
||||
# Now ensure that the queries for this new category
|
||||
# don't contain negations for the variant which could
|
||||
# come from the global queries
|
||||
new_queries = []
|
||||
for query in new_categories[new_variant_category]["queries"]:
|
||||
if any(
|
||||
[
|
||||
query == PerfParser.variants.get(variant)["negation"]
|
||||
for variant in variant_combination
|
||||
]
|
||||
):
|
||||
# This query is a negation of one of the variants,
|
||||
# exclude it
|
||||
continue
|
||||
new_queries.append(query)
|
||||
new_queries = {}
|
||||
for suite, suite_queries in new_categories[new_variant_category][
|
||||
"queries"
|
||||
].items():
|
||||
for query in suite_queries:
|
||||
if any(
|
||||
[
|
||||
query
|
||||
== PerfParser.variants.get(variant)["negation"]
|
||||
for variant in variant_combination
|
||||
]
|
||||
):
|
||||
# This query is a negation of one of the variants,
|
||||
# exclude it
|
||||
continue
|
||||
new_queries.setdefault(suite, []).append(query)
|
||||
new_categories[new_variant_category]["queries"] = new_queries
|
||||
|
||||
expanded_categories.update(new_categories)
|
||||
|
||||
@ -30,6 +30,130 @@ TASKS = [
|
||||
"test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot",
|
||||
]
|
||||
|
||||
# The TEST_VARIANTS, and TEST_CATEGORIES are used to force
|
||||
# a particular set of categories to show up in testing. Otherwise,
|
||||
# every time someone adds a category, or a variant, we'll need
|
||||
# to redo all the category counts. The platforms, and apps are
|
||||
# not forced because they change infrequently.
|
||||
TEST_VARIANTS = {
|
||||
"no-fission": {
|
||||
"query": "'nofis",
|
||||
"negation": "!nofis",
|
||||
"platforms": ["android"],
|
||||
"apps": ["fenix", "geckoview"],
|
||||
},
|
||||
"bytecode-cached": {
|
||||
"query": "'bytecode",
|
||||
"negation": "!bytecode",
|
||||
"platforms": ["desktop"],
|
||||
"apps": ["firefox"],
|
||||
},
|
||||
"live-sites": {
|
||||
"query": "'live",
|
||||
"negation": "!live",
|
||||
"platforms": ["desktop", "android"],
|
||||
"apps": list(ps.PerfParser.apps.keys()),
|
||||
},
|
||||
"profiling": {
|
||||
"query": "'profil",
|
||||
"negation": "!profil",
|
||||
"platforms": ["desktop", "android"],
|
||||
"apps": ["firefox", "geckoview", "fenix"],
|
||||
},
|
||||
"swr": {
|
||||
"query": "'swr",
|
||||
"negation": "!swr",
|
||||
"platforms": ["desktop"],
|
||||
"apps": ["firefox"],
|
||||
},
|
||||
}
|
||||
|
||||
TEST_CATEGORIES = {
|
||||
"Pageload": {
|
||||
"query": {
|
||||
"raptor": ["'browsertime 'tp6"],
|
||||
},
|
||||
"suites": ["raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Pageload (essential)": {
|
||||
"query": {
|
||||
"raptor": ["'browsertime 'tp6 'essential"],
|
||||
},
|
||||
"suites": ["raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Pageload (live)": {
|
||||
"query": {
|
||||
"raptor": ["'browsertime 'tp6 'live"],
|
||||
},
|
||||
"suites": ["raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Bytecode Cached": {
|
||||
"query": {
|
||||
"raptor": ["'browsertime 'bytecode"],
|
||||
},
|
||||
"suites": ["raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Responsiveness": {
|
||||
"query": {
|
||||
"raptor": ["'browsertime 'responsive"],
|
||||
},
|
||||
"suites": ["raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Benchmarks": {
|
||||
"query": {
|
||||
"raptor": ["'browsertime 'benchmark"],
|
||||
},
|
||||
"suites": ["raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
"DAMP (Devtools)": {
|
||||
"query": {
|
||||
"talos": ["'talos 'damp"],
|
||||
},
|
||||
"suites": ["talos"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Talos PerfTests": {
|
||||
"query": {
|
||||
"talos": ["'talos"],
|
||||
},
|
||||
"suites": ["talos"],
|
||||
"tasks": [],
|
||||
},
|
||||
"Resource Usage": {
|
||||
"query": {
|
||||
"talos": ["'talos 'xperf | 'tp5"],
|
||||
"raptor": ["'power 'osx"],
|
||||
"awsy": ["'awsy"],
|
||||
},
|
||||
"suites": ["talos", "raptor", "awsy"],
|
||||
"platform-restrictions": ["desktop"],
|
||||
"variant-restrictions": {
|
||||
"raptor": [],
|
||||
"talos": [],
|
||||
},
|
||||
"app-restrictions": {
|
||||
"raptor": ["firefox"],
|
||||
"talos": ["firefox"],
|
||||
},
|
||||
"tasks": [],
|
||||
},
|
||||
"Graphics, & Media Playback": {
|
||||
"query": {
|
||||
# XXX This might not be an exhaustive list for talos atm
|
||||
"talos": ["'talos 'svgr | 'bcv | 'webgl"],
|
||||
"raptor": ["'browsertime 'youtube-playback"],
|
||||
},
|
||||
"suites": ["talos", "raptor"],
|
||||
"tasks": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"category_options, expected_counts, unique_categories, missing",
|
||||
@ -39,150 +163,230 @@ TASKS = [
|
||||
# except for when there are requested apps/variants/platforms
|
||||
(
|
||||
{},
|
||||
48,
|
||||
76,
|
||||
{
|
||||
"Benchmarks desktop": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
],
|
||||
"Pageload (live) macosx": [
|
||||
"'browsertime 'tp6 'live",
|
||||
"'osx 'shippable",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
],
|
||||
"Benchmarks desktop": {
|
||||
"raptor": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
]
|
||||
},
|
||||
"Pageload (live) macosx": {
|
||||
"raptor": [
|
||||
"'browsertime 'tp6 'live",
|
||||
"'osx 'shippable",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
]
|
||||
},
|
||||
"Resource Usage linux": {
|
||||
"awsy": ["'awsy", "!clang 'linux 'shippable"],
|
||||
"raptor": [
|
||||
"'power 'osx",
|
||||
"!clang 'linux 'shippable",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
],
|
||||
"talos": ["'talos 'xperf | 'tp5", "!clang 'linux 'shippable"],
|
||||
},
|
||||
},
|
||||
["Responsiveness android-p2 geckoview", "Benchmarks desktop chromium"],
|
||||
[
|
||||
"Responsiveness android-p2 geckoview",
|
||||
"Benchmarks desktop chromium",
|
||||
],
|
||||
), # Default settings
|
||||
(
|
||||
{"live_sites": True},
|
||||
240,
|
||||
332,
|
||||
{
|
||||
"Benchmarks desktop": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
],
|
||||
"Pageload (live) macosx": [
|
||||
"'browsertime 'tp6 'live",
|
||||
"'osx 'shippable",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
],
|
||||
"Benchmarks desktop firefox live-sites+profiling": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!chrom",
|
||||
"!chrom !geckoview !fenix",
|
||||
"'live",
|
||||
"'profil",
|
||||
],
|
||||
"Benchmarks desktop": {
|
||||
"raptor": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
],
|
||||
},
|
||||
"Pageload (live) macosx": {
|
||||
"raptor": [
|
||||
"'browsertime 'tp6 'live",
|
||||
"'osx 'shippable",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
],
|
||||
},
|
||||
"Benchmarks desktop firefox live-sites+profiling": {
|
||||
"raptor": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!chrom",
|
||||
"!chrom !geckoview !fenix",
|
||||
"'live",
|
||||
"'profil",
|
||||
],
|
||||
},
|
||||
"Graphics, & Media Playback desktop live-sites+profiling+swr": {
|
||||
"raptor": [
|
||||
"'browsertime 'youtube-playback",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!chrom",
|
||||
"'live",
|
||||
"'profil",
|
||||
],
|
||||
"talos": [
|
||||
"'talos 'svgr | 'bcv | 'webgl",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"'profil",
|
||||
"'swr",
|
||||
],
|
||||
},
|
||||
},
|
||||
[
|
||||
"Responsiveness android-p2 geckoview",
|
||||
"Benchmarks desktop chromium",
|
||||
"Benchmarks desktop firefox profiling",
|
||||
"Talos desktop live-sites",
|
||||
"Talos desktop profiling+swr",
|
||||
],
|
||||
),
|
||||
(
|
||||
{"live_sites": True, "chrome": True},
|
||||
480,
|
||||
644,
|
||||
{
|
||||
"Benchmarks desktop": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!profil",
|
||||
],
|
||||
"Pageload (live) macosx": [
|
||||
"'browsertime 'tp6 'live",
|
||||
"'osx 'shippable",
|
||||
"!profil",
|
||||
],
|
||||
"Benchmarks desktop chromium": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!profil",
|
||||
"'chromium",
|
||||
],
|
||||
"Benchmarks desktop": {
|
||||
"raptor": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!profil",
|
||||
],
|
||||
},
|
||||
"Pageload (live) macosx": {
|
||||
"raptor": [
|
||||
"'browsertime 'tp6 'live",
|
||||
"'osx 'shippable",
|
||||
"!profil",
|
||||
],
|
||||
},
|
||||
"Benchmarks desktop chromium": {
|
||||
"raptor": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!profil",
|
||||
"'chromium",
|
||||
],
|
||||
},
|
||||
},
|
||||
["Responsiveness android-p2 geckoview"],
|
||||
[
|
||||
"Responsiveness android-p2 geckoview",
|
||||
"Firefox Pageload linux chrome",
|
||||
"Talos PerfTests desktop swr",
|
||||
],
|
||||
),
|
||||
(
|
||||
{"android": True},
|
||||
420,
|
||||
542,
|
||||
{
|
||||
"Benchmarks desktop": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
],
|
||||
"Responsiveness android-a51 geckoview": [
|
||||
"'browsertime 'responsive",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'geckoview",
|
||||
],
|
||||
"Benchmarks desktop": {
|
||||
"raptor": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
],
|
||||
},
|
||||
"Responsiveness android-a51 geckoview": {
|
||||
"raptor": [
|
||||
"'browsertime 'responsive",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'geckoview",
|
||||
],
|
||||
},
|
||||
},
|
||||
["Responsiveness android-a51 chrome-m"],
|
||||
[
|
||||
"Responsiveness android-a51 chrome-m",
|
||||
"Firefox Pageload android",
|
||||
],
|
||||
),
|
||||
(
|
||||
{"android": True, "chrome": True},
|
||||
720,
|
||||
924,
|
||||
{
|
||||
"Benchmarks desktop": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!live",
|
||||
"!profil",
|
||||
],
|
||||
"Responsiveness android-a51 chrome-m": [
|
||||
"'browsertime 'responsive",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!live",
|
||||
"!profil",
|
||||
"'chrome-m",
|
||||
],
|
||||
"Benchmarks desktop": {
|
||||
"raptor": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!live",
|
||||
"!profil",
|
||||
],
|
||||
},
|
||||
"Responsiveness android-a51 chrome-m": {
|
||||
"raptor": [
|
||||
"'browsertime 'responsive",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!live",
|
||||
"!profil",
|
||||
"'chrome-m",
|
||||
],
|
||||
},
|
||||
},
|
||||
["Responsiveness android-p2 chrome-m"],
|
||||
["Responsiveness android-p2 chrome-m", "Resource Usage android"],
|
||||
),
|
||||
(
|
||||
{"android": True, "chrome": True, "profile": True},
|
||||
1008,
|
||||
1324,
|
||||
{
|
||||
"Benchmarks desktop": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!live",
|
||||
]
|
||||
"Benchmarks desktop": {
|
||||
"raptor": [
|
||||
"'browsertime 'benchmark",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"!live",
|
||||
]
|
||||
},
|
||||
"Talos PerfTests desktop profiling+swr": {
|
||||
"talos": [
|
||||
"'talos",
|
||||
"!android 'shippable !-32 !clang",
|
||||
"'profil",
|
||||
"'swr",
|
||||
]
|
||||
},
|
||||
},
|
||||
[],
|
||||
[
|
||||
"Resource Usage desktop profiling",
|
||||
"DAMP (Devtools) desktop chrome",
|
||||
"Resource Usage android",
|
||||
"Resource Usage windows chromium",
|
||||
],
|
||||
),
|
||||
# Show all available windows tests, no other platform should exist
|
||||
# including the desktop catgeory
|
||||
(
|
||||
{"requested_platforms": ["windows"]},
|
||||
84,
|
||||
123,
|
||||
{
|
||||
"Benchmarks windows firefox bytecode-cached+profiling": [
|
||||
"'browsertime 'benchmark",
|
||||
"!-32 'windows 'shippable",
|
||||
"!live",
|
||||
"!chrom",
|
||||
"!chrom !geckoview !fenix",
|
||||
"'bytecode",
|
||||
"'profil",
|
||||
]
|
||||
"Benchmarks windows firefox bytecode-cached+profiling": {
|
||||
"raptor": [
|
||||
"'browsertime 'benchmark",
|
||||
"!-32 'windows 'shippable",
|
||||
"!live",
|
||||
"!chrom",
|
||||
"!chrom !geckoview !fenix",
|
||||
"'bytecode",
|
||||
"'profil",
|
||||
]
|
||||
},
|
||||
},
|
||||
[
|
||||
"Resource Usage desktop",
|
||||
"Benchmarks desktop",
|
||||
"Benchmarks linux firefox bytecode-cached+profiling",
|
||||
],
|
||||
@ -208,17 +412,19 @@ TASKS = [
|
||||
"requested_apps": ["fenix"],
|
||||
"android": True,
|
||||
},
|
||||
42,
|
||||
49,
|
||||
{
|
||||
"Bytecode Cached android fenix no-fission+live-sites+profiling": [
|
||||
"'browsertime 'bytecode",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'nofis",
|
||||
"'live",
|
||||
"'profil",
|
||||
],
|
||||
"Bytecode Cached android fenix no-fission+live-sites+profiling": {
|
||||
"raptor": [
|
||||
"'browsertime 'bytecode",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'nofis",
|
||||
"'live",
|
||||
"'profil",
|
||||
],
|
||||
}
|
||||
},
|
||||
["Benchmarks desktop", "Pageload (live) android"],
|
||||
),
|
||||
@ -229,25 +435,29 @@ TASKS = [
|
||||
"requested_apps": ["fenix", "geckoview"],
|
||||
"android": True,
|
||||
},
|
||||
84,
|
||||
98,
|
||||
{
|
||||
"Benchmarks android geckoview live-sites+profiling": [
|
||||
"'browsertime 'benchmark",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!chrom",
|
||||
"'geckoview",
|
||||
"'live",
|
||||
"'profil",
|
||||
],
|
||||
"Bytecode Cached android fenix no-fission+live-sites+profiling": [
|
||||
"'browsertime 'bytecode",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'nofis",
|
||||
"'live",
|
||||
"'profil",
|
||||
],
|
||||
"Benchmarks android geckoview live-sites+profiling": {
|
||||
"raptor": [
|
||||
"'browsertime 'benchmark",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!chrom",
|
||||
"'geckoview",
|
||||
"'live",
|
||||
"'profil",
|
||||
],
|
||||
},
|
||||
"Bytecode Cached android fenix no-fission+live-sites+profiling": {
|
||||
"raptor": [
|
||||
"'browsertime 'bytecode",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'nofis",
|
||||
"'live",
|
||||
"'profil",
|
||||
],
|
||||
},
|
||||
},
|
||||
[
|
||||
"Benchmarks desktop",
|
||||
@ -263,34 +473,40 @@ TASKS = [
|
||||
"requested_apps": ["fenix"],
|
||||
"android": True,
|
||||
},
|
||||
60,
|
||||
70,
|
||||
{
|
||||
"Pageload android-a51 fenix": [
|
||||
"'browsertime 'tp6",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
],
|
||||
"Pageload android-a51 fenix no-fission": [
|
||||
"'browsertime 'tp6",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'nofis",
|
||||
],
|
||||
"Pageload (essential) android fenix no-fission+live-sites": [
|
||||
"'browsertime 'tp6 'essential",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'nofis",
|
||||
"'live",
|
||||
],
|
||||
"Pageload android-a51 fenix": {
|
||||
"raptor": [
|
||||
"'browsertime 'tp6",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
],
|
||||
},
|
||||
"Pageload android-a51 fenix no-fission": {
|
||||
"raptor": [
|
||||
"'browsertime 'tp6",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'nofis",
|
||||
],
|
||||
},
|
||||
"Pageload (essential) android fenix no-fission+live-sites": {
|
||||
"raptor": [
|
||||
"'browsertime 'tp6 'essential",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'nofis",
|
||||
"'live",
|
||||
],
|
||||
},
|
||||
},
|
||||
[
|
||||
"Benchmarks desktop",
|
||||
@ -306,42 +522,50 @@ TASKS = [
|
||||
"requested_apps": ["fenix"],
|
||||
"android": True,
|
||||
},
|
||||
84,
|
||||
98,
|
||||
{
|
||||
"Pageload android-a51 fenix": [
|
||||
"'browsertime 'tp6",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
],
|
||||
"Pageload android-a51 fenix no-fission": [
|
||||
"'browsertime 'tp6",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'nofis",
|
||||
],
|
||||
"Pageload android-a51 fenix live-sites": [
|
||||
"'browsertime 'tp6",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'live",
|
||||
],
|
||||
"Pageload (essential) android fenix no-fission+live-sites": [
|
||||
"'browsertime 'tp6 'essential",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'nofis",
|
||||
"'live",
|
||||
],
|
||||
"Pageload android-a51 fenix": {
|
||||
"raptor": [
|
||||
"'browsertime 'tp6",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
],
|
||||
},
|
||||
"Pageload android-a51 fenix no-fission": {
|
||||
"raptor": [
|
||||
"'browsertime 'tp6",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'nofis",
|
||||
],
|
||||
},
|
||||
"Pageload android-a51 fenix live-sites": {
|
||||
"raptor": [
|
||||
"'browsertime 'tp6",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'live",
|
||||
],
|
||||
},
|
||||
"Pageload (essential) android fenix no-fission+live-sites": {
|
||||
"raptor": [
|
||||
"'browsertime 'tp6 'essential",
|
||||
"'android 'a51 'shippable 'aarch64",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"'fenix",
|
||||
"'nofis",
|
||||
"'live",
|
||||
],
|
||||
},
|
||||
},
|
||||
[
|
||||
"Benchmarks desktop",
|
||||
@ -356,16 +580,18 @@ TASKS = [
|
||||
"requested_variants": ["no-fission"],
|
||||
"requested_platforms": ["windows"],
|
||||
},
|
||||
12,
|
||||
19,
|
||||
{
|
||||
"Responsiveness windows firefox": [
|
||||
"'browsertime 'responsive",
|
||||
"!-32 'windows 'shippable",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"!chrom !geckoview !fenix",
|
||||
],
|
||||
"Responsiveness windows firefox": {
|
||||
"raptor": [
|
||||
"'browsertime 'responsive",
|
||||
"!-32 'windows 'shippable",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"!chrom !geckoview !fenix",
|
||||
],
|
||||
},
|
||||
},
|
||||
["Benchmarks desktop", "Responsiveness windows firefox no-fisson"],
|
||||
),
|
||||
@ -376,29 +602,50 @@ TASKS = [
|
||||
"requested_platforms": ["windows"],
|
||||
"android": True,
|
||||
},
|
||||
60,
|
||||
83,
|
||||
{
|
||||
"Responsiveness windows firefox": [
|
||||
"'browsertime 'responsive",
|
||||
"!-32 'windows 'shippable",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"!chrom !geckoview !fenix",
|
||||
],
|
||||
"Responsiveness windows firefox live-sites": [
|
||||
"'browsertime 'responsive",
|
||||
"!-32 'windows 'shippable",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"!chrom !geckoview !fenix",
|
||||
"'live",
|
||||
],
|
||||
"Responsiveness windows firefox": {
|
||||
"raptor": [
|
||||
"'browsertime 'responsive",
|
||||
"!-32 'windows 'shippable",
|
||||
"!live",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"!chrom !geckoview !fenix",
|
||||
],
|
||||
},
|
||||
"Responsiveness windows firefox live-sites": {
|
||||
"raptor": [
|
||||
"'browsertime 'responsive",
|
||||
"!-32 'windows 'shippable",
|
||||
"!profil",
|
||||
"!chrom",
|
||||
"!chrom !geckoview !fenix",
|
||||
"'live",
|
||||
],
|
||||
},
|
||||
"Graphics, & Media Playback windows live-sites+profiling+swr": {
|
||||
"raptor": [
|
||||
"'browsertime 'youtube-playback",
|
||||
"!-32 'windows 'shippable",
|
||||
"!chrom",
|
||||
"'live",
|
||||
"'profil",
|
||||
],
|
||||
"talos": [
|
||||
"'talos 'svgr | 'bcv | 'webgl",
|
||||
"!-32 'windows 'shippable",
|
||||
"'profil",
|
||||
"'swr",
|
||||
],
|
||||
},
|
||||
},
|
||||
[
|
||||
"Benchmarks desktop",
|
||||
"Responsiveness windows firefox no-fisson",
|
||||
"Pageload (live) android",
|
||||
"Talos desktop live-sites",
|
||||
"Talos android",
|
||||
],
|
||||
),
|
||||
],
|
||||
@ -406,6 +653,10 @@ TASKS = [
|
||||
def test_category_expansion(
|
||||
category_options, expected_counts, unique_categories, missing
|
||||
):
|
||||
# Set the categories, and variants to expand
|
||||
ps.PerfParser.categories = TEST_CATEGORIES
|
||||
ps.PerfParser.variants = TEST_VARIANTS
|
||||
|
||||
# Expand the categories, then either check if the unique_categories,
|
||||
# exist or are missing from the categories
|
||||
expanded_cats = ps.PerfParser.expand_categories(**category_options)
|
||||
@ -480,6 +731,7 @@ def test_full_run(options, call_counts, log_ind, expected_log_message):
|
||||
["", TASKS],
|
||||
["", TASKS],
|
||||
["", TASKS],
|
||||
["", TASKS],
|
||||
]
|
||||
|
||||
ps.run(**options)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user