diff --git a/tools/tryselect/selectors/perf.py b/tools/tryselect/selectors/perf.py index e488d6e01942..e2713441d8ef 100644 --- a/tools/tryselect/selectors/perf.py +++ b/tools/tryselect/selectors/perf.py @@ -2,6 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import copy import itertools import os import re @@ -165,36 +166,137 @@ class PerfParser(CompareParser): "platforms": ["desktop", "android"], "apps": ["firefox", "geckoview", "fenix"], }, + "swr": { + "query": "'swr", + "negation": "!swr", + "platforms": ["desktop"], + "apps": ["firefox"], + }, } - # The tasks field can be used to hardcode tasks to run, - # otherwise we run the query selector combined with the platform - # and variants queries + """ + Here you can find the base categories that are defined for the perf + selector. The following fields are available: + * query: Set the queries to use for each suite you need. + * suites: The suites that are needed for this category. + * tasks: A hard-coded list of tasks to select. + * platforms: The platforms that it can run on. + * app-restrictions: A list of apps that the category can run. + * variant-restrictions: A list of variants available for each suite. + + Note that setting the App/Variant-Restriction fields should be used to + restrict the available apps and variants, not expand them. + """ categories = { "Pageload": { - "query": "'browsertime 'tp6", + "query": { + "raptor": ["'browsertime 'tp6"], + }, + "suites": ["raptor"], "tasks": [], }, "Pageload (essential)": { - "query": "'browsertime 'tp6 'essential", + "query": { + "raptor": ["'browsertime 'tp6 'essential"], + }, + "suites": ["raptor"], "tasks": [], }, "Pageload (live)": { - "query": "'browsertime 'tp6 'live", + "query": { + "raptor": ["'browsertime 'tp6 'live"], + }, + "suites": ["raptor"], "tasks": [], }, "Bytecode Cached": { - "query": "'browsertime 'bytecode", + "query": { + "raptor": ["'browsertime 'bytecode"], + }, + "suites": ["raptor"], "tasks": [], }, "Responsiveness": { - "query": "'browsertime 'responsive", + "query": { + "raptor": ["'browsertime 'responsive"], + }, + "suites": ["raptor"], "tasks": [], }, "Benchmarks": { - "query": "'browsertime 'benchmark", + "query": { + "raptor": ["'browsertime 'benchmark"], + }, + "suites": ["raptor"], "tasks": [], }, + "DAMP (Devtools)": { + "query": { + "talos": ["'talos 'damp"], + }, + "suites": ["talos"], + "tasks": [], + }, + "Talos PerfTests": { + "query": { + "talos": ["'talos"], + }, + "suites": ["talos"], + "tasks": [], + }, + "Resource Usage": { + "query": { + "talos": ["'talos 'xperf | 'tp5"], + "raptor": ["'power 'osx"], + "awsy": ["'awsy"], + }, + "suites": ["talos", "raptor", "awsy"], + "platform-restrictions": ["desktop"], + "variant-restrictions": { + "raptor": [], + "talos": [], + }, + "app-restrictions": { + "raptor": ["firefox"], + "talos": ["firefox"], + }, + "tasks": [], + }, + "Graphics, & Media Playback": { + "query": { + # XXX This might not be an exhaustive list for talos atm + "talos": ["'talos 'svgr | 'bcv | 'webgl"], + "raptor": ["'browsertime 'youtube-playback"], + }, + "suites": ["talos", "raptor"], + "tasks": [], + }, + } + + suites = { + "raptor": { + "apps": list(apps.keys()), + "platforms": list(platforms.keys()), + "variants": [ + "no-fission", + "live-sites", + "profiling", + "bytecode-cached", + ], + }, + "talos": { + "apps": ["firefox"], + "platforms": ["desktop"], + "variants": [ + "profiling", + "swr", + ], + }, + "awsy": { + "apps": ["firefox"], + "platforms": ["desktop"], + "variants": [], + }, } arguments = [ @@ -306,28 +408,38 @@ class PerfParser(CompareParser): continue print("Gathering tasks for %s category" % category) - # Either perform a query to get the tasks (recommended), or - # use a hardcoded task list category_tasks = set() - if category_info["queries"]: - print("Executing queries: %s" % ", ".join(category_info["queries"])) + for suite in PerfParser.suites: + # Either perform a query to get the tasks (recommended), or + # use a hardcoded task list + suite_queries = category_info["queries"].get(suite) - for perf_query in category_info["queries"]: - if not category_tasks: - # Get all tasks selected with the first query - category_tasks |= PerfParser.get_tasks( - base_cmd, queries, perf_query, all_tg_tasks - ) - else: - # Keep only those tasks that matched in all previous queries - category_tasks &= PerfParser.get_tasks( - base_cmd, queries, perf_query, category_tasks - ) + category_suite_tasks = set() + if suite_queries: + print( + "Executing %s queries: %s" % (suite, ", ".join(suite_queries)) + ) - if len(category_tasks) == 0: - print("Failed to find any tasks for query: %s" % perf_query) - break - else: + for perf_query in suite_queries: + if not category_suite_tasks: + # Get all tasks selected with the first query + category_suite_tasks |= PerfParser.get_tasks( + base_cmd, queries, perf_query, all_tg_tasks + ) + else: + # Keep only those tasks that matched in all previous queries + category_suite_tasks &= PerfParser.get_tasks( + base_cmd, queries, perf_query, category_suite_tasks + ) + + if len(category_suite_tasks) == 0: + print("Failed to find any tasks for query: %s" % perf_query) + break + + if category_suite_tasks: + category_tasks |= category_suite_tasks + + if category_info["tasks"]: category_tasks = set(category_info["tasks"]) & all_tg_tasks if category_tasks != set(category_info["tasks"]): print( @@ -351,6 +463,82 @@ class PerfParser(CompareParser): return selected_tasks, selected_categories, queries + def _accept_variant(suite, category_info, variant): + """Checks if the variant can run in the given suite.""" + variant_restrictions = PerfParser.suites[suite]["variants"] + if ( + category_info.get("original", {}) + .get("variant-restrictions", {}) + .get(suite, None) + is not None + ): + # The category itself might have restricted variants + variant_restrictions = category_info["original"]["variant-restrictions"][ + suite + ] + if variant in variant_restrictions: + return True + return False + + def _accept_variants(suite, category_info, variants): + """Checks if some part of the variant combination can run in the suite.""" + for variant in variants: + if PerfParser._accept_variant(suite, category_info, variant): + return True + return False + + def _maybe_add_variant_queries( + suite, + category_info, + current_queries, + variants, + ): + """Used to add variant queries to any suite that needs them.""" + modified_queries = copy.deepcopy(current_queries) + if variants is not None: + # We might not be able to add all of them to this suite, + # so we go through them one by one + for variant in variants: + if PerfParser._accept_variant(suite, category_info, variant): + modified_queries.append(PerfParser.variants[variant]["query"]) + return modified_queries + + def _accept_category( + suite, + category_info, + platforms=None, + app=None, + ): + """Used for accepting categories based on the suite.""" + if platforms is not None and not any( + platform in PerfParser.suites[suite]["platforms"] for platform in platforms + ): + return False + app_restrictions = ( + PerfParser.suites[suite]["apps"] + if category_info.get("app-restrictions", None) is None + else category_info["app-restrictions"] + ) + if app is not None and app not in app_restrictions: + return False + return True + + def _maybe_add_queries( + suite, + category_info, + current_queries, + new_queries, + platforms=None, + app=None, + ): + """This is a helper method to apply suite-based restrictions.""" + modified_queries = copy.deepcopy(current_queries) + if PerfParser._accept_category( + suite, category_info, platforms=platforms, app=app + ): + modified_queries.extend(new_queries) + return modified_queries + def expand_categories( android=False, chrome=False, @@ -384,21 +572,25 @@ class PerfParser(CompareParser): # These global queries get applied to all of the categories. They make it # simpler to prevent, for example, chrome tests running in the # "Pageload desktop" category - global_queries = [] + global_queries = {suite: [] for suite in PerfParser.suites} # Rather than dealing with these flags below, simply add the # variants related to them here if live_sites: requested_variants.append("live-sites") else: - global_queries.append(PerfParser.variants["live-sites"]["negation"]) + global_queries["raptor"].append( + PerfParser.variants["live-sites"]["negation"] + ) if profile: requested_variants.append("profiling") else: - global_queries.append(PerfParser.variants["profiling"]["negation"]) + global_queries["raptor"].append( + PerfParser.variants["profiling"]["negation"] + ) if not chrome: - global_queries.append("!chrom") + global_queries["raptor"].append("!chrom") # Start by expanding the variants the variants to include combinatorial # options, searching for these tasks is "best-effort" and we can't @@ -427,17 +619,50 @@ class PerfParser(CompareParser): # Skip android if it wasn't requested continue + # Skip category if it can't use this platform + if ( + category_info.get("platform-restrictions", None) + and platform_type not in category_info["platform-restrictions"] + ): + continue + + if not any( + PerfParser._accept_category( + suite, + category_info, + platforms=( + [platform_type] + + category_info.get("platforms", {}).get(suite, []) + ), + ) + for suite in category_info["suites"] + ): + continue + # The queries field will hold all the queries needed to run # (in any order). Combinations of queries are used to make the # selected tests increasingly more specific. new_category = category + " %s" % platform cur_cat = { - "queries": [category_info["query"]] - + [platform_info["query"]] - + global_queries, + "queries": {}, "tasks": category_info["tasks"], "platform": platform_type, + "suites": category_info["suites"], + "original": category_info, } + + for suite, query in category_info["query"].items(): + cur_cat["queries"][suite] = PerfParser._maybe_add_queries( + suite, + category_info, + category_info["query"][suite], + [platform_info["query"]] + global_queries[suite], + platforms=( + [platform_type] + + category_info.get("platforms", {}).get(suite, []) + ), + ) + # If we didn't request apps, add the global category if len(requested_apps) == 0: expanded_categories[new_category] = cur_cat @@ -453,12 +678,28 @@ class PerfParser(CompareParser): if platform_type not in app_info["platforms"]: # Ensure this app can run on this platform continue + if not any( + PerfParser._accept_category(suite, category_info, app=app) + for suite in category_info["suites"] + ): + continue new_app_category = new_category + " %s" % app expanded_categories[new_app_category] = { - "queries": cur_cat["queries"] + [app_info["query"]], + "queries": { + suite: PerfParser._maybe_add_queries( + suite, + category_info, + query, + [app_info["query"]], + app=app, + ) + for suite, query in cur_cat["queries"].items() + }, "tasks": category_info["tasks"], "platform": platform_type, + "suites": category_info["suites"], + "original": category_info, } # Finally, handle expanding the variants. This needs to be done @@ -486,39 +727,65 @@ class PerfParser(CompareParser): ): runnable = False break + if ( + len(info["suites"]) == 1 + and variant + not in PerfParser.suites[info["suites"][0]]["variants"] + ): + # This section differs from below because we're handling + # single-suite categories and a single variant being + # non-existent prevents the entire category from existing but + # this isn't the case with more than 1 suite. + runnable = False + break if not runnable: continue + # Prevent running this variant combination when none of the suites + # specified can run it + if len(info["suites"]) > 1 and not any( + PerfParser._accept_variants(suite, info, variant_combination) + for suite in info["suites"] + ): + continue + # Build the category name, and setup the queries/tasks # that it would use/select new_variant_category = expanded_category + " %s" % "+".join( variant_combination ) - variant_queries = [ - v_info["query"] - for v, v_info in PerfParser.variants.items() - if v in variant_combination - ] new_categories[new_variant_category] = { - "queries": info["queries"] + variant_queries, + "queries": { + suite: PerfParser._maybe_add_variant_queries( + suite, + info, + suite_queries, + variant_combination, + ) + for suite, suite_queries in info["queries"].items() + }, "tasks": info["tasks"], } # Now ensure that the queries for this new category # don't contain negations for the variant which could # come from the global queries - new_queries = [] - for query in new_categories[new_variant_category]["queries"]: - if any( - [ - query == PerfParser.variants.get(variant)["negation"] - for variant in variant_combination - ] - ): - # This query is a negation of one of the variants, - # exclude it - continue - new_queries.append(query) + new_queries = {} + for suite, suite_queries in new_categories[new_variant_category][ + "queries" + ].items(): + for query in suite_queries: + if any( + [ + query + == PerfParser.variants.get(variant)["negation"] + for variant in variant_combination + ] + ): + # This query is a negation of one of the variants, + # exclude it + continue + new_queries.setdefault(suite, []).append(query) new_categories[new_variant_category]["queries"] = new_queries expanded_categories.update(new_categories) diff --git a/tools/tryselect/test/test_perf.py b/tools/tryselect/test/test_perf.py index ee49914f18ba..87b589faccc4 100644 --- a/tools/tryselect/test/test_perf.py +++ b/tools/tryselect/test/test_perf.py @@ -30,6 +30,130 @@ TASKS = [ "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot", ] +# The TEST_VARIANTS, and TEST_CATEGORIES are used to force +# a particular set of categories to show up in testing. Otherwise, +# every time someone adds a category, or a variant, we'll need +# to redo all the category counts. The platforms, and apps are +# not forced because they change infrequently. +TEST_VARIANTS = { + "no-fission": { + "query": "'nofis", + "negation": "!nofis", + "platforms": ["android"], + "apps": ["fenix", "geckoview"], + }, + "bytecode-cached": { + "query": "'bytecode", + "negation": "!bytecode", + "platforms": ["desktop"], + "apps": ["firefox"], + }, + "live-sites": { + "query": "'live", + "negation": "!live", + "platforms": ["desktop", "android"], + "apps": list(ps.PerfParser.apps.keys()), + }, + "profiling": { + "query": "'profil", + "negation": "!profil", + "platforms": ["desktop", "android"], + "apps": ["firefox", "geckoview", "fenix"], + }, + "swr": { + "query": "'swr", + "negation": "!swr", + "platforms": ["desktop"], + "apps": ["firefox"], + }, +} + +TEST_CATEGORIES = { + "Pageload": { + "query": { + "raptor": ["'browsertime 'tp6"], + }, + "suites": ["raptor"], + "tasks": [], + }, + "Pageload (essential)": { + "query": { + "raptor": ["'browsertime 'tp6 'essential"], + }, + "suites": ["raptor"], + "tasks": [], + }, + "Pageload (live)": { + "query": { + "raptor": ["'browsertime 'tp6 'live"], + }, + "suites": ["raptor"], + "tasks": [], + }, + "Bytecode Cached": { + "query": { + "raptor": ["'browsertime 'bytecode"], + }, + "suites": ["raptor"], + "tasks": [], + }, + "Responsiveness": { + "query": { + "raptor": ["'browsertime 'responsive"], + }, + "suites": ["raptor"], + "tasks": [], + }, + "Benchmarks": { + "query": { + "raptor": ["'browsertime 'benchmark"], + }, + "suites": ["raptor"], + "tasks": [], + }, + "DAMP (Devtools)": { + "query": { + "talos": ["'talos 'damp"], + }, + "suites": ["talos"], + "tasks": [], + }, + "Talos PerfTests": { + "query": { + "talos": ["'talos"], + }, + "suites": ["talos"], + "tasks": [], + }, + "Resource Usage": { + "query": { + "talos": ["'talos 'xperf | 'tp5"], + "raptor": ["'power 'osx"], + "awsy": ["'awsy"], + }, + "suites": ["talos", "raptor", "awsy"], + "platform-restrictions": ["desktop"], + "variant-restrictions": { + "raptor": [], + "talos": [], + }, + "app-restrictions": { + "raptor": ["firefox"], + "talos": ["firefox"], + }, + "tasks": [], + }, + "Graphics, & Media Playback": { + "query": { + # XXX This might not be an exhaustive list for talos atm + "talos": ["'talos 'svgr | 'bcv | 'webgl"], + "raptor": ["'browsertime 'youtube-playback"], + }, + "suites": ["talos", "raptor"], + "tasks": [], + }, +} + @pytest.mark.parametrize( "category_options, expected_counts, unique_categories, missing", @@ -39,150 +163,230 @@ TASKS = [ # except for when there are requested apps/variants/platforms ( {}, - 48, + 76, { - "Benchmarks desktop": [ - "'browsertime 'benchmark", - "!android 'shippable !-32 !clang", - "!live", - "!profil", - "!chrom", - ], - "Pageload (live) macosx": [ - "'browsertime 'tp6 'live", - "'osx 'shippable", - "!live", - "!profil", - "!chrom", - ], + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!live", + "!profil", + "!chrom", + ] + }, + "Pageload (live) macosx": { + "raptor": [ + "'browsertime 'tp6 'live", + "'osx 'shippable", + "!live", + "!profil", + "!chrom", + ] + }, + "Resource Usage linux": { + "awsy": ["'awsy", "!clang 'linux 'shippable"], + "raptor": [ + "'power 'osx", + "!clang 'linux 'shippable", + "!live", + "!profil", + "!chrom", + ], + "talos": ["'talos 'xperf | 'tp5", "!clang 'linux 'shippable"], + }, }, - ["Responsiveness android-p2 geckoview", "Benchmarks desktop chromium"], + [ + "Responsiveness android-p2 geckoview", + "Benchmarks desktop chromium", + ], ), # Default settings ( {"live_sites": True}, - 240, + 332, { - "Benchmarks desktop": [ - "'browsertime 'benchmark", - "!android 'shippable !-32 !clang", - "!profil", - "!chrom", - ], - "Pageload (live) macosx": [ - "'browsertime 'tp6 'live", - "'osx 'shippable", - "!profil", - "!chrom", - ], - "Benchmarks desktop firefox live-sites+profiling": [ - "'browsertime 'benchmark", - "!android 'shippable !-32 !clang", - "!chrom", - "!chrom !geckoview !fenix", - "'live", - "'profil", - ], + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!profil", + "!chrom", + ], + }, + "Pageload (live) macosx": { + "raptor": [ + "'browsertime 'tp6 'live", + "'osx 'shippable", + "!profil", + "!chrom", + ], + }, + "Benchmarks desktop firefox live-sites+profiling": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!chrom", + "!chrom !geckoview !fenix", + "'live", + "'profil", + ], + }, + "Graphics, & Media Playback desktop live-sites+profiling+swr": { + "raptor": [ + "'browsertime 'youtube-playback", + "!android 'shippable !-32 !clang", + "!chrom", + "'live", + "'profil", + ], + "talos": [ + "'talos 'svgr | 'bcv | 'webgl", + "!android 'shippable !-32 !clang", + "'profil", + "'swr", + ], + }, }, [ "Responsiveness android-p2 geckoview", "Benchmarks desktop chromium", "Benchmarks desktop firefox profiling", + "Talos desktop live-sites", + "Talos desktop profiling+swr", ], ), ( {"live_sites": True, "chrome": True}, - 480, + 644, { - "Benchmarks desktop": [ - "'browsertime 'benchmark", - "!android 'shippable !-32 !clang", - "!profil", - ], - "Pageload (live) macosx": [ - "'browsertime 'tp6 'live", - "'osx 'shippable", - "!profil", - ], - "Benchmarks desktop chromium": [ - "'browsertime 'benchmark", - "!android 'shippable !-32 !clang", - "!profil", - "'chromium", - ], + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!profil", + ], + }, + "Pageload (live) macosx": { + "raptor": [ + "'browsertime 'tp6 'live", + "'osx 'shippable", + "!profil", + ], + }, + "Benchmarks desktop chromium": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!profil", + "'chromium", + ], + }, }, - ["Responsiveness android-p2 geckoview"], + [ + "Responsiveness android-p2 geckoview", + "Firefox Pageload linux chrome", + "Talos PerfTests desktop swr", + ], ), ( {"android": True}, - 420, + 542, { - "Benchmarks desktop": [ - "'browsertime 'benchmark", - "!android 'shippable !-32 !clang", - "!live", - "!profil", - "!chrom", - ], - "Responsiveness android-a51 geckoview": [ - "'browsertime 'responsive", - "'android 'a51 'shippable 'aarch64", - "!live", - "!profil", - "!chrom", - "'geckoview", - ], + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!live", + "!profil", + "!chrom", + ], + }, + "Responsiveness android-a51 geckoview": { + "raptor": [ + "'browsertime 'responsive", + "'android 'a51 'shippable 'aarch64", + "!live", + "!profil", + "!chrom", + "'geckoview", + ], + }, }, - ["Responsiveness android-a51 chrome-m"], + [ + "Responsiveness android-a51 chrome-m", + "Firefox Pageload android", + ], ), ( {"android": True, "chrome": True}, - 720, + 924, { - "Benchmarks desktop": [ - "'browsertime 'benchmark", - "!android 'shippable !-32 !clang", - "!live", - "!profil", - ], - "Responsiveness android-a51 chrome-m": [ - "'browsertime 'responsive", - "'android 'a51 'shippable 'aarch64", - "!live", - "!profil", - "'chrome-m", - ], + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!live", + "!profil", + ], + }, + "Responsiveness android-a51 chrome-m": { + "raptor": [ + "'browsertime 'responsive", + "'android 'a51 'shippable 'aarch64", + "!live", + "!profil", + "'chrome-m", + ], + }, }, - ["Responsiveness android-p2 chrome-m"], + ["Responsiveness android-p2 chrome-m", "Resource Usage android"], ), ( {"android": True, "chrome": True, "profile": True}, - 1008, + 1324, { - "Benchmarks desktop": [ - "'browsertime 'benchmark", - "!android 'shippable !-32 !clang", - "!live", - ] + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!live", + ] + }, + "Talos PerfTests desktop profiling+swr": { + "talos": [ + "'talos", + "!android 'shippable !-32 !clang", + "'profil", + "'swr", + ] + }, }, - [], + [ + "Resource Usage desktop profiling", + "DAMP (Devtools) desktop chrome", + "Resource Usage android", + "Resource Usage windows chromium", + ], ), # Show all available windows tests, no other platform should exist # including the desktop catgeory ( {"requested_platforms": ["windows"]}, - 84, + 123, { - "Benchmarks windows firefox bytecode-cached+profiling": [ - "'browsertime 'benchmark", - "!-32 'windows 'shippable", - "!live", - "!chrom", - "!chrom !geckoview !fenix", - "'bytecode", - "'profil", - ] + "Benchmarks windows firefox bytecode-cached+profiling": { + "raptor": [ + "'browsertime 'benchmark", + "!-32 'windows 'shippable", + "!live", + "!chrom", + "!chrom !geckoview !fenix", + "'bytecode", + "'profil", + ] + }, }, [ + "Resource Usage desktop", "Benchmarks desktop", "Benchmarks linux firefox bytecode-cached+profiling", ], @@ -208,17 +412,19 @@ TASKS = [ "requested_apps": ["fenix"], "android": True, }, - 42, + 49, { - "Bytecode Cached android fenix no-fission+live-sites+profiling": [ - "'browsertime 'bytecode", - "'android 'a51 'shippable 'aarch64", - "!chrom", - "'fenix", - "'nofis", - "'live", - "'profil", - ], + "Bytecode Cached android fenix no-fission+live-sites+profiling": { + "raptor": [ + "'browsertime 'bytecode", + "'android 'a51 'shippable 'aarch64", + "!chrom", + "'fenix", + "'nofis", + "'live", + "'profil", + ], + } }, ["Benchmarks desktop", "Pageload (live) android"], ), @@ -229,25 +435,29 @@ TASKS = [ "requested_apps": ["fenix", "geckoview"], "android": True, }, - 84, + 98, { - "Benchmarks android geckoview live-sites+profiling": [ - "'browsertime 'benchmark", - "'android 'a51 'shippable 'aarch64", - "!chrom", - "'geckoview", - "'live", - "'profil", - ], - "Bytecode Cached android fenix no-fission+live-sites+profiling": [ - "'browsertime 'bytecode", - "'android 'a51 'shippable 'aarch64", - "!chrom", - "'fenix", - "'nofis", - "'live", - "'profil", - ], + "Benchmarks android geckoview live-sites+profiling": { + "raptor": [ + "'browsertime 'benchmark", + "'android 'a51 'shippable 'aarch64", + "!chrom", + "'geckoview", + "'live", + "'profil", + ], + }, + "Bytecode Cached android fenix no-fission+live-sites+profiling": { + "raptor": [ + "'browsertime 'bytecode", + "'android 'a51 'shippable 'aarch64", + "!chrom", + "'fenix", + "'nofis", + "'live", + "'profil", + ], + }, }, [ "Benchmarks desktop", @@ -263,34 +473,40 @@ TASKS = [ "requested_apps": ["fenix"], "android": True, }, - 60, + 70, { - "Pageload android-a51 fenix": [ - "'browsertime 'tp6", - "'android 'a51 'shippable 'aarch64", - "!live", - "!profil", - "!chrom", - "'fenix", - ], - "Pageload android-a51 fenix no-fission": [ - "'browsertime 'tp6", - "'android 'a51 'shippable 'aarch64", - "!live", - "!profil", - "!chrom", - "'fenix", - "'nofis", - ], - "Pageload (essential) android fenix no-fission+live-sites": [ - "'browsertime 'tp6 'essential", - "'android 'a51 'shippable 'aarch64", - "!profil", - "!chrom", - "'fenix", - "'nofis", - "'live", - ], + "Pageload android-a51 fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "!live", + "!profil", + "!chrom", + "'fenix", + ], + }, + "Pageload android-a51 fenix no-fission": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "!live", + "!profil", + "!chrom", + "'fenix", + "'nofis", + ], + }, + "Pageload (essential) android fenix no-fission+live-sites": { + "raptor": [ + "'browsertime 'tp6 'essential", + "'android 'a51 'shippable 'aarch64", + "!profil", + "!chrom", + "'fenix", + "'nofis", + "'live", + ], + }, }, [ "Benchmarks desktop", @@ -306,42 +522,50 @@ TASKS = [ "requested_apps": ["fenix"], "android": True, }, - 84, + 98, { - "Pageload android-a51 fenix": [ - "'browsertime 'tp6", - "'android 'a51 'shippable 'aarch64", - "!live", - "!profil", - "!chrom", - "'fenix", - ], - "Pageload android-a51 fenix no-fission": [ - "'browsertime 'tp6", - "'android 'a51 'shippable 'aarch64", - "!live", - "!profil", - "!chrom", - "'fenix", - "'nofis", - ], - "Pageload android-a51 fenix live-sites": [ - "'browsertime 'tp6", - "'android 'a51 'shippable 'aarch64", - "!profil", - "!chrom", - "'fenix", - "'live", - ], - "Pageload (essential) android fenix no-fission+live-sites": [ - "'browsertime 'tp6 'essential", - "'android 'a51 'shippable 'aarch64", - "!profil", - "!chrom", - "'fenix", - "'nofis", - "'live", - ], + "Pageload android-a51 fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "!live", + "!profil", + "!chrom", + "'fenix", + ], + }, + "Pageload android-a51 fenix no-fission": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "!live", + "!profil", + "!chrom", + "'fenix", + "'nofis", + ], + }, + "Pageload android-a51 fenix live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "!profil", + "!chrom", + "'fenix", + "'live", + ], + }, + "Pageload (essential) android fenix no-fission+live-sites": { + "raptor": [ + "'browsertime 'tp6 'essential", + "'android 'a51 'shippable 'aarch64", + "!profil", + "!chrom", + "'fenix", + "'nofis", + "'live", + ], + }, }, [ "Benchmarks desktop", @@ -356,16 +580,18 @@ TASKS = [ "requested_variants": ["no-fission"], "requested_platforms": ["windows"], }, - 12, + 19, { - "Responsiveness windows firefox": [ - "'browsertime 'responsive", - "!-32 'windows 'shippable", - "!live", - "!profil", - "!chrom", - "!chrom !geckoview !fenix", - ], + "Responsiveness windows firefox": { + "raptor": [ + "'browsertime 'responsive", + "!-32 'windows 'shippable", + "!live", + "!profil", + "!chrom", + "!chrom !geckoview !fenix", + ], + }, }, ["Benchmarks desktop", "Responsiveness windows firefox no-fisson"], ), @@ -376,29 +602,50 @@ TASKS = [ "requested_platforms": ["windows"], "android": True, }, - 60, + 83, { - "Responsiveness windows firefox": [ - "'browsertime 'responsive", - "!-32 'windows 'shippable", - "!live", - "!profil", - "!chrom", - "!chrom !geckoview !fenix", - ], - "Responsiveness windows firefox live-sites": [ - "'browsertime 'responsive", - "!-32 'windows 'shippable", - "!profil", - "!chrom", - "!chrom !geckoview !fenix", - "'live", - ], + "Responsiveness windows firefox": { + "raptor": [ + "'browsertime 'responsive", + "!-32 'windows 'shippable", + "!live", + "!profil", + "!chrom", + "!chrom !geckoview !fenix", + ], + }, + "Responsiveness windows firefox live-sites": { + "raptor": [ + "'browsertime 'responsive", + "!-32 'windows 'shippable", + "!profil", + "!chrom", + "!chrom !geckoview !fenix", + "'live", + ], + }, + "Graphics, & Media Playback windows live-sites+profiling+swr": { + "raptor": [ + "'browsertime 'youtube-playback", + "!-32 'windows 'shippable", + "!chrom", + "'live", + "'profil", + ], + "talos": [ + "'talos 'svgr | 'bcv | 'webgl", + "!-32 'windows 'shippable", + "'profil", + "'swr", + ], + }, }, [ "Benchmarks desktop", "Responsiveness windows firefox no-fisson", "Pageload (live) android", + "Talos desktop live-sites", + "Talos android", ], ), ], @@ -406,6 +653,10 @@ TASKS = [ def test_category_expansion( category_options, expected_counts, unique_categories, missing ): + # Set the categories, and variants to expand + ps.PerfParser.categories = TEST_CATEGORIES + ps.PerfParser.variants = TEST_VARIANTS + # Expand the categories, then either check if the unique_categories, # exist or are missing from the categories expanded_cats = ps.PerfParser.expand_categories(**category_options) @@ -480,6 +731,7 @@ def test_full_run(options, call_counts, log_ind, expected_log_message): ["", TASKS], ["", TASKS], ["", TASKS], + ["", TASKS], ] ps.run(**options)