diff --git a/python/mozperftest/mozperftest/test/browsertime/script.py b/python/mozperftest/mozperftest/test/browsertime/script.py index b3e373880bf2..444a3e533bc1 100644 --- a/python/mozperftest/mozperftest/test/browsertime/script.py +++ b/python/mozperftest/mozperftest/test/browsertime/script.py @@ -95,7 +95,7 @@ class ScriptInfo(defaultdict): def __str__(self): """Used to generate docs.""" - d = {} + d = defaultdict(lambda: "N/A") for field, value in self.items(): if field == "filename": d[field] = self.script.name @@ -108,7 +108,7 @@ class ScriptInfo(defaultdict): value = ", ".join(value) d[field] = value - d["filename_underline"] = "-" * len(d["filename"]) + d["filename_underline"] = "=" * len(d["filename"]) return _INFO % d def __missing__(self, key): diff --git a/python/mozperftest/perfdocs/config.yml b/python/mozperftest/perfdocs/config.yml index c3a4fefaf8ca..3b262dcf9a7a 100644 --- a/python/mozperftest/perfdocs/config.yml +++ b/python/mozperftest/perfdocs/config.yml @@ -4,5 +4,31 @@ --- name: mozperftest manifest: None -static-only: True -suites: {} +static-only: False +suites: + netwerk/test/perf: + description: "Performance tests from the 'network/test/perf' folder." + tests: + youtube-scroll: "" + facebook-scroll: "" + cloudflare: "" + g-search: "" + g-image: "" + lq-fetch: "" + youtube-noscroll: "" + testing/performance: + description: "Performance tests from the 'testing/performance' folder." + tests: + Politico Link: "" + BBC Link: "" + JSConf (cold): "" + VIEW: "" + main: "" + Facebook: "" + YouTube Link: "" + pageload: "" + JSConf (warm): "" + browser/base/content/test: + description: "Performance tests from the 'browser/base/content/test' folder." + tests: + Dom-size: "" diff --git a/python/mozperftest/perfdocs/index.rst b/python/mozperftest/perfdocs/index.rst index a801cf3d30b8..6f1ff7467aed 100644 --- a/python/mozperftest/perfdocs/index.rst +++ b/python/mozperftest/perfdocs/index.rst @@ -10,3 +10,9 @@ mozperftest running writing developing + +The following documents all testing we have for mozperftest. +If the owner does not specify the Usage and Description, it's marked N/A. + +{documentation} +If you have any questions, please see this `wiki page `_. diff --git a/testing/perfdocs/generated/mozperftest.rst b/testing/perfdocs/generated/mozperftest.rst index a801cf3d30b8..1d4793ad98ff 100644 --- a/testing/perfdocs/generated/mozperftest.rst +++ b/testing/perfdocs/generated/mozperftest.rst @@ -10,3 +10,348 @@ mozperftest running writing developing + +The following documents all testing we have for mozperftest. +If the owner does not specify the Usage and Description, it's marked N/A. + +browser/base/content/test +------------------------- +Performance tests from the 'browser/base/content/test' folder. + +perftest_browser_xhtml_dom.js +============================= + +Measures the size of the DOM + +| Owner: Browser Front-end team +| Test Name: Dom-size +| Usage: + +:: + + N/A + +| Description: + +N/A + + + +netwerk/test/perf +----------------- +Performance tests from the 'network/test/perf' folder. + +perftest_http3_cloudflareblog.js +================================ + +User-journey live site test for cloudflare blog. + +| Owner: Network Team +| Test Name: cloudflare +| Usage: + +:: + + N/A + +| Description: + +N/A + + +perftest_http3_facebook_scroll.js +================================= + +Measures the number of requests per second after a scroll. + +| Owner: Network Team +| Test Name: facebook-scroll +| Usage: + +:: + + N/A + +| Description: + +N/A + + +perftest_http3_google_image.js +============================== + +Measures the number of images per second after a scroll. + +| Owner: Network Team +| Test Name: g-image +| Usage: + +:: + + N/A + +| Description: + +N/A + + +perftest_http3_google_search.js +=============================== + +User-journey live site test for google search + +| Owner: Network Team +| Test Name: g-search +| Usage: + +:: + + N/A + +| Description: + +N/A + + +perftest_http3_lucasquicfetch.js +================================ + +Measures the amount of time it takes to load a set of images. + +| Owner: Network Team +| Test Name: lq-fetch +| Usage: + +:: + + N/A + +| Description: + +N/A + + +perftest_http3_youtube_watch.js +=============================== + +Measures quality of the video being played. + +| Owner: Network Team +| Test Name: youtube-noscroll +| Usage: + +:: + + N/A + +| Description: + +N/A + + +perftest_http3_youtube_watch_scroll.js +====================================== + +Measures quality of the video being played. + +| Owner: Network Team +| Test Name: youtube-scroll +| Usage: + +:: + + N/A + +| Description: + +N/A + + + +testing/performance +------------------- +Performance tests from the 'testing/performance' folder. + +perftest_bbc_link.js +==================== + +Measures time to load BBC homepage + +| Owner: Performance Team +| Test Name: BBC Link +| Usage: + +:: + + N/A + +| Description: + +N/A + + +perftest_facebook.js +==================== + +Measures time to log in to Facebook + +| Owner: Performance Team +| Test Name: Facebook +| Usage: + +:: + + N/A + +| Description: + +N/A + + +perftest_jsconf_cold.js +======================= + +Measures time to load JSConf page (cold) + +| Owner: Performance Team +| Test Name: JSConf (cold) +| Usage: + +:: + + N/A + +| Description: + +N/A + + +perftest_jsconf_warm.js +======================= + +Measures time to load JSConf page (warm) + +| Owner: Performance Team +| Test Name: JSConf (warm) +| Usage: + +:: + + N/A + +| Description: + +N/A + + +perftest_politico_link.js +========================= + +Measures time to load Politico homepage + +| Owner: Performance Team +| Test Name: Politico Link +| Usage: + +:: + + N/A + +| Description: + +N/A + + +perftest_android_view.js +======================== + +Measures cold process view time + +| Owner: Performance Team +| Test Name: VIEW +| Usage: + +:: + + ./mach perftest testing/performance/perftest_android_view.js + --android-install-apk ~/fenix.v2.fennec-nightly.2020.04.22-arm32.apk + --hooks testing/performance/hooks_android_view.py --android-app-name + org.mozilla.fenix --perfherder-metrics processLaunchToNavStart + +| Description: + +This test launches the appropriate android app, simulating a opening a +link through VIEW intent workflow. The application is launched with +the intent action android.intent.action.VIEW loading a trivially +simple website. The reported metric is the time from process start to +navigationStart, reported as processLaunchToNavStart + + +perftest_youtube_link.js +======================== + +Measures time to load YouTube video + +| Owner: Performance Team +| Test Name: YouTube Link +| Usage: + +:: + + N/A + +| Description: + +N/A + + +perftest_android_main.js +======================== + +Measures the time from process start until the Fenix main activity +(HomeActivity) reports Fully Drawn + +| Owner: Performance Team +| Test Name: main +| Usage: + +:: + + ./mach perftest testing/performance/perftest_android_main.js --android + --flavor mobile-browser --hooks + testing/performance/hooks_home_activity.py --perfherder + --android-app-name org.mozilla.fenix --android-activity .App + --android-install-apk ~/Downloads/fenix.apk --android-clear-logcat + --android-capture-logcat logcat --androidlog-first-timestamp ".*Start + proc.*org.mozilla.fenix.*.App.*" --androidlog-second-timestamp + ".*Fully drawn.*org.mozilla.fenix.*" --androidlog-subtest-name "MAIN" + --androidlog + +| Description: + +This test launches Fenix to its main activity (HomeActivity). The +application logs "Fully Drawn" when the activity is drawn. Using the +android log transformer we measure the time from process start to this +event. + + +perftest_pageload.js +==================== + +Measures time to load mozilla page + +| Owner: Performance Team +| Test Name: pageload +| Usage: + +:: + + N/A + +| Description: + +N/A + + + +If you have any questions, please see this `wiki page `_. diff --git a/testing/performance/perftest_pageload.js b/testing/performance/perftest_pageload.js index db2c936d16cd..6f08f2ead3b5 100644 --- a/testing/performance/perftest_pageload.js +++ b/testing/performance/perftest_pageload.js @@ -19,4 +19,7 @@ module.exports = { // eslint-disable-line setUp, tearDown, test, + owner: "Performance Team", + name: "pageload", + description: "Measures time to load mozilla page", }; diff --git a/tools/lint/perfdocs/framework_gatherers.py b/tools/lint/perfdocs/framework_gatherers.py index 9ace287f2f3a..dbe782a62563 100644 --- a/tools/lint/perfdocs/framework_gatherers.py +++ b/tools/lint/perfdocs/framework_gatherers.py @@ -5,10 +5,12 @@ from __future__ import absolute_import import collections import os +import pathlib import re from perfdocs.utils import read_yaml from manifestparser import TestManifest +from mozperftest.test.browsertime.script import ScriptInfo """ This file is for framework specific gatherers since manifests @@ -33,6 +35,7 @@ class FrameworkGatherer(object): self._urls = {} self._manifest_path = "" self._manifest = None + self.script_infos = {} def get_manifest_path(self): """ @@ -71,11 +74,10 @@ class FrameworkGatherer(object): and paragraph as content mentioned. :param title: title of the section :param content: content of section paragraph - :param documentation: documentation object to add section to - :param type: type of the title heading + :param header_type: type of the title heading """ - heading_map = {"H4": "-", "H5": "^"} - return [title, heading_map.get(type, "^") * len(title), content, ""] + heading_map = {"H3": "=", "H4": "-", "H5": "^"} + return [title, heading_map.get(header_type, "^") * len(title), content, ""] class RaptorGatherer(FrameworkGatherer): @@ -147,10 +149,10 @@ class RaptorGatherer(FrameworkGatherer): Returns a dictionary containing the tests in every suite ini file. :return dict: A dictionary with the following structure: { - "suite_name": [ + "suite_name": { 'raptor_test1', 'raptor_test2' - ] + }, } """ if self._test_list: @@ -193,10 +195,62 @@ class RaptorGatherer(FrameworkGatherer): + ">`__" ] + def build_suite_section(self, title, content): + return self._build_section_with_header( + title.capitalize(), content, header_type="H4" + ) + class MozperftestGatherer(FrameworkGatherer): """ Gatherer for the Mozperftest framework. """ - pass + def get_test_list(self): + """ + Returns a dictionary containing the tests that start with perftest_*. + + :return dict: A dictionary with the following structure: { + "suite_name": { + 'perftest_test1', + 'perftest_test2', + }, + } + """ + exclude_dir = [".hg", "mozperftest/tests/data"] + for path in pathlib.Path(self.workspace_dir).rglob("perftest_*"): + if any(re.search(d, str(path)) for d in exclude_dir): + continue + + suite_name = re.sub(self.workspace_dir, "", os.path.dirname(path)) + si = ScriptInfo(path) + self.script_infos[si["name"]] = si + self._test_list.setdefault(suite_name, {}).update({si["name"]: ""}) + + return self._test_list + + def build_test_description(self, title, test_description="", suite_name=""): + result, tab_flag = "", False + desc = str(self.script_infos[title]) + category = ("Owner: ", "Test Name: ", "Usage:", "Description:") + + for s in desc.split("\n"): + if s.startswith(category): + result += "| " + s + "\n" + if s in category[2:]: + result += "\n" + tab_flag = False + else: + if tab_flag and s: + result += " " + s + "\n" + else: + result += s + "\n" + + if s == category[2]: + result += "::\n\n" + tab_flag = True + + return [result] + + def build_suite_section(self, title, content): + return self._build_section_with_header(title, content, header_type="H4") diff --git a/tools/lint/perfdocs/generator.py b/tools/lint/perfdocs/generator.py index e7806d23400b..6d3c99d00b97 100644 --- a/tools/lint/perfdocs/generator.py +++ b/tools/lint/perfdocs/generator.py @@ -58,19 +58,6 @@ class Generator(object): } """ - def _append_rst_section(title, content, documentation, type=None): - """ - Adds a section to the documentation with the title as the type mentioned - and paragraph as content mentioned. - :param title: title of the section - :param content: content of section paragraph - :param documentation: documentation object to add section to - :param type: type of the title heading - """ - heading_map = {"H4": "-", "H5": "^"} - heading_symbol = heading_map.get(type, "-") - documentation.extend([title, heading_symbol * len(title), content, ""]) - # Using the verified `perfdocs_tree`, build up the documentation. frameworks_info = {} for framework in self._perfdocs_tree: @@ -86,13 +73,13 @@ class Generator(object): for suite_name in sorted(suites.keys()): suite_info = suites[suite_name] - # Add the suite with an H4 heading - _append_rst_section( - suite_name.capitalize(), - suite_info["description"], - documentation, - type="H4", + # Add the suite section + documentation.extend( + self._verifier._gatherer.framework_gatherers[ + yaml_content["name"] + ].build_suite_section(suite_name, suite_info["description"]) ) + tests = suite_info.get("tests", {}) for test_name in sorted(tests.keys()): documentation.extend( diff --git a/tools/lint/perfdocs/verifier.py b/tools/lint/perfdocs/verifier.py index 9e8be60a812b..0883a8d5d308 100644 --- a/tools/lint/perfdocs/verifier.py +++ b/tools/lint/perfdocs/verifier.py @@ -210,7 +210,7 @@ class Verifier(object): Recompute the description in case it's a file. """ desc_path = os.path.join(self.workspace_dir, desc) - if os.path.exists(desc_path): + if os.path.exists(desc_path) and os.path.isfile(desc_path): with open(desc_path, "r") as f: desc = f.readlines() return desc