mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-24 21:31:04 +00:00
Bug 1735324 [perfdocs] Enumerate AWSY tests using PerfDocs r=perftest-reviewers,sparky,AlexandruIonescu
Differential Revision: https://phabricator.services.mozilla.com/D128198
This commit is contained in:
afinder
parent
b39be51d3d
commit
bb49296ca2
@ -4,5 +4,12 @@
|
||||
---
|
||||
name: awsy
|
||||
manifest: None
|
||||
static-only: True
|
||||
suites: {}
|
||||
static-only: False
|
||||
suites:
|
||||
Awsy tests:
|
||||
description: ""
|
||||
tests:
|
||||
base: "FF"
|
||||
dmd: "FF"
|
||||
tp5: "FF"
|
||||
tp6: "FF"
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
====
|
||||
AWSY
|
||||
#####
|
||||
====
|
||||
|
||||
Are We Slim Yet project (commonly known as AWSY) tracks memory usage across builds.
|
||||
|
||||
@ -9,6 +10,8 @@ AWSY tests consist of three types: TP5*, TP6, and Base Memory Usage.
|
||||
|
||||
*\*TP5 tests are out of date and no longer maintained. These tests are scheduled to be removed:* https://bugzilla.mozilla.org/show_bug.cgi?id=1712406
|
||||
|
||||
{documentation}
|
||||
|
||||
Running AWSY Locally
|
||||
*********************
|
||||
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
====
|
||||
AWSY
|
||||
#####
|
||||
====
|
||||
|
||||
Are We Slim Yet project (commonly known as AWSY) tracks memory usage across builds.
|
||||
|
||||
@ -9,6 +10,160 @@ AWSY tests consist of three types: TP5*, TP6, and Base Memory Usage.
|
||||
|
||||
*\*TP5 tests are out of date and no longer maintained. These tests are scheduled to be removed:* https://bugzilla.mozilla.org/show_bug.cgi?id=1712406
|
||||
|
||||
Awsy tests
|
||||
----------
|
||||
|
||||
|
||||
.. dropdown:: base (FF)
|
||||
:container: + anchor-id-base-Awsy-tests
|
||||
|
||||
* **Test Task**:
|
||||
* test-linux1804-64-clang-trunk-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-base-e10s: None
|
||||
* awsy-base-fis-e10s: None
|
||||
|
||||
* test-linux1804-64-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-base-e10s: None
|
||||
* awsy-base-fis-e10s: None
|
||||
|
||||
* test-linux1804-64-shippable-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-base-e10s: integration, mozilla-beta, mozilla-central, mozilla-release
|
||||
* awsy-base-fis-e10s: integration, mozilla-beta, mozilla-central, mozilla-release
|
||||
|
||||
* test-macosx1015-64-shippable-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-base-e10s: integration, mozilla-beta, mozilla-central, mozilla-release
|
||||
* awsy-base-fis-e10s: integration, mozilla-beta, mozilla-central, mozilla-release
|
||||
|
||||
* test-windows10-32-2004-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-base-e10s: None
|
||||
* awsy-base-fis-e10s: None
|
||||
|
||||
* test-windows10-32-2004-shippable-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-base-e10s: None
|
||||
* awsy-base-fis-e10s: None
|
||||
|
||||
* test-windows10-64-2004-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-base-e10s: None
|
||||
* awsy-base-fis-e10s: None
|
||||
|
||||
* test-windows10-64-2004-shippable-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-base-e10s: integration, mozilla-beta, mozilla-central, mozilla-release
|
||||
* awsy-base-fis-e10s: integration, mozilla-beta, mozilla-central, mozilla-release
|
||||
|
||||
|
||||
.. dropdown:: dmd (FF)
|
||||
:container: + anchor-id-dmd-Awsy-tests
|
||||
|
||||
* **Test Task**:
|
||||
* test-linux1804-64-clang-trunk-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-dmd-e10s: None
|
||||
|
||||
* test-linux1804-64-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-dmd-e10s: None
|
||||
|
||||
* test-linux1804-64-shippable-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-dmd-e10s: None
|
||||
|
||||
* test-macosx1015-64-shippable-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-dmd-e10s: None
|
||||
|
||||
* test-windows10-32-2004-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-dmd-e10s: None
|
||||
|
||||
* test-windows10-32-2004-shippable-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-dmd-e10s: None
|
||||
|
||||
* test-windows10-64-2004-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-dmd-e10s: None
|
||||
|
||||
* test-windows10-64-2004-shippable-qr/opt
|
||||
* awsy-base-dmd-e10s: None
|
||||
* awsy-dmd-e10s: None
|
||||
|
||||
|
||||
.. dropdown:: tp5 (FF)
|
||||
:container: + anchor-id-tp5-Awsy-tests
|
||||
|
||||
* **Test Task**:
|
||||
* test-linux1804-64-clang-trunk-qr/opt
|
||||
* awsy-e10s: None
|
||||
|
||||
* test-linux1804-64-qr/opt
|
||||
* awsy-e10s: None
|
||||
|
||||
* test-linux1804-64-shippable-qr/opt
|
||||
* awsy-e10s: None
|
||||
|
||||
* test-macosx1015-64-shippable-qr/opt
|
||||
* awsy-e10s: None
|
||||
|
||||
* test-windows10-32-2004-qr/opt
|
||||
* awsy-e10s: None
|
||||
|
||||
* test-windows10-32-2004-shippable-qr/opt
|
||||
* awsy-e10s: None
|
||||
|
||||
* test-windows10-64-2004-qr/opt
|
||||
* awsy-e10s: None
|
||||
|
||||
* test-windows10-64-2004-shippable-qr/opt
|
||||
* awsy-e10s: None
|
||||
|
||||
|
||||
.. dropdown:: tp6 (FF)
|
||||
:container: + anchor-id-tp6-Awsy-tests
|
||||
|
||||
* **Test Task**:
|
||||
* test-linux1804-64-clang-trunk-qr/opt
|
||||
* awsy-tp6-e10s: None
|
||||
* awsy-tp6-fis-e10s: None
|
||||
|
||||
* test-linux1804-64-qr/opt
|
||||
* awsy-tp6-e10s: None
|
||||
* awsy-tp6-fis-e10s: None
|
||||
|
||||
* test-linux1804-64-shippable-qr/opt
|
||||
* awsy-tp6-e10s: integration, mozilla-beta, mozilla-central, mozilla-release
|
||||
* awsy-tp6-fis-e10s: integration, mozilla-beta, mozilla-central, mozilla-release
|
||||
|
||||
* test-macosx1015-64-shippable-qr/opt
|
||||
* awsy-tp6-e10s: integration, mozilla-beta, mozilla-central, mozilla-release
|
||||
* awsy-tp6-fis-e10s: integration, mozilla-beta, mozilla-central, mozilla-release
|
||||
|
||||
* test-windows10-32-2004-qr/opt
|
||||
* awsy-tp6-e10s: None
|
||||
* awsy-tp6-fis-e10s: None
|
||||
|
||||
* test-windows10-32-2004-shippable-qr/opt
|
||||
* awsy-tp6-e10s: None
|
||||
* awsy-tp6-fis-e10s: None
|
||||
|
||||
* test-windows10-64-2004-qr/opt
|
||||
* awsy-tp6-e10s: None
|
||||
* awsy-tp6-fis-e10s: None
|
||||
|
||||
* test-windows10-64-2004-shippable-qr/opt
|
||||
* awsy-tp6-e10s: integration, mozilla-beta, mozilla-central, mozilla-release
|
||||
* awsy-tp6-fis-e10s: integration, mozilla-beta, mozilla-central, mozilla-release
|
||||
|
||||
|
||||
|
||||
|
||||
Running AWSY Locally
|
||||
*********************
|
||||
|
||||
|
||||
@ -40,6 +40,10 @@ class FrameworkGatherer(object):
|
||||
self._manifest = None
|
||||
self.script_infos = {}
|
||||
self._task_list = {}
|
||||
self._task_match_pattern = re.compile(r"([\w\W]*/[pgo|opt]*)-([\w\W]*)")
|
||||
|
||||
def get_task_match(self, task_name):
|
||||
return re.search(self._task_match_pattern, task_name)
|
||||
|
||||
def get_manifest_path(self):
|
||||
"""
|
||||
@ -135,7 +139,7 @@ class RaptorGatherer(FrameworkGatherer):
|
||||
run_on_projects = self._taskgraph[task].attributes["run_on_projects"]
|
||||
|
||||
test_match = re.search(r"[\s']--test[\s=](.+?)[\s']", str(command))
|
||||
task_match = re.search(r"([\w\W]*/[pgo|opt]*)-([\w\W]*)", task)
|
||||
task_match = self.get_task_match(task)
|
||||
if test_match and task_match:
|
||||
test = test_match.group(1)
|
||||
platform = task_match.group(1)
|
||||
@ -376,6 +380,78 @@ class TalosGatherer(FrameworkGatherer):
|
||||
return self._build_section_with_header(title, content, header_type="H3")
|
||||
|
||||
|
||||
class AwsyGatherer(FrameworkGatherer):
|
||||
"""
|
||||
Gatherer for the Awsy framework.
|
||||
"""
|
||||
|
||||
def _generate_ci_tasks(self):
|
||||
for task_name in self._taskgraph.keys():
|
||||
task = self._taskgraph[task_name]
|
||||
|
||||
if type(task) == dict:
|
||||
awsy_test = task["task"]["extra"].get("suite", [])
|
||||
run_on_projects = task["attributes"]["run_on_projects"]
|
||||
else:
|
||||
awsy_test = task.task["extra"].get("suite", [])
|
||||
run_on_projects = task.attributes["run_on_projects"]
|
||||
|
||||
task_match = self.get_task_match(task_name)
|
||||
|
||||
if "awsy" in awsy_test and task_match:
|
||||
platform = task_match.group(1)
|
||||
test_name = task_match.group(2)
|
||||
item = {"test_name": test_name, "run_on_projects": run_on_projects}
|
||||
self._task_list.setdefault(platform, []).append(item)
|
||||
|
||||
def get_suite_list(self):
|
||||
self._suite_list = {"Awsy tests": ["tp6", "base", "dmd", "tp5"]}
|
||||
return self._suite_list
|
||||
|
||||
def get_test_list(self):
|
||||
self._generate_ci_tasks()
|
||||
return {
|
||||
"Awsy tests": {
|
||||
"tp6": "",
|
||||
"base": "",
|
||||
"dmd": "",
|
||||
"tp5": "",
|
||||
}
|
||||
}
|
||||
|
||||
def build_suite_section(self, title, content):
|
||||
return self._build_section_with_header(
|
||||
title.capitalize(), content, header_type="H4"
|
||||
)
|
||||
|
||||
def build_test_description(self, title, test_description="", suite_name=""):
|
||||
dropdown_suite_name = suite_name.replace(" ", "-")
|
||||
result = f".. dropdown:: {title} ({test_description})\n"
|
||||
result += f" :container: + anchor-id-{title}-{dropdown_suite_name}\n\n"
|
||||
result += " * **Test Task**:\n"
|
||||
|
||||
# tp5 tests are represented by awsy-e10s test names
|
||||
# while the others have their title in test names
|
||||
search_tag = "awsy-e10s" if title == "tp5" else title
|
||||
for platform in sorted(self._task_list.keys()):
|
||||
result += f" * {platform}\n"
|
||||
for test_dict in sorted(
|
||||
self._task_list[platform], key=lambda d: d["test_name"]
|
||||
):
|
||||
if search_tag in test_dict["test_name"]:
|
||||
run_on_project = ": " + (
|
||||
", ".join(test_dict["run_on_projects"])
|
||||
if test_dict["run_on_projects"]
|
||||
else "None"
|
||||
)
|
||||
result += (
|
||||
f" * {test_dict['test_name']}{run_on_project}\n"
|
||||
)
|
||||
result += "\n"
|
||||
|
||||
return [result]
|
||||
|
||||
|
||||
class StaticGatherer(FrameworkGatherer):
|
||||
"""
|
||||
A noop gatherer for frameworks with static-only documentation.
|
||||
|
||||
@ -13,6 +13,7 @@ from perfdocs.framework_gatherers import (
|
||||
RaptorGatherer,
|
||||
StaticGatherer,
|
||||
TalosGatherer,
|
||||
AwsyGatherer,
|
||||
)
|
||||
|
||||
logger = PerfDocLogger()
|
||||
@ -22,6 +23,7 @@ frameworks = {
|
||||
"raptor": RaptorGatherer,
|
||||
"mozperftest": MozperftestGatherer,
|
||||
"talos": TalosGatherer,
|
||||
"awsy": AwsyGatherer,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -53,7 +53,6 @@ module.exports = {
|
||||
};
|
||||
"""
|
||||
|
||||
|
||||
SAMPLE_CONFIG = """
|
||||
name: mozperftest
|
||||
manifest: None
|
||||
@ -65,7 +64,6 @@ suites:
|
||||
Example: ""
|
||||
"""
|
||||
|
||||
|
||||
DYNAMIC_SAMPLE_CONFIG = """
|
||||
name: {}
|
||||
manifest: None
|
||||
@ -81,7 +79,6 @@ suites:
|
||||
Example: "Performance test Example from another_suite."
|
||||
"""
|
||||
|
||||
|
||||
SAMPLE_INI = """
|
||||
[Example]
|
||||
test_url = Example_url
|
||||
@ -487,6 +484,12 @@ def test_perfdocs_framework_gatherers(logger, structured_logger, perfdocs_sample
|
||||
assert suitetests
|
||||
continue
|
||||
|
||||
if framework == "awsy":
|
||||
for suite, suitetests in fg.get_test_list().items():
|
||||
assert suite == "Awsy tests"
|
||||
assert suitetests
|
||||
continue
|
||||
|
||||
for suite, suitetests in fg.get_test_list().items():
|
||||
assert suite == "suite"
|
||||
for test, manifest in suitetests.items():
|
||||
|
||||
Loading…
Reference in New Issue
Block a user