Bug 1830741 - Expose benchmark options in Raptor for custom testing. r=perftest-reviewers,AlexandruIonescu

This patch adds options for `--benchmark-repository`, `--benchmark-revision`, and `--benchmark-branch` to make it easier to run custom benchark tests in Raptor. It also makes it possible to set the benchmark settings through mach try perf.

Differential Revision: https://phabricator.services.mozilla.com/D177030

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -614,6 +614,41 @@ class Raptor(
                     ),
                 },
             ],
+            [
+                ["--benchmark-repository"],
+                {
+                    "dest": "benchmark_repository",
+                    "type": "str",
+                    "default": None,
+                    "help": (
+                        "Repository that should be used for a particular benchmark test. "
+                        "e.g. https://github.com/mozilla-mobile/firefox-android"
+                    ),
+                },
+            ],
+            [
+                ["--benchmark-revision"],
+                {
+                    "dest": "benchmark_revision",
+                    "type": "str",
+                    "default": None,
+                    "help": (
+                        "Repository revision that should be used for a particular "
+                        "benchmark test."
+                    ),
+                },
+            ],
+            [
+                ["--benchmark-branch"],
+                {
+                    "dest": "benchmark_branch",
+                    "type": "str",
+                    "default": None,
+                    "help": (
+                        "Repository branch that should be used for a particular benchmark test."
+                    ),
+                },
+            ],
         ]
         + testing_config_options
         + copy.deepcopy(code_coverage_config_options)
@@ -976,6 +1011,12 @@ class Raptor(
             kw_options["activity"] = self.config["activity"]
         if self.config.get("conditioned_profile") is not None:
             kw_options["conditioned-profile"] = self.config["conditioned_profile"]
+        if self.config.get("benchmark_repository"):
+            kw_options["benchmark_repository"] = self.config["benchmark_repository"]
+        if self.config.get("benchmark_revision"):
+            kw_options["benchmark_revision"] = self.config["benchmark_revision"]
+        if self.config.get("benchmark_repository"):
+            kw_options["benchmark_branch"] = self.config["benchmark_branch"]
 
         kw_options.update(kw)
         if self.host:

testing/raptor/raptor/benchmark.py

@@ -24,6 +24,16 @@ class Benchmark(object):
         self.config = config
         self.test = test
 
+        # Note that we can only change the repository, revision, and branch through here.
+        # The path to the test should remain constant. If it needs to be changed, make a
+        # patch that changes it for the new test.
+        if self.config.get("benchmark_repository", None):
+            self.test["repository"] = self.config["benchmark_repository"]
+            self.test["repository_revision"] = self.config["benchmark_revision"]
+
+            if self.config.get("benchmark_branch", None):
+                self.test["branch"] = self.config["benchmark_branch"]
+
         self.setup_benchmarks(
             os.getenv("MOZ_DEVELOPER_REPO_DIR"),
             os.getenv("MOZ_MOZBUILD_DIR"),

testing/raptor/raptor/cmdline.py

@@ -513,6 +513,28 @@ def create_parser(mach_interface=False):
         help="Alternative methods for summarizing technical and visual pageload metrics. "
         "Options: geomean, mean.",
     )
+    add_arg(
+        "--benchmark-repository",
+        dest="benchmark_repository",
+        default=None,
+        type=str,
+        help="Repository that should be used for a particular benchmark test. "
+        "e.g. https://github.com/mozilla-mobile/firefox-android",
+    )
+    add_arg(
+        "--benchmark-revision",
+        dest="benchmark_revision",
+        default=None,
+        type=str,
+        help="Repository revision that should be used for a particular benchmark test.",
+    )
+    add_arg(
+        "--benchmark-branch",
+        dest="benchmark_branch",
+        default=None,
+        type=str,
+        help="Repository branch that should be used for a particular benchmark test.",
+    )
 
     add_logging_group(parser)
     return parser
@@ -618,6 +640,12 @@ def verify_options(parser, args):
                 # otherwise fail out
                 parser.error("--intent command-line argument is required!")
 
+    if args.benchmark_repository:
+        if not args.benchmark_revision:
+            parser.error(
+                "When a benchmark repository is provided, a revision is also required."
+            )
+
 
 def parse_args(argv=None):
     parser = create_parser()

testing/raptor/raptor/perftest.py

@@ -105,6 +105,9 @@ class Perftest(object):
         python=None,
         fission=True,
         extra_summary_methods=[],
+        benchmark_repository=None,
+        benchmark_revision=None,
+        benchmark_branch=None,
         **kwargs
     ):
         self._remote_test_root = None
@@ -151,6 +154,9 @@ class Perftest(object):
             "project": project,
             "verbose": verbose,
             "extra_summary_methods": extra_summary_methods,
+            "benchmark_repository": benchmark_repository,
+            "benchmark_revision": benchmark_revision,
+            "benchmark_branch": benchmark_branch,
         }
 
         self.firefox_android_apps = FIREFOX_ANDROID_APPS

testing/raptor/raptor/raptor.py

@@ -138,6 +138,9 @@ def main(args=sys.argv[1:]):
             verbose=args.verbose,
             fission=args.fission,
             extra_summary_methods=args.extra_summary_methods,
+            benchmark_repository=args.benchmark_repository,
+            benchmark_revision=args.benchmark_revision,
+            benchmark_branch=args.benchmark_branch,
         )
     except Exception:
         traceback.print_exc()

testing/raptor/test/test_cmdline.py

@@ -33,6 +33,9 @@ def test_verify_options(filedir):
         test_bytecode_cache=False,
         webext=False,
         extra_prefs=[],
+        benchmark_repository=None,
+        benchmark_revision=None,
+        benchmark_branch=None,
     )
     parser = ArgumentParser()
 
@@ -61,6 +64,9 @@ def test_verify_options(filedir):
         test_bytecode_cache=False,
         webext=False,
         extra_prefs=[],
+        benchmark_repository=None,
+        benchmark_revision=None,
+        benchmark_branch=None,
     )
     verify_options(parser, args)  # assert no exception
 
@@ -83,6 +89,9 @@ def test_verify_options(filedir):
         test_bytecode_cache=False,
         webext=False,
         extra_prefs=[],
+        benchmark_repository=None,
+        benchmark_revision=None,
+        benchmark_branch=None,
     )
     verify_options(parser, args)  # assert no exception
 
@@ -105,6 +114,9 @@ def test_verify_options(filedir):
         test_bytecode_cache=False,
         webext=False,
         extra_prefs=[],
+        benchmark_repository=None,
+        benchmark_revision=None,
+        benchmark_branch=None,
     )
     verify_options(parser, args)  # assert no exception
 
@@ -127,6 +139,9 @@ def test_verify_options(filedir):
         test_bytecode_cache=False,
         webext=False,
         extra_prefs=[],
+        benchmark_repository=None,
+        benchmark_revision=None,
+        benchmark_branch=None,
     )
     verify_options(parser, args)  # assert no exception
 
@@ -149,6 +164,9 @@ def test_verify_options(filedir):
         test_bytecode_cache=False,
         webext=False,
         extra_prefs=[],
+        benchmark_repository=None,
+        benchmark_revision=None,
+        benchmark_branch=None,
     )
     parser = ArgumentParser()