Bug 1665556 - make the try-platform option handle multiple platforms r=sparky

Differential Revision: https://phabricator.services.mozilla.com/D90550

netwerk/test/unit/test_http3_perf.js

@@ -16,6 +16,7 @@ var perfMetadata = {
       ],
       xpcshell_cycles: 13,
       verbose: true,
+      try_platform: ["linux", "mac"],
     },
   },
   tags: ["network", "http3", "quic"],

python/mozperftest/mozperftest/argparser.py

@@ -62,8 +62,9 @@ class Options:
             "help": "Pushin the test to try",
         },
         "--try-platform": {
+            "nargs": "*",
             "type": str,
-            "default": "g5",
+            "default": "linux",
             "help": "Platform to use on try",
             "choices": ["g5", "pixel2", "linux", "mac", "win"],
         },

python/mozperftest/mozperftest/mach_commands.py

@@ -6,17 +6,20 @@ import sys
 from functools import partial
 import subprocess
 import shlex
+import json
 
 from mach.decorators import CommandProvider, Command, CommandArgument
 from mozbuild.base import MachCommandBase, MachCommandConditions as conditions
 
 
 _TRY_PLATFORMS = {
-    "g5": "perftest-android-hw-g5",
-    "p2": "perftest-android-hw-p2",
-    "linux": "perftest-linux-try",
-    "mac": "perftest-macosx-try",
-    "win": "perftest-windows-try",
+    "g5-browsertime": "perftest-android-hw-g5-browsertime",
+    "p2-browsertime": "perftest-android-hw-p2-browsertime",
+    "linux-xpcshell": "perftest-linux-try-xpcshell",
+    "mac-xpcshell": "perftest-macosx-try-xpcshell",
+    "linux-browsertime": "perftest-linux-try-browsertime",
+    "mac-browsertime": "perftest-macosx-try-browsertime",
+    "win-browsertimee": "perftest-windows-try-browsertime",
 }
 
 
@@ -57,6 +60,7 @@ class Perftest(MachCommandBase):
 
             resolver = self._spawn(TestResolver)
             test_objects = list(resolver.resolve_tests(paths=None, flavor="perftest"))
+            selected = select(test_objects)
 
             def full_path(selection):
                 __, script_name, __, location = selection.split(" ")
@@ -68,7 +72,7 @@ class Perftest(MachCommandBase):
                     )
                 )
 
-            kwargs["tests"] = [full_path(s) for s in select(test_objects)]
+            kwargs["tests"] = [full_path(s) for s in selected]
 
             if kwargs["tests"] == []:
                 print("\nNo selection. Bye!")
@@ -80,7 +84,6 @@ class Perftest(MachCommandBase):
 
         sel = "\n".join(kwargs["tests"])
         print("\nGood job! Best selection.\n%s" % sel)
-
         # if the script is xpcshell, we can force the flavor here
         # XXX on multi-selection,  what happens if we have seeveral flavors?
         try:
@@ -105,31 +108,51 @@ class Perftest(MachCommandBase):
 
             from tryselect.push import push_to_try
 
-            platform = kwargs.pop("try_platform")
-            if platform not in _TRY_PLATFORMS:
-                # we can extend platform support here: linux, win, macOs, pixel2
-                # by adding more jobs in taskcluster/ci/perftest/kind.yml
-                # then picking up the right one here
-                raise NotImplementedError("%r not supported yet" % platform)
-
             perftest_parameters = {}
             args = script_info.update_args(**original_parser.get_user_args(kwargs))
+            platform = args.pop("try_platform", "linux")
+            if isinstance(platform, str):
+                platform = [platform]
+
+            platform = [
+                "%s-%s" % (plat, script_info.script_type.name) for plat in platform
+            ]
+
+            for plat in platform:
+                if plat not in _TRY_PLATFORMS:
+                    # we can extend platform support here: linux, win, macOs, pixel2
+                    # by adding more jobs in taskcluster/ci/perftest/kind.yml
+                    # then picking up the right one here
+                    raise NotImplementedError(
+                        "%r doesn't exist or is not yet supported" % plat
+                    )
+
+            def relative(path):
+                if path.startswith(self.topsrcdir):
+                    return path[len(self.topsrcdir) :].lstrip(os.sep)
+                return path
 
             for name, value in args.items():
                 # ignore values that are set to default
                 if original_parser.get_default(name) == value:
                     continue
+                if name == "tests":
+                    value = [relative(path) for path in value]
                 perftest_parameters[name] = value
 
             parameters = {
                 "try_task_config": {
-                    "tasks": [_TRY_PLATFORMS[platform]],
+                    "tasks": [_TRY_PLATFORMS[plat] for plat in platform],
                     "perftest-options": perftest_parameters,
                 },
                 "try_mode": "try_task_config",
             }
 
             task_config = {"parameters": parameters, "version": 2}
+            if args["verbose"]:
+                print("Pushing run to try...")
+                print(json.dumps(task_config, indent=4, sort_keys=True))
+
             push_to_try("perftest", "perftest", try_task_config=task_config)
             return
 

python/mozperftest/mozperftest/runner.py

@@ -101,6 +101,8 @@ def run_tests(mach_cmd, kwargs, client_args):
     # trying to get the arguments from the task params
     if on_try:
         try_options = json.loads(os.environ["PERFTEST_OPTIONS"])
+        print("Loading options from $PERFTEST_OPTIONS")
+        print(json.dumps(try_options, indent=4, sort_keys=True))
         kwargs.update(try_options)
 
     from mozperftest.utils import build_test_list

python/mozperftest/mozperftest/script.py

@@ -247,4 +247,6 @@ class ScriptInfo(defaultdict):
         result = options.get("default", {})
         result.update(options.get(simple_platform(), {}))
         result.update(args)
+        if self.script_type == ScriptType.xpcshell:
+            result["flavor"] = "xpcshell"
         return result

python/mozperftest/mozperftest/tests/test_mach_commands.py

@@ -4,12 +4,14 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import mozunit
 import os
+import sys
 from unittest import mock
 import tempfile
 import shutil
 from contextlib import contextmanager
 from pathlib import Path
 
+import pytest
 from mach.registrar import Registrar
 
 Registrar.categories = {"testing": []}
@@ -42,6 +44,20 @@ class _TestMachEnvironment(MachEnvironment):
         pass
 
 
+@contextmanager
+def running_on_try(on_try=True):
+    old = utils.ON_TRY
+    utils.ON_TRY = on_try
+    try:
+        if on_try:
+            with temporary_env(MOZ_AUTOMATION="1"):
+                yield
+        else:
+            yield
+    finally:
+        utils.ON_TRY = old
+
+
 @contextmanager
 def _get_command(klass=Perftest):
     from mozbuild.base import MozbuildObject
@@ -56,9 +72,23 @@ def _get_command(klass=Perftest):
         log_manager = mock.Mock()
         state_dir = tempfile.mkdtemp()
 
+    # used to make arguments passed by the test as
+    # being set by the user.
+    def _run_perftest(func):
+        def _run(**kwargs):
+            parser.set_by_user = list(kwargs.keys())
+            return func(**kwargs)
+
+        return _run
+
     try:
         obj = klass(context())
-        obj.get_parser = lambda: PerftestArgumentParser()
+        parser = PerftestArgumentParser()
+        obj.get_parser = lambda: parser
+
+        if isinstance(obj, Perftest):
+            obj.run_perftest = _run_perftest(obj.run_perftest)
+
         yield obj
     finally:
         shutil.rmtree(context.state_dir)
@@ -101,6 +131,38 @@ def test_push_command(push_to_try, venv):
         # XXX add assertions
 
 
+@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
+@mock.patch("mozperftest.mach_commands.MachCommandBase.activate_virtualenv")
+@mock.patch("tryselect.push.push_to_try")
+def test_push_command_unknown_platforms(push_to_try, venv):
+    # full stop when a platform is unknown
+    with _get_command() as test, pytest.raises(NotImplementedError):
+        test.run_perftest(
+            tests=[EXAMPLE_TEST],
+            flavor="desktop-browser",
+            push_to_try=True,
+            try_platform=["solaris", "linux", "mac"],
+        )
+
+
+@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
+@mock.patch("mozperftest.mach_commands.MachCommandBase.activate_virtualenv")
+@mock.patch("tryselect.push.push_to_try")
+def test_push_command_several_platforms(push_to_try, venv):
+    with running_on_try(False), _get_command() as test:  # , silence(test):
+        test.run_perftest(
+            tests=[EXAMPLE_TEST],
+            flavor="desktop-browser",
+            push_to_try=True,
+            try_platform=["linux", "mac"],
+        )
+        push_to_try.assert_called()
+        name, args, kwargs = push_to_try.mock_calls[0]
+        params = kwargs["try_task_config"]["parameters"]["try_task_config"]
+        assert "perftest-linux-try-browsertime" in params["tasks"]
+        assert "perftest-macosx-try-browsertime" in params["tasks"]
+
+
 @mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
 @mock.patch("mozperftest.mach_commands.MachCommandBase.activate_virtualenv")
 def test_doc_flavor(mocked_func):
@@ -110,17 +172,33 @@ def test_doc_flavor(mocked_func):
 
 @mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
 @mock.patch("mozperftest.mach_commands.MachCommandBase.activate_virtualenv")
-@mock.patch("mozperftest.mach_commands.PerftestTests._run_python_script")
+@mock.patch("mozperftest.mach_commands.PerftestTests._run_script")
 def test_test_runner(*mocked):
+    with running_on_try(False), _get_command(PerftestTests) as test:
+        test.run_tests(tests=[EXAMPLE_TEST], verbose=True)
+
+
+@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
+@mock.patch("mozperftest.mach_commands.MachCommandBase.activate_virtualenv")
+@mock.patch("mozperftest.mach_commands.PerftestTests._run_python_script")
+def test_test_runner_on_try(*mocked):
     # simulating on try to run the paths parser
-    old = utils.ON_TRY
-    utils.ON_TRY = True
-    with _get_command(PerftestTests) as test, silence(test), temporary_env(
-        MOZ_AUTOMATION="1"
-    ):
+    with running_on_try(), _get_command(PerftestTests) as test:
         test.run_tests(tests=[EXAMPLE_TEST])
 
-    utils.ON_TRY = old
+
+@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
+@mock.patch("mozperftest.mach_commands.MachCommandBase.activate_virtualenv")
+@mock.patch("mozperftest.mach_commands.PerftestTests._run_script")
+def test_test_runner_coverage(*mocked):
+    # simulating with coverage not installed
+    with running_on_try(False), _get_command(PerftestTests) as test:
+        old = list(sys.meta_path)
+        sys.meta_path = []
+        try:
+            test.run_tests(tests=[EXAMPLE_TEST])
+        finally:
+            sys.meta_path = old
 
 
 @mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
@@ -165,24 +243,12 @@ def resolve_tests(tests=None):
     return _resolve
 
 
-@contextmanager
-def not_on_try():
-    # forcing ON_TRY to false, so when the test runs in the CI,
-    # we test the desktop behavior (fzf is a UI that is deactivated in the CI)
-    old = utils.ON_TRY
-    utils.ON_TRY = False
-    try:
-        yield
-    finally:
-        utils.ON_TRY = old
-
-
 @mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
 @mock.patch("mozperftest.mach_commands.MachCommandBase.activate_virtualenv")
 @mock.patch("mozperftest.fzf.fzf.select", new=fzf_selection)
 @mock.patch("moztest.resolve.TestResolver.resolve_tests", new=resolve_tests())
 def test_fzf_flavor(*mocked):
-    with not_on_try(), _get_command() as test:  # , silence():
+    with running_on_try(False), _get_command() as test:  # , silence():
         test.run_perftest(flavor="desktop-browser")
 
 
@@ -191,7 +257,7 @@ def test_fzf_flavor(*mocked):
 @mock.patch("mozperftest.fzf.fzf.select", new=fzf_selection)
 @mock.patch("moztest.resolve.TestResolver.resolve_tests", new=resolve_tests([]))
 def test_fzf_nothing_selected(*mocked):
-    with not_on_try(), _get_command() as test, silence():
+    with running_on_try(False), _get_command() as test, silence():
         test.run_perftest(flavor="desktop-browser")
 
 

python/mozperftest/mozperftest/utils.py

@@ -154,6 +154,14 @@ def install_package(virtualenv_manager, package, ignore_failure=False):
     return False
 
 
+# on try, we create tests packages where tests, like
+# xpcshell tests, don't have the same path.
+# see - python/mozbuild/mozbuild/action/test_archive.py
+# this mapping will map paths when running there.
+# The key is the source path, and the value the ci path
+_TRY_MAPPING = {Path("netwerk"): Path("xpcshell", "tests", "netwerk")}
+
+
 def build_test_list(tests):
     """Collects tests given a list of directories, files and URLs.
 
@@ -174,13 +182,24 @@ def build_test_list(tests):
             res.append(str(target))
             continue
 
-        test = Path(test)
+        p_test = Path(test)
+        if ON_TRY and not p_test.resolve().exists():
+            # until we have pathlib.Path.is_relative_to() (3.9)
+            for src_path, ci_path in _TRY_MAPPING.items():
+                src_path, ci_path = str(src_path), str(ci_path)
+                if test.startswith(src_path):
+                    p_test = Path(test.replace(src_path, ci_path))
+                    break
+
+        test = p_test.resolve()
 
         if test.is_file():
             res.append(str(test))
         elif test.is_dir():
             for file in test.rglob("perftest_*.js"):
                 res.append(str(file))
+        else:
+            raise FileNotFoundError(str(test))
     res.sort()
     return res, temp_dir
 

taskcluster/ci/perftest/linux.yml

@@ -18,10 +18,27 @@ job-defaults:
     require-build:
         linux64-shippable/opt: build-linux64-shippable/opt
 
-try:
+try-xpcshell:
     description: Run ./mach perftest on Linux
     treeherder:
-        symbol: perftest(linux)
+        symbol: perftest(linux-xpcshell)
+    run:
+        command: >-
+            mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
+            cd $MOZ_FETCHES_DIR &&
+            python3.8 python/mozperftest/mozperftest/runner.py
+            --on-try
+            --flavor desktop-browser
+            --output $MOZ_FETCHES_DIR/../artifacts
+            --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell
+            --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json
+            --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node
+            --xpcshell-xre-path ${MOZ_FETCHES_DIR}/firefox
+
+try-browsertime:
+    description: Run ./mach perftest on Linux
+    treeherder:
+        symbol: perftest(linux-bt)
     run:
         command: >-
             mkdir -p $MOZ_FETCHES_DIR/../artifacts &&

taskcluster/ci/perftest/macosx.yml

@@ -19,10 +19,28 @@ job-defaults:
     require-build:
         macosx64-shippable/opt: build-macosx64-shippable/opt
 
-try:
+try-xpcshell:
     description: Run ./mach perftest on macOs
     treeherder:
-        symbol: perftest(macos)
+        symbol: perftest(macos-xpcshell)
+    run:
+        command: >-
+            mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
+            cd $MOZ_FETCHES_DIR &&
+            python3 -m venv . &&
+            bin/python3 python/mozperftest/mozperftest/runner.py
+            --on-try
+            --flavor desktop-browser
+            --output $MOZ_FETCHES_DIR/../artifacts
+            --xpcshell-binary ${MOZ_FETCHES_DIR}/bin/xpcshell
+            --xpcshell-mozinfo ${MOZ_FETCHES_DIR}/target.mozinfo.json
+            --xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node
+            --xpcshell-xre-path ${MOZ_FETCHES_DIR}/target.dmg
+
+try-browsertime:
+    description: Run ./mach perftest on macOs
+    treeherder:
+        symbol: perftest(macos-bt)
     run:
         command: >-
             mkdir -p $MOZ_FETCHES_DIR/../artifacts &&

taskcluster/ci/perftest/windows.yml

@@ -15,10 +15,10 @@ job-defaults:
     require-build:
         win64-shippable/opt: build-win64-shippable/opt
 
-try:
+try-browsertime:
     description: Run ./mach perftest on windows
     treeherder:
-        symbol: perftest(win)
+        symbol: perftest(win-bt)
     run:
         command: >-
             mkdir -p $MOZ_FETCHES_DIR/../artifacts &&