Bug 1634349 - Split a pure Python runner r=sparky

In order to be able to run outside mach, we want to split the runner so we can
call it directly from Python.

Differential Revision: https://phabricator.services.mozilla.com/D73252

python/mozperftest/mozperftest/argparser.py

@@ -28,7 +28,7 @@ class Options:
         "--flavor": {
             "choices": FLAVORS,
             "metavar": "{{{}}}".format(", ".join(FLAVORS)),
-            "default": None,
+            "default": "script",
             "help": "Only run tests of this flavor.",
         },
         "tests": {

python/mozperftest/mozperftest/browser/browsertime/runner.py

@@ -12,6 +12,7 @@ import shutil
 from mozbuild.util import mkdir
 import mozpack.path as mozpath
 
+from mozperftest.utils import install_package
 from mozperftest.browser.noderunner import NodeRunner
 from mozperftest.browser.browsertime.setup import (
     system_prerequisites,
@@ -106,8 +107,7 @@ class BrowsertimeRunner(NodeRunner):
 
         # installing Python deps on the fly
         for dep in ("Pillow==%s" % PILLOW_VERSION, "pyssim==%s" % PYSSIM_VERSION):
-            if self._need_install(dep):
-                self.virtualenv_manager._run_pip(["install", dep])
+            install_package(self.virtualenv_manager, dep)
 
         # check if the browsertime package has been deployed correctly
         # for this we just check for the browsertime directory presence

python/mozperftest/mozperftest/mach_commands.py

@@ -1,8 +1,6 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-import os
-import random
 from functools import partial
 from mach.decorators import CommandProvider, Command
 from mozbuild.base import MachCommandBase, MachCommandConditions as conditions
@@ -16,25 +14,6 @@ def get_perftest_parser():
 
 @CommandProvider
 class Perftest(MachCommandBase):
-    def _build_test_list(self, tests, randomized=False):
-        res = []
-        for test in tests:
-            if os.path.isfile(test):
-                tests.append(test)
-            elif os.path.isdir(test):
-                for root, dirs, files in os.walk(test):
-                    for file in files:
-                        if not file.startswith("perftest"):
-                            continue
-                        res.append(os.path.join(root, file))
-        if not randomized:
-            res.sort()
-        else:
-            # random shuffling is used to make sure
-            # we don't always run tests in the same order
-            random.shuffle(res)
-        return res
-
     @Command(
         "perftest",
         category="testing",
@@ -42,36 +21,9 @@ class Perftest(MachCommandBase):
         description="Run any flavor of perftest",
         parser=get_perftest_parser,
     )
-    def run_perftest(
-        self, flavor="script", test_objects=None, resolve_tests=True, **kwargs
-    ):
-
+    def run_perftest(self, **kwargs):
         MachCommandBase._activate_virtualenv(self)
-        kwargs["tests"] = self._build_test_list(
-            kwargs["tests"], randomized=flavor != "doc"
-        )
 
-        if flavor == "doc":
-            from mozperftest.utils import install_package
+        from mozperftest.runner import run_tests
 
-            location = os.path.join(self.topsrcdir, "third_party", "python", "esprima")
-            install_package(self.virtualenv_manager, location)
-
-            from mozperftest.scriptinfo import ScriptInfo
-
-            for test in kwargs["tests"]:
-                print(ScriptInfo(test))
-            return
-
-        from mozperftest import MachEnvironment, Metadata
-
-        kwargs["test_objects"] = test_objects
-        kwargs["resolve_tests"] = resolve_tests
-        env = MachEnvironment(self, flavor, **kwargs)
-        metadata = Metadata(self, env, flavor)
-        env.run_hook("before_runs")
-        try:
-            with env.frozen() as e:
-                e.run(metadata)
-        finally:
-            env.run_hook("after_runs")
+        run_tests(mach_cmd=self, **kwargs)

python/mozperftest/mozperftest/runner.py

@@ -0,0 +1,91 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Pure Python runner so we can execute perftest in the CI without
+depending on the mach toolchain, that is not fully available in
+all worker environments.
+"""
+import sys
+import os
+
+
+HERE = os.path.dirname(__file__)
+SRC_ROOT = os.path.join(HERE, "..", "..", "..")
+SEARCH_PATHS = [
+    "python/mach",
+    "python/mozboot",
+    "python/mozbuild",
+    "python/mozperftest",
+    "python/mozterm",
+    "python/mozversioncontrol",
+    "testing/mozbase/mozdevice",
+    "testing/mozbase/mozfile",
+    "testing/mozbase/mozinfo",
+    "testing/mozbase/mozlog",
+    "testing/mozbase/mozprocess",
+    "testing/mozbase/mozprofile",
+    "testing/mozbase/mozproxy",
+    "third_party/python/dlmanager",
+    "third_party/python/esprima",
+    "third_party/python/pyyaml/lib3",
+    "third_party/python/redo",
+    "third_party/python/requests",
+    "third_party/python/six",
+]
+
+
+# XXX need to make that for all systems flavors
+if "SHELL" not in os.environ:
+    os.environ["SHELL"] = "/bin/bash"
+
+
+def main():
+    for path in SEARCH_PATHS:
+        path = os.path.abspath(path)
+        if not os.path.exists(path):
+            raise IOError("Can't find %s" % path)
+        sys.path.insert(0, os.path.join(SRC_ROOT, path))
+
+    from mozbuild.base import MachCommandBase, MozbuildObject
+    from mozperftest import PerftestArgumentParser
+    from mozboot.util import get_state_dir
+
+    config = MozbuildObject.from_environment()
+    config.topdir = config.topsrcdir
+    config.cwd = os.getcwd()
+    config.state_dir = get_state_dir()
+    mach_cmd = MachCommandBase(config)
+    parser = PerftestArgumentParser(description="vanilla perftest")
+    args = parser.parse_args()
+    run_tests(mach_cmd, **dict(args._get_kwargs()))
+
+
+def run_tests(mach_cmd, **kwargs):
+    from mozperftest.utils import build_test_list, install_package
+    from mozperftest import MachEnvironment, Metadata
+
+    flavor = kwargs["flavor"]
+    kwargs["tests"] = build_test_list(kwargs["tests"], randomized=flavor != "doc")
+
+    if flavor == "doc":
+        install_package(mach_cmd.virtualenv_manager, "esprima")
+
+        from mozperftest.scriptinfo import ScriptInfo
+
+        for test in kwargs["tests"]:
+            print(ScriptInfo(test))
+        return
+
+    env = MachEnvironment(mach_cmd, **kwargs)
+    metadata = Metadata(mach_cmd, env, flavor)
+    env.run_hook("before_runs")
+    try:
+        with env.frozen() as e:
+            e.run(metadata)
+    finally:
+        env.run_hook("after_runs")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

python/mozperftest/mozperftest/tests/support.py

@@ -47,12 +47,13 @@ def get_running_env(**kwargs):
     mach_cmd._mach_context.state_dir = tempfile.mkdtemp()
 
     mach_args = {
+        "flavor": "script",
         "test_objects": None,
         "resolve_tests": True,
         "browsertime-clobber": False,
         "browsertime-install-url": None,
     }
     mach_args.update(kwargs)
-    env = MachEnvironment(mach_cmd, "script", **mach_args)
+    env = MachEnvironment(mach_cmd, **mach_args)
     metadata = Metadata(mach_cmd, env, "script")
     return mach_cmd, metadata, env

python/mozperftest/mozperftest/tests/test_browsertime.py

@@ -17,6 +17,7 @@ def fetch(self, url):
     return os.path.join(HERE, "fetched_artifact.zip")
 
 
+@mock.patch("mozperftest.browser.browsertime.runner.install_package")
 @mock.patch(
     "mozperftest.browser.noderunner.NodeRunner.verify_node_install", new=lambda x: True
 )
@@ -25,7 +26,7 @@ def fetch(self, url):
     "mozperftest.browser.browsertime.runner.BrowsertimeRunner._setup_node_packages",
     new=lambda x, y: None,
 )
-def test_browser():
+def test_browser(*mocked):
     mach_cmd, metadata, env = get_running_env()
     browser = env.layers[BROWSER]
     env.set_arg("tests", [EXAMPLE_TEST])

python/mozperftest/mozperftest/tests/test_mach_commands.py

@@ -54,7 +54,7 @@ def _get_perftest():
 @mock.patch("mozperftest.mach_commands.MachCommandBase._activate_virtualenv")
 def test_command(mocked_func):
     with _get_perftest() as test:
-        test.run_perftest(tests=[EXAMPLE_TESTS_DIR])
+        test.run_perftest(tests=[EXAMPLE_TESTS_DIR], flavor="script")
         # XXX add assertions
 
 

python/mozperftest/mozperftest/utils.py

@@ -5,6 +5,7 @@ import logging
 import contextlib
 import sys
 import os
+import random
 
 from six import StringIO
 
@@ -73,3 +74,25 @@ def install_package(virtualenv_manager, package):
             # already installed in this venv, we can skip
             return
     virtualenv_manager._run_pip(["install", package])
+
+
+def build_test_list(tests, randomized=False):
+    if isinstance(tests, str):
+        tests = [tests]
+    res = []
+    for test in tests:
+        if os.path.isfile(test):
+            res.append(test)
+        elif os.path.isdir(test):
+            for root, dirs, files in os.walk(test):
+                for file in files:
+                    if not file.startswith("perftest"):
+                        continue
+                    res.append(os.path.join(root, file))
+    if not randomized:
+        res.sort()
+    else:
+        # random shuffling is used to make sure
+        # we don't always run tests in the same order
+        random.shuffle(res)
+    return res