Bug 1635136 - add a --push-to-try option r=aerickson

This patch is adding an option to push a perftest run in the CI.

It's based on :
- sparse profiles
- push_to_try
- options passed through try_task_config.json

Differential Revision: https://phabricator.services.mozilla.com/D74115

build/sparse-profiles/perftest

@@ -0,0 +1,5 @@
+%include build/sparse-profiles/docker-image
+
+[include]
+path:tools/lint/eslint/
+path:testing/performance

python/mozboot/mozboot/util.py

@@ -6,9 +6,9 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 import hashlib
 import os
-
 import six
 
+
 here = os.path.join(os.path.dirname(__file__))
 
 

python/mozperftest/mozperftest/argparser.py

@@ -46,6 +46,22 @@ class Options:
         },
         "--hooks": {"type": str, "default": "", "help": "Python hooks"},
         "--verbose": {"action": "store_true", "default": False, "help": "Verbose mode"},
+        "--push-to-try": {
+            "action": "store_true",
+            "default": False,
+            "help": "Pushin the test to try",
+        },
+        "--try-platform": {
+            "type": str,
+            "default": "g5",
+            "help": "Platform to use on try",
+            "choices": ["g5", "pixel2", "linux", "mac", "win"],
+        },
+        "--on-try": {
+            "action": "store_true",
+            "default": False,
+            "help": "Running the test on try",
+        },
     }
 
     args = copy.deepcopy(general_args)

python/mozperftest/mozperftest/mach_commands.py

@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import os
+import sys
 from functools import partial
 import subprocess
 
@@ -9,6 +10,9 @@ from mach.decorators import CommandProvider, Command
 from mozbuild.base import MachCommandBase, MachCommandConditions as conditions
 
 
+_TRY_PLATFORMS = {"g5": "perftest-android-hw-g5", "p2": "perftest-android-hw-p2"}
+
+
 def get_perftest_parser():
     from mozperftest import PerftestArgumentParser
 
@@ -25,6 +29,39 @@ class Perftest(MachCommandBase):
         parser=get_perftest_parser,
     )
     def run_perftest(self, **kwargs):
+        push_to_try = kwargs.pop("push_to_try", False)
+        if push_to_try:
+            from pathlib import Path
+
+            sys.path.append(str(Path(self.topsrcdir, "tools", "tryselect")))
+
+            from tryselect.push import push_to_try
+
+            platform = kwargs.pop("try_platform")
+            if platform not in _TRY_PLATFORMS:
+                # we can extend platform support here: linux, win, macOs, pixel2
+                # by adding more jobs in taskcluster/ci/perftest/kind.yml
+                # then picking up the right one here
+                raise NotImplementedError("%r not supported yet" % platform)
+
+            perftest_parameters = {}
+            parser = get_perftest_parser()()
+            for name, value in kwargs.items():
+                # ignore values that are set to default
+                if parser.get_default(name) == value:
+                    continue
+                perftest_parameters[name] = value
+
+            parameters = {"try_options": {"perftest": perftest_parameters}}
+            try_config = {"tasks": [_TRY_PLATFORMS[platform]]}
+            parameters["try_task_config"] = try_config
+            parameters["try_mode"] = "try_task_config"
+
+            task_config = {"parameters": parameters, "version": 2}
+            push_to_try("perftest", "perftest", try_task_config=task_config)
+            return
+
+        # run locally
         MachCommandBase._activate_virtualenv(self)
 
         from mozperftest.runner import run_tests

python/mozperftest/mozperftest/runner.py

@@ -3,8 +3,25 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Pure Python runner so we can execute perftest in the CI without
-depending on the mach toolchain, that is not fully available in
+depending on a full mach toolchain, that is not fully available in
 all worker environments.
+
+This runner can be executed in two different ways:
+
+- by calling run_tests() from the mach command
+- by executing this module directly
+
+When the module is executed directly, if the --on-try option is used,
+it will fetch arguments from Tascluster's parameters, that were
+populated via a local --push-to-try call.
+
+The --push-to-try flow is:
+
+- a user calls ./mach perftest --push-to-try --option1 --option2
+- a new push to try commit is made and includes all options in its parameters
+- a generic TC job triggers the perftest by calling this module with --on-try
+- run_test() grabs the parameters artifact and converts them into args for
+  perftest
 """
 import sys
 import os
@@ -36,6 +53,7 @@ SEARCH_PATHS = [
     "third_party/python/redo",
     "third_party/python/requests",
     "third_party/python/six",
+    "third_party/python/zipp",
 ]
 
 
@@ -45,32 +63,51 @@ if "SHELL" not in os.environ:
 
 
 def _setup_path():
+    """Adds all dependencies in the path.
+
+    This is done so the runner can be used with no prior
+    install in all execution environments.
+    """
     for path in SEARCH_PATHS:
         path = os.path.abspath(path)
+        path = os.path.join(SRC_ROOT, path)
         if not os.path.exists(path):
             raise IOError("Can't find %s" % path)
-        sys.path.insert(0, os.path.join(SRC_ROOT, path))
+        sys.path.insert(0, path)
 
 
-def main():
-    _setup_path()
+def _get_params():
+    """Fetches the parameters.yml artifact and returns its content.
+    """
+    # XXX - this already exists in taskcluster code
+    # in a more robust way, but for now let's not depend on it.
+    import requests
+    import yaml
 
-    from mozbuild.base import MachCommandBase, MozbuildObject
-    from mozperftest import PerftestArgumentParser
-    from mozboot.util import get_state_dir
-
-    config = MozbuildObject.from_environment()
-    config.topdir = config.topsrcdir
-    config.cwd = os.getcwd()
-    config.state_dir = get_state_dir()
-    mach_cmd = MachCommandBase(config)
-    parser = PerftestArgumentParser(description="vanilla perftest")
-    args = parser.parse_args()
-    run_tests(mach_cmd, **dict(args._get_kwargs()))
+    root = os.environ.get(
+        "TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com"
+    )
+    # set by require-decision-task-id
+    tid = os.environ["DECISION_TASK_ID"]
+    url = root + "/api/queue/v1/task/%s/artifacts/public/parameters.yml" % tid
+    response = requests.get(url)
+    return yaml.load(response.text)
 
 
 def run_tests(mach_cmd, **kwargs):
+    """This tests runner can be used directly via main or via Mach.
+
+    When the --on-try option is used, the test runner looks for the
+    `parameters.yml` artifact that contains all options passed by
+    the used via a ./mach perftest --push-to-try call.
+    """
     _setup_path()
+    on_try = kwargs.pop("on_try", False)
+
+    # trying to get the arguments from the task params
+    if on_try:
+        params = _get_params()
+        kwargs.update(params["try_options"]["perftest"])
 
     from mozperftest.utils import build_test_list, install_package
     from mozperftest import MachEnvironment, Metadata
@@ -98,5 +135,24 @@ def run_tests(mach_cmd, **kwargs):
         env.run_hook("after_runs")
 
 
+def main():
+    """Used when the runner is directly called from the shell
+    """
+    _setup_path()
+
+    from mozbuild.base import MachCommandBase, MozbuildObject
+    from mozperftest import PerftestArgumentParser
+    from mozboot.util import get_state_dir
+
+    config = MozbuildObject.from_environment()
+    config.topdir = config.topsrcdir
+    config.cwd = os.getcwd()
+    config.state_dir = get_state_dir()
+    mach_cmd = MachCommandBase(config)
+    parser = PerftestArgumentParser(description="vanilla perftest")
+    args = parser.parse_args()
+    run_tests(mach_cmd, **dict(args._get_kwargs()))
+
+
 if __name__ == "__main__":
     sys.exit(main())

taskcluster/ci/perftest/kind.yml

@@ -0,0 +1,58 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - toolchain
+    - build
+
+transforms:
+    - taskgraph.transforms.source_test:transforms
+    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.task:transforms
+
+job-defaults:
+    run-on-projects:
+        - try
+    treeherder:
+        kind: other
+        tier: 3
+    require-decision-task-id: true
+    worker:
+        taskcluster-proxy: true
+        max-run-time: 10800
+        artifacts:
+            - type: directory
+              name: public/build
+              path: artifacts
+    fetches:
+        toolchain:
+            - linux64-node-10
+            - linux64-geckodriver
+    run:
+        sparse-profile: perftest
+        run-as-root: true
+        using: run-task
+        command: >-
+            cd $GECKO_PATH &&
+            python3 python/mozperftest/mozperftest/runner.py
+            --on-try
+            --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
+            --output $MOZ_FETCHES_DIR/../artifacts
+
+jobs:
+    android-hw-g5:
+        worker-type: t-bitbar-gw-perf-g5
+        description: Run ./mach perftest on a G5
+        treeherder:
+            symbol: perftest-g5
+        platform: android-hw-g5-7-0-arm7-api-16/opt
+
+    android-hw-p2:
+        worker-type: t-bitbar-gw-perf-p2
+        description: Run ./mach perftest on a Pixel 2
+        treeherder:
+            symbol: perftest-p2
+        platform: android-hw-p2-8-0-android-aarch64/opt

taskcluster/docs/kinds.rst

@@ -280,6 +280,10 @@ push-apk-checks
 ---------------
 Runs the checks done in push-apk to ensure APKs are sane before submitting them
 
+perftest
+--------
+Runs performance tests using mozperftest.
+
 release-balrog-submit-toplevel
 ------------------------------
 Toplevel tasks are responsible for submitting metadata to Balrog that is not specific to any

taskcluster/taskgraph/transforms/job/common.py

@@ -81,7 +81,7 @@ def support_vcs_checkout(config, job, taskdesc, sparse=False):
     worker = job['worker']
     is_mac = worker['os'] == 'macosx'
     is_win = worker['os'] == 'windows'
-    is_linux = worker['os'] == 'linux'
+    is_linux = worker['os'] == 'linux' or 'linux-bitbar'
     assert is_mac or is_win or is_linux
 
     if is_win:

tools/tryselect/util/manage_estimates.py

@@ -8,9 +8,9 @@ import os
 import requests
 import json
 from datetime import datetime, timedelta
-
 import six
 
+
 TASK_DURATION_URL = 'https://storage.googleapis.com/mozilla-mach-data/task_duration_history.json'
 GRAPH_QUANTILES_URL = 'https://storage.googleapis.com/mozilla-mach-data/machtry_quantiles.csv'
 from .estimates import TASK_DURATION_CACHE, GRAPH_QUANTILE_CACHE, TASK_DURATION_TAG_FILE