diff --git a/build/sparse-profiles/perftest b/build/sparse-profiles/perftest new file mode 100644 index 000000000000..48b2628398fa --- /dev/null +++ b/build/sparse-profiles/perftest @@ -0,0 +1,5 @@ +%include build/sparse-profiles/docker-image + +[include] +path:tools/lint/eslint/ +path:testing/performance diff --git a/python/mozboot/mozboot/util.py b/python/mozboot/mozboot/util.py index 950ea899428e..6dc5db5101d8 100644 --- a/python/mozboot/mozboot/util.py +++ b/python/mozboot/mozboot/util.py @@ -6,9 +6,9 @@ from __future__ import absolute_import, print_function, unicode_literals import hashlib import os - import six + here = os.path.join(os.path.dirname(__file__)) diff --git a/python/mozperftest/mozperftest/argparser.py b/python/mozperftest/mozperftest/argparser.py index 1b73dce6f04d..493d0b1644ae 100644 --- a/python/mozperftest/mozperftest/argparser.py +++ b/python/mozperftest/mozperftest/argparser.py @@ -46,6 +46,22 @@ class Options: }, "--hooks": {"type": str, "default": "", "help": "Python hooks"}, "--verbose": {"action": "store_true", "default": False, "help": "Verbose mode"}, + "--push-to-try": { + "action": "store_true", + "default": False, + "help": "Pushin the test to try", + }, + "--try-platform": { + "type": str, + "default": "g5", + "help": "Platform to use on try", + "choices": ["g5", "pixel2", "linux", "mac", "win"], + }, + "--on-try": { + "action": "store_true", + "default": False, + "help": "Running the test on try", + }, } args = copy.deepcopy(general_args) diff --git a/python/mozperftest/mozperftest/mach_commands.py b/python/mozperftest/mozperftest/mach_commands.py index 7d80a5516b8a..ea735e77f100 100644 --- a/python/mozperftest/mozperftest/mach_commands.py +++ b/python/mozperftest/mozperftest/mach_commands.py @@ -2,6 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os +import sys from functools import partial import subprocess @@ -9,6 +10,9 @@ from mach.decorators import CommandProvider, Command from mozbuild.base import MachCommandBase, MachCommandConditions as conditions +_TRY_PLATFORMS = {"g5": "perftest-android-hw-g5", "p2": "perftest-android-hw-p2"} + + def get_perftest_parser(): from mozperftest import PerftestArgumentParser @@ -25,6 +29,39 @@ class Perftest(MachCommandBase): parser=get_perftest_parser, ) def run_perftest(self, **kwargs): + push_to_try = kwargs.pop("push_to_try", False) + if push_to_try: + from pathlib import Path + + sys.path.append(str(Path(self.topsrcdir, "tools", "tryselect"))) + + from tryselect.push import push_to_try + + platform = kwargs.pop("try_platform") + if platform not in _TRY_PLATFORMS: + # we can extend platform support here: linux, win, macOs, pixel2 + # by adding more jobs in taskcluster/ci/perftest/kind.yml + # then picking up the right one here + raise NotImplementedError("%r not supported yet" % platform) + + perftest_parameters = {} + parser = get_perftest_parser()() + for name, value in kwargs.items(): + # ignore values that are set to default + if parser.get_default(name) == value: + continue + perftest_parameters[name] = value + + parameters = {"try_options": {"perftest": perftest_parameters}} + try_config = {"tasks": [_TRY_PLATFORMS[platform]]} + parameters["try_task_config"] = try_config + parameters["try_mode"] = "try_task_config" + + task_config = {"parameters": parameters, "version": 2} + push_to_try("perftest", "perftest", try_task_config=task_config) + return + + # run locally MachCommandBase._activate_virtualenv(self) from mozperftest.runner import run_tests diff --git a/python/mozperftest/mozperftest/runner.py b/python/mozperftest/mozperftest/runner.py index b403789a6a46..473909b453e5 100644 --- a/python/mozperftest/mozperftest/runner.py +++ b/python/mozperftest/mozperftest/runner.py @@ -3,8 +3,25 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Pure Python runner so we can execute perftest in the CI without -depending on the mach toolchain, that is not fully available in +depending on a full mach toolchain, that is not fully available in all worker environments. + +This runner can be executed in two different ways: + +- by calling run_tests() from the mach command +- by executing this module directly + +When the module is executed directly, if the --on-try option is used, +it will fetch arguments from Tascluster's parameters, that were +populated via a local --push-to-try call. + +The --push-to-try flow is: + +- a user calls ./mach perftest --push-to-try --option1 --option2 +- a new push to try commit is made and includes all options in its parameters +- a generic TC job triggers the perftest by calling this module with --on-try +- run_test() grabs the parameters artifact and converts them into args for + perftest """ import sys import os @@ -36,6 +53,7 @@ SEARCH_PATHS = [ "third_party/python/redo", "third_party/python/requests", "third_party/python/six", + "third_party/python/zipp", ] @@ -45,32 +63,51 @@ if "SHELL" not in os.environ: def _setup_path(): + """Adds all dependencies in the path. + + This is done so the runner can be used with no prior + install in all execution environments. + """ for path in SEARCH_PATHS: path = os.path.abspath(path) + path = os.path.join(SRC_ROOT, path) if not os.path.exists(path): raise IOError("Can't find %s" % path) - sys.path.insert(0, os.path.join(SRC_ROOT, path)) + sys.path.insert(0, path) -def main(): - _setup_path() +def _get_params(): + """Fetches the parameters.yml artifact and returns its content. + """ + # XXX - this already exists in taskcluster code + # in a more robust way, but for now let's not depend on it. + import requests + import yaml - from mozbuild.base import MachCommandBase, MozbuildObject - from mozperftest import PerftestArgumentParser - from mozboot.util import get_state_dir - - config = MozbuildObject.from_environment() - config.topdir = config.topsrcdir - config.cwd = os.getcwd() - config.state_dir = get_state_dir() - mach_cmd = MachCommandBase(config) - parser = PerftestArgumentParser(description="vanilla perftest") - args = parser.parse_args() - run_tests(mach_cmd, **dict(args._get_kwargs())) + root = os.environ.get( + "TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com" + ) + # set by require-decision-task-id + tid = os.environ["DECISION_TASK_ID"] + url = root + "/api/queue/v1/task/%s/artifacts/public/parameters.yml" % tid + response = requests.get(url) + return yaml.load(response.text) def run_tests(mach_cmd, **kwargs): + """This tests runner can be used directly via main or via Mach. + + When the --on-try option is used, the test runner looks for the + `parameters.yml` artifact that contains all options passed by + the used via a ./mach perftest --push-to-try call. + """ _setup_path() + on_try = kwargs.pop("on_try", False) + + # trying to get the arguments from the task params + if on_try: + params = _get_params() + kwargs.update(params["try_options"]["perftest"]) from mozperftest.utils import build_test_list, install_package from mozperftest import MachEnvironment, Metadata @@ -98,5 +135,24 @@ def run_tests(mach_cmd, **kwargs): env.run_hook("after_runs") +def main(): + """Used when the runner is directly called from the shell + """ + _setup_path() + + from mozbuild.base import MachCommandBase, MozbuildObject + from mozperftest import PerftestArgumentParser + from mozboot.util import get_state_dir + + config = MozbuildObject.from_environment() + config.topdir = config.topsrcdir + config.cwd = os.getcwd() + config.state_dir = get_state_dir() + mach_cmd = MachCommandBase(config) + parser = PerftestArgumentParser(description="vanilla perftest") + args = parser.parse_args() + run_tests(mach_cmd, **dict(args._get_kwargs())) + + if __name__ == "__main__": sys.exit(main()) diff --git a/taskcluster/ci/perftest/kind.yml b/taskcluster/ci/perftest/kind.yml new file mode 100644 index 000000000000..4ac5c42f3e86 --- /dev/null +++ b/taskcluster/ci/perftest/kind.yml @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + - build + +transforms: + - taskgraph.transforms.source_test:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + run-on-projects: + - try + treeherder: + kind: other + tier: 3 + require-decision-task-id: true + worker: + taskcluster-proxy: true + max-run-time: 10800 + artifacts: + - type: directory + name: public/build + path: artifacts + fetches: + toolchain: + - linux64-node-10 + - linux64-geckodriver + run: + sparse-profile: perftest + run-as-root: true + using: run-task + command: >- + cd $GECKO_PATH && + python3 python/mozperftest/mozperftest/runner.py + --on-try + --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver + --output $MOZ_FETCHES_DIR/../artifacts + +jobs: + android-hw-g5: + worker-type: t-bitbar-gw-perf-g5 + description: Run ./mach perftest on a G5 + treeherder: + symbol: perftest-g5 + platform: android-hw-g5-7-0-arm7-api-16/opt + + android-hw-p2: + worker-type: t-bitbar-gw-perf-p2 + description: Run ./mach perftest on a Pixel 2 + treeherder: + symbol: perftest-p2 + platform: android-hw-p2-8-0-android-aarch64/opt diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst index 1d571d37d3ba..5d26fab71712 100644 --- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -280,6 +280,10 @@ push-apk-checks --------------- Runs the checks done in push-apk to ensure APKs are sane before submitting them +perftest +-------- +Runs performance tests using mozperftest. + release-balrog-submit-toplevel ------------------------------ Toplevel tasks are responsible for submitting metadata to Balrog that is not specific to any diff --git a/taskcluster/taskgraph/transforms/job/common.py b/taskcluster/taskgraph/transforms/job/common.py index 0f28832567d2..90ce40689bc4 100644 --- a/taskcluster/taskgraph/transforms/job/common.py +++ b/taskcluster/taskgraph/transforms/job/common.py @@ -81,7 +81,7 @@ def support_vcs_checkout(config, job, taskdesc, sparse=False): worker = job['worker'] is_mac = worker['os'] == 'macosx' is_win = worker['os'] == 'windows' - is_linux = worker['os'] == 'linux' + is_linux = worker['os'] == 'linux' or 'linux-bitbar' assert is_mac or is_win or is_linux if is_win: diff --git a/tools/tryselect/util/manage_estimates.py b/tools/tryselect/util/manage_estimates.py index 2e2dcb6a558d..41ce46c90618 100644 --- a/tools/tryselect/util/manage_estimates.py +++ b/tools/tryselect/util/manage_estimates.py @@ -8,9 +8,9 @@ import os import requests import json from datetime import datetime, timedelta - import six + TASK_DURATION_URL = 'https://storage.googleapis.com/mozilla-mach-data/task_duration_history.json' GRAPH_QUANTILES_URL = 'https://storage.googleapis.com/mozilla-mach-data/machtry_quantiles.csv' from .estimates import TASK_DURATION_CACHE, GRAPH_QUANTILE_CACHE, TASK_DURATION_TAG_FILE