From e1af17b6c1399b59ad5daf94d640c16f065d73ae Mon Sep 17 00:00:00 2001 From: Csoregi Natalia Date: Wed, 1 Apr 2020 00:09:34 +0300 Subject: [PATCH] Backed out 4 changesets (bug 1624902) for failures on test_optimize_strategies.py. CLOSED TREE Backed out changeset b500403daa0c (bug 1624902) Backed out changeset bf817363344e (bug 1624902) Backed out changeset c7a62f2d9034 (bug 1624902) Backed out changeset 08356d478d16 (bug 1624902) --- .../ci/source-test/shadow-scheduler.yml | 19 ++----- taskcluster/taskgraph/optimize/__init__.py | 45 ++++++---------- taskcluster/taskgraph/optimize/bugbug.py | 34 ++---------- .../test/test_optimize_strategies.py | 52 +++++-------------- tools/tryselect/selectors/auto.py | 3 +- tools/tryselect/task_config.py | 14 +---- tools/tryselect/test/test_task_configs.py | 7 +-- 7 files changed, 39 insertions(+), 135 deletions(-) diff --git a/taskcluster/ci/source-test/shadow-scheduler.yml b/taskcluster/ci/source-test/shadow-scheduler.yml index d950b8597af3..bd2f8dd86f47 100644 --- a/taskcluster/ci/source-test/shadow-scheduler.yml +++ b/taskcluster/ci/source-test/shadow-scheduler.yml @@ -31,10 +31,10 @@ relevant_tests: job-name: shadow-scheduler-relevant_tests worker: env: - TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:experimental.relevant_tests + TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:relevant_tests bugbug_push_schedules: - description: Runs the bugbug.all optimization strategy instead of the default. + description: Runs the bugbug_push_schedules optimization strategy instead of the default. treeherder: symbol: SS(bugbug_push_schedules) index: @@ -42,18 +42,7 @@ bugbug_push_schedules: job-name: shadow-scheduler-bugbug_push_schedules worker: env: - TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:experimental.bugbug.all - -bugbug_debug: - description: Runs the bugbug.debug optimization strategy instead of the default. - treeherder: - symbol: SS(bugbug_debug) - index: - product: source - job-name: shadow-scheduler-bugbug_debug - worker: - env: - TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:experimental.bugbug.debug + TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:bugbug_push_schedules seta: description: Runs the SETA optimization strategy instead of the default. @@ -64,4 +53,4 @@ seta: job-name: shadow-scheduler-seta worker: env: - TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:experimental.seta + TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:seta diff --git a/taskcluster/taskgraph/optimize/__init__.py b/taskcluster/taskgraph/optimize/__init__.py index 31c290cba0d6..e87287bf2b94 100644 --- a/taskcluster/taskgraph/optimize/__init__.py +++ b/taskcluster/taskgraph/optimize/__init__.py @@ -313,36 +313,21 @@ register_strategy('test-try', args=('skip-unless-schedules',))(Alias) register_strategy('fuzzing-builds', args=('skip-unless-schedules', 'seta'))(Either) -class experimental(object): - """Experimental strategies either under development or used as benchmarks. +# Experimental strategies that run in 'shadow-scheduler' tasks. - These run as "shadow-schedulers" on each autoland push (tier 3) and/or can be used - with `./mach try auto`. E.g: +# Runs task containing tests in the same directories as modified files. +relevant_tests = { + 'test': Either('skip-unless-schedules', 'skip-unless-has-relevant-tests'), +} - ./mach try auto --strategy relevant_tests - """ +# Queries the bugbug push schedules endpoint which uses machine learning to +# determine which tasks to run. +bugbug_push_schedules = { + 'test': Either('skip-unless-schedules', 'bugbug-push-schedules'), +} - relevant_tests = { - 'test': Either('skip-unless-schedules', 'skip-unless-has-relevant-tests'), - } - """Runs task containing tests in the same directories as modified files.""" - - seta = { - 'test': Either('skip-unless-schedules', 'seta'), - } - """Provides a stable history of SETA's performance in the event we make it - non-default in the future. Only useful as a benchmark.""" - - class bugbug(object): - """Strategies that query the bugbug push schedules endpoint which uses machine - learning to determine which tasks to run.""" - - all = { - 'test': Either('skip-unless-schedules', 'bugbug-all'), - } - """Doesn't limit platforms.""" - - debug = { - 'test': Either('skip-unless-schedules', 'bugbug-debug'), - } - """Restricts tests to debug platforms.""" +# Provides a stable history of SETA's performance in the event we make it +# non-default in the future. +seta = { + 'test': Either('skip-unless-schedules', 'seta'), +} diff --git a/taskcluster/taskgraph/optimize/bugbug.py b/taskcluster/taskgraph/optimize/bugbug.py index c97ffe7aaa14..b05e75c7bc75 100644 --- a/taskcluster/taskgraph/optimize/bugbug.py +++ b/taskcluster/taskgraph/optimize/bugbug.py @@ -22,36 +22,12 @@ class BugbugTimeoutException(Exception): pass -class platform(object): - """Strategies for dealing with platforms.""" - - @staticmethod - def all(task, params): - """Don't filter any platforms.""" - return False - - @staticmethod - def debug(task, params): - """Only run debug platforms.""" - return not (task.attributes.get('build_type') == "debug") - - -@register_strategy("bugbug-all", args=(platform.all,)) -@register_strategy("bugbug-debug", args=(platform.debug,)) +@register_strategy("bugbug-push-schedules") class BugBugPushSchedules(OptimizationStrategy): - """Query the 'bugbug' service to retrieve relevant tasks and manifests. - - Args: - filterfn (func): A function to further reduce tasks after the bugbug - algorithm. - """ BUGBUG_BASE_URL = "https://bugbug.herokuapp.com" RETRY_TIMEOUT = 4 * 60 # seconds RETRY_INTERVAL = 5 # seconds - def __init__(self, filterfn): - self.filterfn = filterfn - @memoized_property def session(self): s = requests.Session() @@ -90,10 +66,8 @@ class BugBugPushSchedules(OptimizationStrategy): test_manifests = task.attributes.get('test_manifests') if test_manifests is None: - if task.label not in data.get('tasks', {}): - return True - - elif not bool(set(task.attributes['test_manifests']) & set(data.get('groups', {}))): + if task.label in data.get('tasks', {}): + return False return True - return self.filterfn(task, params) + return not bool(set(task.attributes['test_manifests']) & set(data.get('groups', {}))) diff --git a/taskcluster/taskgraph/test/test_optimize_strategies.py b/taskcluster/taskgraph/test/test_optimize_strategies.py index b008253126f4..446367b65088 100644 --- a/taskcluster/taskgraph/test/test_optimize_strategies.py +++ b/taskcluster/taskgraph/test/test_optimize_strategies.py @@ -8,7 +8,7 @@ import time import pytest from mozunit import main -from taskgraph.optimize.bugbug import BugBugPushSchedules, BugbugTimeoutException, platform +from taskgraph.optimize.bugbug import BugBugPushSchedules, BugbugTimeoutException from taskgraph.task import Task @@ -44,62 +44,34 @@ def generate_tasks(): def tasks(generate_tasks): return list(generate_tasks( {'attributes': {'test_manifests': ['foo/test.ini', 'bar/test.ini']}}, - {'attributes': {'test_manifests': ['bar/test.ini'], 'build_type': 'debug'}}, - {'attributes': {'build_type': 'debug'}}, + {'attributes': {'test_manifests': ['bar/test.ini']}}, + {'attributes': {}}, {'attributes': {'test_manifests': []}}, )) -def idfn(param): - if isinstance(param, tuple): - return param[0].__name__ - return None - - -@pytest.mark.parametrize("args,data,expected", [ - # empty +@pytest.mark.parametrize("data,expected", [ + pytest.param({}, [], id='empty'), pytest.param( - (platform.all,), - {}, - [], - ), - - # only tasks without test manifests selected - pytest.param( - (platform.all,), {'tasks': {'task-1': 0.9, 'task-2': 0.1, 'task-3': 0.5}}, ['task-2'], + id='only tasks without test manifests selected' ), - - # tasks containing groups selected pytest.param( - (platform.all,), {'groups': {'foo/test.ini': 0.4}}, ['task-0'], + id='tasks containing group selected', ), - - # tasks matching "tasks" or "groups" selected pytest.param( - (platform.all,), {'tasks': {'task-2': 0.2}, 'groups': {'foo/test.ini': 0.25, 'bar/test.ini': 0.75}}, ['task-0', 'task-1', 'task-2'], - ), - - # old format + id='tasks matching "tasks" or "groups" selected'), pytest.param( - (platform.all,), {'tasks': ['task-2'], 'groups': ['foo/test.ini', 'bar/test.ini']}, ['task-0', 'task-1', 'task-2'], - ), - - # debug platform filter - pytest.param( - (platform.debug,), - {'tasks': ['task-2', 'task-3'], 'groups': ['foo/test.ini', 'bar/test.ini']}, - ['task-1', 'task-2'], - ), -], ids=idfn) -def test_bugbug_push_schedules(responses, params, tasks, args, data, expected): + id='test old format'), +]) +def test_bugbug_push_schedules(responses, params, tasks, data, expected): query = "/push/{branch}/{head_rev}/schedules".format(**params) url = BugBugPushSchedules.BUGBUG_BASE_URL + query @@ -110,7 +82,7 @@ def test_bugbug_push_schedules(responses, params, tasks, args, data, expected): status=200, ) - opt = BugBugPushSchedules(*args) + opt = BugBugPushSchedules() labels = [t.label for t in tasks if not opt.should_remove_task(t, params, None)] assert sorted(labels) == sorted(expected) diff --git a/tools/tryselect/selectors/auto.py b/tools/tryselect/selectors/auto.py index 0b6403738b4a..ec2e1edd4000 100644 --- a/tools/tryselect/selectors/auto.py +++ b/tools/tryselect/selectors/auto.py @@ -27,8 +27,7 @@ def run(message='{msg}', push=True, closed_tree=False, try_config=None): # XXX Remove once an intelligent scheduling algorithm is running on # autoland by default. This ensures `mach try auto` doesn't run SETA. - try_config.setdefault('optimize-strategies', - 'taskgraph.optimize:experimental.bugbug.all') + try_config.setdefault('optimize-strategies', 'taskgraph.optimize:bugbug_push_schedules') task_config = { 'version': 2, diff --git a/tools/tryselect/task_config.py b/tools/tryselect/task_config.py index 8cf3bbb96195..ed660755c801 100644 --- a/tools/tryselect/task_config.py +++ b/tools/tryselect/task_config.py @@ -20,8 +20,6 @@ from textwrap import dedent import mozpack.path as mozpath from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject -from taskgraph.util.python_path import find_object - from .tasks import resolve_tests_by_suite here = os.path.abspath(os.path.dirname(__file__)) @@ -296,17 +294,7 @@ class OptimizeStrategies(TryConfig): def try_config(self, strategy, **kwargs): if strategy: if ':' not in strategy: - strategy = "taskgraph.optimize:experimental.{}".format(strategy) - - try: - obj = find_object(strategy) - except (ImportError, AttributeError): - print("error: invalid module path '{}'".format(strategy)) - sys.exit(1) - - if not isinstance(obj, dict): - print("error: object at '{}' must be a dict".format(strategy)) - sys.exit(1) + strategy = "taskgraph.optimize:{}".format(strategy) return { 'optimize-strategies': strategy, diff --git a/tools/tryselect/test/test_task_configs.py b/tools/tryselect/test/test_task_configs.py index 4b5d0ab8bd1f..96687c95acd1 100644 --- a/tools/tryselect/test/test_task_configs.py +++ b/tools/tryselect/test/test_task_configs.py @@ -42,11 +42,8 @@ TASK_CONFIG_TESTS = { ], 'strategy': [ ([], None), - (['--strategy', 'seta'], {'optimize-strategies': 'taskgraph.optimize:experimental.seta'}), - (['--strategy', 'taskgraph.optimize:experimental.seta'], {'optimize-strategies': 'taskgraph.optimize:experimental.seta'}), # noqa - (['--strategy', 'taskgraph.optimize:experimental'], SystemExit), - (['--strategy', 'foo'], SystemExit), - (['--strategy', 'foo:bar'], SystemExit), + (['--strategy', 'foo'], {'optimize-strategies': 'taskgraph.optimize:foo'}), + (['--strategy', 'foo:bar'], {'optimize-strategies': 'foo:bar'}), ], 'worker-overrides': [ ([], None),