Backed out 4 changesets (bug 1624902) for failures on test_optimize_strategies.py. CLOSED TREE

Backed out changeset b500403daa0c (bug 1624902)
Backed out changeset bf817363344e (bug 1624902)
Backed out changeset c7a62f2d9034 (bug 1624902)
Backed out changeset 08356d478d16 (bug 1624902)

taskcluster/ci/source-test/shadow-scheduler.yml

@@ -31,10 +31,10 @@ relevant_tests:
         job-name: shadow-scheduler-relevant_tests
     worker:
         env:
-            TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:experimental.relevant_tests
+            TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:relevant_tests
 
 bugbug_push_schedules:
-    description: Runs the bugbug.all optimization strategy instead of the default.
+    description: Runs the bugbug_push_schedules optimization strategy instead of the default.
     treeherder:
         symbol: SS(bugbug_push_schedules)
     index:
@@ -42,18 +42,7 @@ bugbug_push_schedules:
         job-name: shadow-scheduler-bugbug_push_schedules
     worker:
         env:
-            TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:experimental.bugbug.all
+            TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:bugbug_push_schedules
-
-bugbug_debug:
-    description: Runs the bugbug.debug optimization strategy instead of the default.
-    treeherder:
-        symbol: SS(bugbug_debug)
-    index:
-        product: source
-        job-name: shadow-scheduler-bugbug_debug
-    worker:
-        env:
-            TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:experimental.bugbug.debug
 
 seta:
     description: Runs the SETA optimization strategy instead of the default.
@@ -64,4 +53,4 @@ seta:
         job-name: shadow-scheduler-seta
     worker:
         env:
-            TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:experimental.seta
+            TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:seta

taskcluster/taskgraph/optimize/__init__.py

@@ -313,36 +313,21 @@ register_strategy('test-try', args=('skip-unless-schedules',))(Alias)
 register_strategy('fuzzing-builds', args=('skip-unless-schedules', 'seta'))(Either)
 
 
-class experimental(object):
+# Experimental strategies that run in 'shadow-scheduler' tasks.
-    """Experimental strategies either under development or used as benchmarks.
 
-    These run as "shadow-schedulers" on each autoland push (tier 3) and/or can be used
+# Runs task containing tests in the same directories as modified files.
-    with `./mach try auto`.  E.g:
+relevant_tests = {
+    'test': Either('skip-unless-schedules', 'skip-unless-has-relevant-tests'),
+}
 
-        ./mach try auto --strategy relevant_tests
+# Queries the bugbug push schedules endpoint which uses machine learning to
-    """
+# determine which tasks to run.
+bugbug_push_schedules = {
+    'test': Either('skip-unless-schedules', 'bugbug-push-schedules'),
+}
 
-    relevant_tests = {
+# Provides a stable history of SETA's performance in the event we make it
-        'test': Either('skip-unless-schedules', 'skip-unless-has-relevant-tests'),
+# non-default in the future.
-    }
+seta = {
-    """Runs task containing tests in the same directories as modified files."""
+    'test': Either('skip-unless-schedules', 'seta'),
-
+}
-    seta = {
-        'test': Either('skip-unless-schedules', 'seta'),
-    }
-    """Provides a stable history of SETA's performance in the event we make it
-    non-default in the future. Only useful as a benchmark."""
-
-    class bugbug(object):
-        """Strategies that query the bugbug push schedules endpoint which uses machine
-        learning to determine which tasks to run."""
-
-        all = {
-            'test': Either('skip-unless-schedules', 'bugbug-all'),
-        }
-        """Doesn't limit platforms."""
-
-        debug = {
-            'test': Either('skip-unless-schedules', 'bugbug-debug'),
-        }
-        """Restricts tests to debug platforms."""

taskcluster/taskgraph/optimize/bugbug.py

@@ -22,36 +22,12 @@ class BugbugTimeoutException(Exception):
     pass
 
 
-class platform(object):
+@register_strategy("bugbug-push-schedules")
-    """Strategies for dealing with platforms."""
-
-    @staticmethod
-    def all(task, params):
-        """Don't filter any platforms."""
-        return False
-
-    @staticmethod
-    def debug(task, params):
-        """Only run debug platforms."""
-        return not (task.attributes.get('build_type') == "debug")
-
-
-@register_strategy("bugbug-all", args=(platform.all,))
-@register_strategy("bugbug-debug", args=(platform.debug,))
 class BugBugPushSchedules(OptimizationStrategy):
-    """Query the 'bugbug' service to retrieve relevant tasks and manifests.
-
-    Args:
-        filterfn (func): A function to further reduce tasks after the bugbug
-                         algorithm.
-    """
     BUGBUG_BASE_URL = "https://bugbug.herokuapp.com"
     RETRY_TIMEOUT = 4 * 60  # seconds
     RETRY_INTERVAL = 5      # seconds
 
-    def __init__(self, filterfn):
-        self.filterfn = filterfn
-
     @memoized_property
     def session(self):
         s = requests.Session()
@@ -90,10 +66,8 @@ class BugBugPushSchedules(OptimizationStrategy):
 
         test_manifests = task.attributes.get('test_manifests')
         if test_manifests is None:
-            if task.label not in data.get('tasks', {}):
+            if task.label in data.get('tasks', {}):
-                return True
+                return False
-
-        elif not bool(set(task.attributes['test_manifests']) & set(data.get('groups', {}))):
             return True
 
-        return self.filterfn(task, params)
+        return not bool(set(task.attributes['test_manifests']) & set(data.get('groups', {})))

taskcluster/taskgraph/test/test_optimize_strategies.py

@@ -8,7 +8,7 @@ import time
 import pytest
 from mozunit import main
 
-from taskgraph.optimize.bugbug import BugBugPushSchedules, BugbugTimeoutException, platform
+from taskgraph.optimize.bugbug import BugBugPushSchedules, BugbugTimeoutException
 from taskgraph.task import Task
 
 
@@ -44,62 +44,34 @@ def generate_tasks():
 def tasks(generate_tasks):
     return list(generate_tasks(
         {'attributes': {'test_manifests': ['foo/test.ini', 'bar/test.ini']}},
-        {'attributes': {'test_manifests': ['bar/test.ini'], 'build_type': 'debug'}},
+        {'attributes': {'test_manifests': ['bar/test.ini']}},
-        {'attributes': {'build_type': 'debug'}},
+        {'attributes': {}},
         {'attributes': {'test_manifests': []}},
     ))
 
 
-def idfn(param):
+@pytest.mark.parametrize("data,expected", [
-    if isinstance(param, tuple):
+    pytest.param({}, [], id='empty'),
-        return param[0].__name__
-    return None
-
-
-@pytest.mark.parametrize("args,data,expected", [
-    # empty
     pytest.param(
-        (platform.all,),
-        {},
-        [],
-    ),
-
-    # only tasks without test manifests selected
-    pytest.param(
-        (platform.all,),
         {'tasks': {'task-1': 0.9, 'task-2': 0.1, 'task-3': 0.5}},
         ['task-2'],
+        id='only tasks without test manifests selected'
     ),
-
-    # tasks containing groups selected
     pytest.param(
-        (platform.all,),
         {'groups': {'foo/test.ini': 0.4}},
         ['task-0'],
+        id='tasks containing group selected',
     ),
-
-    # tasks matching "tasks" or "groups" selected
     pytest.param(
-        (platform.all,),
         {'tasks': {'task-2': 0.2}, 'groups': {'foo/test.ini': 0.25, 'bar/test.ini': 0.75}},
         ['task-0', 'task-1', 'task-2'],
-    ),
+        id='tasks matching "tasks" or "groups" selected'),
-
-    # old format
     pytest.param(
-        (platform.all,),
         {'tasks': ['task-2'], 'groups': ['foo/test.ini', 'bar/test.ini']},
         ['task-0', 'task-1', 'task-2'],
-    ),
+        id='test old format'),
-
+])
-    # debug platform filter
+def test_bugbug_push_schedules(responses, params, tasks, data, expected):
-    pytest.param(
-        (platform.debug,),
-        {'tasks': ['task-2', 'task-3'], 'groups': ['foo/test.ini', 'bar/test.ini']},
-        ['task-1', 'task-2'],
-    ),
-], ids=idfn)
-def test_bugbug_push_schedules(responses, params, tasks, args, data, expected):
     query = "/push/{branch}/{head_rev}/schedules".format(**params)
     url = BugBugPushSchedules.BUGBUG_BASE_URL + query
 
@@ -110,7 +82,7 @@ def test_bugbug_push_schedules(responses, params, tasks, args, data, expected):
         status=200,
     )
 
-    opt = BugBugPushSchedules(*args)
+    opt = BugBugPushSchedules()
     labels = [t.label for t in tasks if not opt.should_remove_task(t, params, None)]
     assert sorted(labels) == sorted(expected)
 

tools/tryselect/selectors/auto.py

@@ -27,8 +27,7 @@ def run(message='{msg}', push=True, closed_tree=False, try_config=None):
 
     # XXX Remove once an intelligent scheduling algorithm is running on
     # autoland by default. This ensures `mach try auto` doesn't run SETA.
-    try_config.setdefault('optimize-strategies',
+    try_config.setdefault('optimize-strategies', 'taskgraph.optimize:bugbug_push_schedules')
-                          'taskgraph.optimize:experimental.bugbug.all')
 
     task_config = {
         'version': 2,

tools/tryselect/task_config.py

@@ -20,8 +20,6 @@ from textwrap import dedent
 
 import mozpack.path as mozpath
 from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
-from taskgraph.util.python_path import find_object
-
 from .tasks import resolve_tests_by_suite
 
 here = os.path.abspath(os.path.dirname(__file__))
@@ -296,17 +294,7 @@ class OptimizeStrategies(TryConfig):
     def try_config(self, strategy, **kwargs):
         if strategy:
             if ':' not in strategy:
-                strategy = "taskgraph.optimize:experimental.{}".format(strategy)
+                strategy = "taskgraph.optimize:{}".format(strategy)
-
-            try:
-                obj = find_object(strategy)
-            except (ImportError, AttributeError):
-                print("error: invalid module path '{}'".format(strategy))
-                sys.exit(1)
-
-            if not isinstance(obj, dict):
-                print("error: object at '{}' must be a dict".format(strategy))
-                sys.exit(1)
 
             return {
                 'optimize-strategies': strategy,

tools/tryselect/test/test_task_configs.py

@@ -42,11 +42,8 @@ TASK_CONFIG_TESTS = {
     ],
     'strategy': [
         ([], None),
-        (['--strategy', 'seta'], {'optimize-strategies': 'taskgraph.optimize:experimental.seta'}),
+        (['--strategy', 'foo'], {'optimize-strategies': 'taskgraph.optimize:foo'}),
-        (['--strategy', 'taskgraph.optimize:experimental.seta'], {'optimize-strategies': 'taskgraph.optimize:experimental.seta'}),  # noqa
+        (['--strategy', 'foo:bar'], {'optimize-strategies': 'foo:bar'}),
-        (['--strategy', 'taskgraph.optimize:experimental'], SystemExit),
-        (['--strategy', 'foo'], SystemExit),
-        (['--strategy', 'foo:bar'], SystemExit),
     ],
     'worker-overrides': [
         ([], None),