Bug 1286075: fix target task generation, including try; r=Callek

This uses the run_on_projects attribute introduced earlier for most branches,
adjusts the `ash` method to handle that branch as the legacy implementation
did, and updates try syntax to match builds as well as tests.

In the process, this enables optimizing target tasks, meaning that tasks
specifically requested in the try syntax might be optimized.  While this is
probably not ideal, it matches the existing behavior of try (where `-j all` is
the default but all jobs are set to run only when certain files have been
modified).  This change can be reverted later, in a more advanced version of
try.

MozReview-Commit-ID: 5FYeUTAsafr

--HG--
extra : rebase_source : b358e0e7cd8a401c50009e63dd55c59489c9b75b

taskcluster/taskgraph/decision.py

@@ -32,9 +32,10 @@ GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
 PER_PROJECT_PARAMETERS = {
     'try': {
         'target_tasks_method': 'try_option_syntax',
-        # for try, if a task was specified as a target, it should
-        # not be optimized away
-        'optimize_target_tasks': False,
+        # Always perform optimization.  This makes it difficult to use try
+        # pushes to run a task that would otherwise be optimized, but is a
+        # compromise to avoid essentially disabling optimization in try.
+        'optimize_target_tasks': True,
     },
 
     'ash': {
@@ -44,7 +45,7 @@ PER_PROJECT_PARAMETERS = {
 
     # the default parameters are used for projects that do not match above.
     'default': {
-        'target_tasks_method': 'all_builds_and_tests',
+        'target_tasks_method': 'default',
         'optimize_target_tasks': True,
     }
 }

taskcluster/taskgraph/target_tasks.py

@@ -6,12 +6,17 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 from taskgraph import try_option_syntax
-from taskgraph.util.attributes import attrmatch
 
-BUILD_AND_TEST_KINDS = set([
-    'legacy',  # builds
-    'desktop-test',
-    'android-test',
+INTEGRATION_PROJECTS = set([
+    'mozilla-inbound',
+    'autoland',
+])
+
+RELEASE_PROJECTS = set([
+    'mozilla-central',
+    'mozilla-aurora',
+    'mozilla-beta',
+    'mozilla-release',
 ])
 
 _target_task_methods = {}
@@ -55,32 +60,43 @@ def target_tasks_try_option_syntax(full_task_graph, parameters):
     return target_tasks_labels
 
 
-@_target_task('all_builds_and_tests')
-def target_tasks_all_builds_and_tests(full_task_graph, parameters):
-    """Trivially target all build and test tasks.  This is used for
-    branches where we want to build "everyting", but "everything"
-    does not include uninteresting things like docker images"""
+@_target_task('all_builds_and_tests')  # (old name)
+@_target_task('default')
+def target_tasks_default(full_task_graph, parameters):
+    """Target the tasks which have indicated they should be run on this project
+    via the `run_on_projects` attributes."""
     def filter(task):
-        return t.attributes.get('kind') in BUILD_AND_TEST_KINDS
+        run_on_projects = set(t.attributes.get('run_on_projects', []))
+        if 'all' in run_on_projects:
+            return True
+        project = parameters['project']
+        if 'integration' in run_on_projects:
+            if project in INTEGRATION_PROJECTS:
+                return True
+        if 'release' in run_on_projects:
+            if project in RELEASE_PROJECTS:
+                return True
+        return project in run_on_projects
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('ash_tasks')
-def target_tasks_ash_tasks(full_task_graph, parameters):
-    """Special case for builds on ash."""
+def target_tasks_ash(full_task_graph, parameters):
+    """Target tasks that only run on the ash branch."""
     def filter(task):
-        # NOTE: on the ash branch, update taskcluster/ci/desktop-test/tests.yml to
-        # run the M-dt-e10s tasks
-        attrs = t.attributes
-        if attrs.get('kind') not in BUILD_AND_TEST_KINDS:
+        platform = task.attributes.get('build_platform')
+        # only select platforms
+        if platform not in ('linux64', 'linux64-asan', 'linux64-pgo'):
             return False
-        if not attrmatch(attrs, build_platform=set([
-            'linux64',
-            'linux64-asan',
-            'linux64-pgo',
-        ])):
+        # and none of this linux64-asan/debug stuff
+        if platform == 'linux64-asan' and task.attributes['build_type'] == 'debug':
             return False
-        if not attrmatch(attrs, e10s=True):
+        # no non-et10s tests
+        if task.attributes.get('unittest_suite') or task.attributes.get('talos_siute'):
+            if not task.attributes.get('e10s'):
+                return False
+        # don't upload symbols
+        if task.attributes['kind'] == 'upload-symbols':
             return False
         return True
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]

taskcluster/taskgraph/task/legacy.py

@@ -515,7 +515,11 @@ class LegacyTask(base.Task):
 
             # Ensure each build graph is valid after construction.
             validate_build_task(build_task)
-            attributes = build_task['attributes'] = {'kind': 'legacy', 'legacy_kind': 'build'}
+            attributes = build_task['attributes'] = {
+                'kind': 'legacy',
+                'legacy_kind': 'build',
+                'run_on_projects': ['all'],
+            }
             if 'build_name' in build:
                 attributes['build_platform'] = build['build_name']
             if 'build_type' in task_extra:

taskcluster/taskgraph/test/test_target_tasks.py

@@ -30,14 +30,39 @@ class TestTargetTasks(unittest.TestCase):
         self.assertEqual(method(None, {'target_tasks': ['a', 'b']}),
                          ['a', 'b'])
 
-    def test_all_builds_and_tests(self):
-        method = target_tasks.get_method('all_builds_and_tests')
+    def default_matches(self, run_on_projects, project):
+        method = target_tasks.get_method('default')
         graph = TaskGraph(tasks={
-            'a': TestTask(kind='legacy', label='a'),
-            'b': TestTask(kind='legacy', label='b'),
-            'boring': TestTask(kind='docker', label='boring'),
-        }, graph=Graph(nodes={'a', 'b', 'boring'}, edges=set()))
-        self.assertEqual(sorted(method(graph, {})), sorted(['a', 'b']))
+            'a': TestTask(kind='build', label='a',
+                          attributes={'run_on_projects': run_on_projects}),
+        }, graph=Graph(nodes={'a'}, edges=set()))
+        parameters = {'project': project}
+        return 'a' in method(graph, parameters)
+
+    def test_default_all(self):
+        """run_on_projects=[all] includes release, integration, and other projects"""
+        self.assertTrue(self.default_matches(['all'], 'mozilla-central'))
+        self.assertTrue(self.default_matches(['all'], 'mozilla-inbound'))
+        self.assertTrue(self.default_matches(['all'], 'mozilla-aurora'))
+        self.assertTrue(self.default_matches(['all'], 'baobab'))
+
+    def test_default_integration(self):
+        """run_on_projects=[integration] includes integration projects"""
+        self.assertFalse(self.default_matches(['integration'], 'mozilla-central'))
+        self.assertTrue(self.default_matches(['integration'], 'mozilla-inbound'))
+        self.assertFalse(self.default_matches(['integration'], 'baobab'))
+
+    def test_default_relesae(self):
+        """run_on_projects=[release] includes release projects"""
+        self.assertTrue(self.default_matches(['release'], 'mozilla-central'))
+        self.assertFalse(self.default_matches(['release'], 'mozilla-inbound'))
+        self.assertFalse(self.default_matches(['release'], 'baobab'))
+
+    def test_default_nothing(self):
+        """run_on_projects=[] includes nothing"""
+        self.assertFalse(self.default_matches([], 'mozilla-central'))
+        self.assertFalse(self.default_matches([], 'mozilla-inbound'))
+        self.assertFalse(self.default_matches([], 'baobab'))
 
     def test_try_option_syntax(self):
         tasks = {

taskcluster/taskgraph/try_option_syntax.py

@@ -21,6 +21,14 @@ BUILD_TYPE_ALIASES = {
     'd': 'debug'
 }
 
+# consider anything in this whitelist of kinds to be governed by -b/-p
+BUILD_KINDS = set([
+])
+
+# anything in this list is governed by -j
+JOB_KINDS = set([
+])
+
 
 # mapping from shortcut name (usable with -u) to a boolean function identifying
 # matching test names
@@ -124,7 +132,8 @@ UNITTEST_PLATFORM_PRETTY_NAMES = {
 
 # We have a few platforms for which we want to do some "extra" builds, or at
 # least build-ish things.  Sort of.  Anyway, these other things are implemented
-# as different "platforms".
+# as different "platforms".  These do *not* automatically ride along with "-p
+# all"
 RIDEALONG_BUILDS = {
     'android-api-15': [
         'android-api-15-l10n',
@@ -245,7 +254,7 @@ class TryOptionSyntax(object):
             if build in RIDEALONG_BUILDS:
                 results.extend(RIDEALONG_BUILDS[build])
                 logger.info("platform %s triggers ridealong builds %s" %
-                            (build, RIDEALONG_BUILDS[build]))
+                            (build, ', '.join(RIDEALONG_BUILDS[build])))
 
         return results
 
@@ -508,6 +517,21 @@ class TryOptionSyntax(object):
             return False
         elif attr('kind') in ('desktop-test', 'android-test'):
             return match_test(self.unittests, 'unittest_try_name')
+        elif attr('kind') in JOB_KINDS:
+            if self.jobs is None:
+                return True
+            if attr('build_platform') in self.jobs:
+                return True
+        elif attr('kind') in BUILD_KINDS:
+            if attr('build_type') not in self.build_types:
+                return False
+            elif self.platforms is None:
+                # for "-p all", look for try in the 'run_on_projects' attribute
+                return set(['try', 'all']) & set(attr('run_on_projects', []))
+            else:
+                if attr('build_platform') not in self.platforms:
+                    return False
+            return True
         else:
             return False