Bug 1925007 - [tryselect] Use 'filters' parameter rather than 'target_tasks_method' for tryselect, r=taskgraph-reviewers,jmaher

It turns out that overriding the `target_tasks_method` in ./mach try
comes with a major disadvantage. Namely that when using the
`-p/--parameters` flag, any non-standard `target_tasks_method` defined
in there will also be overwritten. For example, this makes it impossible
to make `./mach try` target the set of tasks that would be selected by a
cron task.

Luckily Taskgraph has a concept of "filters" which is pretty much
identical to how `target_tasks_method` works (I think the intent was for
filters to replace target_tasks, but that never ended up happening).

This means we can make the `./mach try` methods sit outside of
`target_tasks` and not override the original `target_tasks_method`
unless we mean to.

This change should be a simple refactor and not change any
functionality.

Differential Revision: https://phabricator.services.mozilla.com/D226554

taskcluster/gecko_taskgraph/__init__.py

@@ -56,7 +56,8 @@ def register(graph_config):
     del registry["skip-unless-changed"]
 
     from gecko_taskgraph import (  # noqa: trigger target task method registration
-        morph,  # noqa: trigger morph registration
+        morph,
+        filter_tasks,
         target_tasks,
     )
 

taskcluster/gecko_taskgraph/filter_tasks.py

@@ -0,0 +1,85 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+
+from taskgraph.filter_tasks import filter_task
+from taskgraph.parameters import Parameters
+from taskgraph.target_tasks import get_method
+
+from gecko_taskgraph.target_tasks import (
+    filter_by_regex,
+    filter_by_uncommon_try_tasks,
+    filter_out_shippable,
+    filter_unsupported_artifact_builds,
+    target_tasks_default,
+)
+
+
+@filter_task("try_auto")
+def target_tasks_try_auto(full_task_graph, parameters, graph_config):
+    """Target the tasks which have indicated they should be run on autoland
+    (rather than try) via the `run_on_projects` attributes.
+
+    Should do the same thing as the `default` target tasks method.
+    """
+    params = dict(parameters)
+    params["project"] = "autoland"
+    params["target_tasks_method"] = "default"
+    parameters = Parameters(**params)
+
+    regex_filters = parameters["try_task_config"].get("tasks-regex")
+    include_regexes = exclude_regexes = []
+    if regex_filters:
+        include_regexes = [re.compile(r) for r in regex_filters.get("include", [])]
+        exclude_regexes = [re.compile(r) for r in regex_filters.get("exclude", [])]
+
+    filtered_for_default = target_tasks_default(
+        full_task_graph, parameters, graph_config
+    )
+    filtered_for_try_auto = [
+        l
+        for l, t in full_task_graph.tasks.items()
+        if filter_by_uncommon_try_tasks(t.label)
+        and filter_by_regex(t.label, include_regexes, mode="include")
+        and filter_by_regex(t.label, exclude_regexes, mode="exclude")
+        and filter_unsupported_artifact_builds(t, parameters)
+        and filter_out_shippable(t)
+    ]
+    return list(set(filtered_for_default) & set(filtered_for_try_auto))
+
+
+@filter_task("try_select_tasks")
+def target_tasks_try_select(full_task_graph, parameters, graph_config):
+    tasks = target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config)
+    return [l for l in tasks if filter_by_uncommon_try_tasks(l)]
+
+
+@filter_task("try_select_tasks_uncommon")
+def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config):
+    from gecko_taskgraph.decision import PER_PROJECT_PARAMETERS
+
+    # Union the tasks between autoland and mozilla-central as a sensible
+    # default. This is likely the set of tasks that most users are
+    # attempting to select from.
+    projects = ("autoland", "mozilla-central")
+    if parameters["project"] not in projects:
+        projects = (parameters["project"],)
+
+    tasks = set()
+    for project in projects:
+        params = dict(parameters)
+        params["project"] = project
+        parameters = Parameters(**params)
+
+        try:
+            target_tasks_method = PER_PROJECT_PARAMETERS[project]["target_tasks_method"]
+        except KeyError:
+            target_tasks_method = "default"
+
+        tasks.update(
+            get_method(target_tasks_method)(full_task_graph, parameters, graph_config)
+        )
+
+    return sorted(tasks)

taskcluster/gecko_taskgraph/target_tasks.py

@@ -11,8 +11,7 @@ from datetime import datetime, timedelta
 
 import requests
 from redo import retry
-from taskgraph.parameters import Parameters
-from taskgraph.target_tasks import get_method, register_target_task
+from taskgraph.target_tasks import register_target_task
 from taskgraph.util.taskcluster import find_task_id, parse_time
 
 from gecko_taskgraph import GECKO, try_option_syntax
@@ -477,69 +476,6 @@ def target_tasks_try(full_task_graph, parameters, graph_config):
     return []
 
 
-@register_target_task("try_select_tasks")
-def target_tasks_try_select(full_task_graph, parameters, graph_config):
-    tasks = target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config)
-    return [l for l in tasks if filter_by_uncommon_try_tasks(l)]
-
-
-@register_target_task("try_select_tasks_uncommon")
-def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config):
-    from gecko_taskgraph.decision import PER_PROJECT_PARAMETERS
-
-    projects = ("autoland", "mozilla-central")
-    if parameters["project"] not in projects:
-        projects = (parameters["project"],)
-
-    tasks = set()
-    for project in projects:
-        params = dict(parameters)
-        params["project"] = project
-        parameters = Parameters(**params)
-
-        try:
-            target_tasks_method = PER_PROJECT_PARAMETERS[project]["target_tasks_method"]
-        except KeyError:
-            target_tasks_method = "default"
-
-        tasks.update(
-            get_method(target_tasks_method)(full_task_graph, parameters, graph_config)
-        )
-
-    return sorted(tasks)
-
-
-@register_target_task("try_auto")
-def target_tasks_try_auto(full_task_graph, parameters, graph_config):
-    """Target the tasks which have indicated they should be run on autoland
-    (rather than try) via the `run_on_projects` attributes.
-
-    Should do the same thing as the `default` target tasks method.
-    """
-    params = dict(parameters)
-    params["project"] = "autoland"
-    parameters = Parameters(**params)
-
-    regex_filters = parameters["try_task_config"].get("tasks-regex")
-    include_regexes = exclude_regexes = []
-    if regex_filters:
-        include_regexes = [re.compile(r) for r in regex_filters.get("include", [])]
-        exclude_regexes = [re.compile(r) for r in regex_filters.get("exclude", [])]
-
-    return [
-        l
-        for l, t in full_task_graph.tasks.items()
-        if standard_filter(t, parameters)
-        and filter_out_shipping_phase(t, parameters)
-        and filter_out_devedition(t, parameters)
-        and filter_by_uncommon_try_tasks(t.label)
-        and filter_by_regex(t.label, include_regexes, mode="include")
-        and filter_by_regex(t.label, exclude_regexes, mode="exclude")
-        and filter_unsupported_artifact_builds(t, parameters)
-        and filter_out_shippable(t)
-    ]
-
-
 @register_target_task("default")
 def target_tasks_default(full_task_graph, parameters, graph_config):
     """Target the tasks which have indicated they should be run on this project

taskcluster/gecko_taskgraph/test/test_target_tasks.py

@@ -10,6 +10,7 @@ import unittest
 import pytest
 from mozunit import main
 from taskgraph.graph import Graph
+from taskgraph.target_tasks import get_method
 from taskgraph.task import Task
 from taskgraph.taskgraph import TaskGraph
 
@@ -57,7 +58,7 @@ class TestTargetTasks(unittest.TestCase):
         )
 
     def default_matches(self, attributes, parameters):
-        method = target_tasks.get_method("default")
+        method = get_method("default")
         graph = TaskGraph(
             tasks={
                 "a": Task(kind="build", label="a", attributes=attributes, task={}),
@@ -169,7 +170,7 @@ class TestTargetTasks(unittest.TestCase):
     def test_empty_try(self):
         "try_mode = None runs nothing"
         tg = self.make_task_graph()
-        method = target_tasks.get_method("try_tasks")
+        method = get_method("try_tasks")
         params = {
             "try_mode": None,
             "project": "try",
@@ -181,7 +182,7 @@ class TestTargetTasks(unittest.TestCase):
     def test_try_option_syntax(self):
         "try_mode = try_option_syntax uses TryOptionSyntax"
         tg = self.make_task_graph()
-        method = target_tasks.get_method("try_tasks")
+        method = get_method("try_tasks")
         with self.fake_TryOptionSyntax():
             params = {
                 "try_mode": "try_option_syntax",
@@ -192,7 +193,7 @@ class TestTargetTasks(unittest.TestCase):
     def test_try_task_config(self):
         "try_mode = try_task_config uses the try config"
         tg = self.make_task_graph()
-        method = target_tasks.get_method("try_tasks")
+        method = get_method("try_tasks")
         params = {
             "try_mode": "try_task_config",
             "try_task_config": {"tasks": ["a"]},
@@ -202,7 +203,7 @@ class TestTargetTasks(unittest.TestCase):
     def test_try_task_config_regex(self):
         "try_mode = try_task_config uses the try config with regex instead of chunk numbers"
         tg = self.make_task_graph()
-        method = target_tasks.get_method("try_tasks")
+        method = get_method("try_tasks")
         params = {
             "try_mode": "try_task_config",
             "try_task_config": {"new-test-config": True, "tasks": ["ddd-*"]},
@@ -213,7 +214,7 @@ class TestTargetTasks(unittest.TestCase):
     def test_try_task_config_regex_with_paths(self):
         "try_mode = try_task_config uses the try config with regex instead of chunk numbers"
         tg = self.make_task_graph()
-        method = target_tasks.get_method("try_tasks")
+        method = get_method("try_tasks")
         params = {
             "try_mode": "try_task_config",
             "try_task_config": {
@@ -228,7 +229,7 @@ class TestTargetTasks(unittest.TestCase):
     def test_try_task_config_absolute(self):
         "try_mode = try_task_config uses the try config with full task labels"
         tg = self.make_task_graph()
-        method = target_tasks.get_method("try_tasks")
+        method = get_method("try_tasks")
         params = {
             "try_mode": "try_task_config",
             "try_task_config": {
@@ -242,7 +243,7 @@ class TestTargetTasks(unittest.TestCase):
     def test_try_task_config_regex_var(self):
         "try_mode = try_task_config uses the try config with regex instead of chunk numbers and a test variant"
         tg = self.make_task_graph()
-        method = target_tasks.get_method("try_tasks")
+        method = get_method("try_tasks")
         params = {
             "try_mode": "try_task_config",
             "try_task_config": {"new-test-config": True, "tasks": ["ddd-var-*"]},

tools/tryselect/selectors/auto.py

@@ -10,9 +10,9 @@ from ..push import push_to_try
 from ..util.dicttools import merge
 
 TRY_AUTO_PARAMETERS = {
+    "filters": ["try_auto"],
     "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium",  # noqa
     "optimize_target_tasks": True,
-    "target_tasks_method": "try_auto",
     "test_manifest_loader": "bugbug",
     "try_mode": "try_auto",
     "try_task_config": {},

tools/tryselect/tasks.py

@@ -65,7 +65,8 @@ def cache_key(attr, params, disable_target_task_filter):
 
 def generate_tasks(params=None, full=False, disable_target_task_filter=False):
     attr = "full_task_set" if full else "target_task_set"
-    target_tasks_method = (
+
+    filter_fn = (
         "try_select_tasks"
         if not disable_target_task_filter
         else "try_select_tasks_uncommon"
@@ -75,7 +76,7 @@ def generate_tasks(params=None, full=False, disable_target_task_filter=False):
         strict=False,
         overrides={
             "try_mode": "try_select",
-            "target_tasks_method": target_tasks_method,
+            "filters": [filter_fn],
         },
     )
     root = os.path.join(build.topsrcdir, "taskcluster")

tools/tryselect/test/test_auto.t

@@ -14,9 +14,11 @@ Test auto selector
   Calculated try_task_config.json:
   {
       "parameters": {
+          "filters": [
+              "try_auto"
+          ],
           "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium",
           "optimize_target_tasks": true,
-          "target_tasks_method": "try_auto",
           "test_manifest_loader": "bugbug",
           "try_mode": "try_auto",
           "try_task_config": {}
@@ -35,9 +37,11 @@ Test auto selector
   Calculated try_task_config.json:
   {
       "parameters": {
+          "filters": [
+              "try_auto"
+          ],
           "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium",
           "optimize_target_tasks": true,
-          "target_tasks_method": "try_auto",
           "test_manifest_loader": "bugbug",
           "try_mode": "try_auto",
           "try_task_config": {}
@@ -55,9 +59,11 @@ Test auto selector
   Calculated try_task_config.json:
   {
       "parameters": {
+          "filters": [
+              "try_auto"
+          ],
           "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium",
           "optimize_target_tasks": true,
-          "target_tasks_method": "try_auto",
           "test_manifest_loader": "bugbug",
           "try_mode": "try_auto",
           "try_task_config": {}