From 34f2b043a9c336fd180c2cf4f65b082650150157 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Tue, 16 Apr 2019 22:02:46 +0000 Subject: [PATCH] Bug 1450012: [taskgraph] Disable retrigger action for many tasks; r=dustin Many tasks (release tasks and cached tasks, in particular) should be re-run rather than retriggered. Disable retrigger action for those tasks by default. Differential Revision: https://phabricator.services.mozilla.com/D27206 --HG-- extra : moz-landing-system : lando --- taskcluster/ci/hazard/kind.yml | 2 + .../ci/mar-signing-autograph-stage/kind.yml | 2 + taskcluster/ci/source-test/kind.yml | 4 ++ taskcluster/ci/spidermonkey/kind.yml | 2 + .../ci/static-analysis-autotest/kind.yml | 2 + taskcluster/ci/test/kind.yml | 3 +- taskcluster/ci/valgrind/kind.yml | 4 ++ taskcluster/ci/webrender/kind.yml | 2 + taskcluster/docs/attributes.rst | 4 ++ taskcluster/taskgraph/actions/registry.py | 1 + taskcluster/taskgraph/actions/retrigger.py | 55 ++++++++++++++++++- taskcluster/taskgraph/transforms/task.py | 4 +- 12 files changed, 82 insertions(+), 3 deletions(-) diff --git a/taskcluster/ci/hazard/kind.yml b/taskcluster/ci/hazard/kind.yml index 4eead51f58e5..02e03feab890 100644 --- a/taskcluster/ci/hazard/kind.yml +++ b/taskcluster/ci/hazard/kind.yml @@ -15,6 +15,8 @@ transforms: - taskgraph.transforms.task:transforms job-defaults: + attributes: + retrigger: true treeherder: kind: build tier: 1 diff --git a/taskcluster/ci/mar-signing-autograph-stage/kind.yml b/taskcluster/ci/mar-signing-autograph-stage/kind.yml index 7dffdaeb30f6..581a334e1a90 100644 --- a/taskcluster/ci/mar-signing-autograph-stage/kind.yml +++ b/taskcluster/ci/mar-signing-autograph-stage/kind.yml @@ -25,3 +25,5 @@ job-template: - mar-signing run-on-projects: [] nightly: false + attributes: + retrigger: true diff --git a/taskcluster/ci/source-test/kind.yml b/taskcluster/ci/source-test/kind.yml index df801adae92c..277b9e9b2a40 100644 --- a/taskcluster/ci/source-test/kind.yml +++ b/taskcluster/ci/source-test/kind.yml @@ -30,6 +30,10 @@ jobs-from: - wpt-metadata.yml - wpt-manifest.yml +job-defaults: + attributes: + retrigger: true + # This is used by run-task based tasks to lookup which build task it # should depend on based on its own platform. dependent-build-platforms: diff --git a/taskcluster/ci/spidermonkey/kind.yml b/taskcluster/ci/spidermonkey/kind.yml index 1f863627d69d..8891b03edede 100644 --- a/taskcluster/ci/spidermonkey/kind.yml +++ b/taskcluster/ci/spidermonkey/kind.yml @@ -16,6 +16,8 @@ transforms: - taskgraph.transforms.task:transforms job-defaults: + attributes: + retrigger: true treeherder: kind: build tier: 1 diff --git a/taskcluster/ci/static-analysis-autotest/kind.yml b/taskcluster/ci/static-analysis-autotest/kind.yml index 91d3e4b4acf0..0879d438b558 100644 --- a/taskcluster/ci/static-analysis-autotest/kind.yml +++ b/taskcluster/ci/static-analysis-autotest/kind.yml @@ -15,6 +15,8 @@ transforms: - taskgraph.transforms.task:transforms job-defaults: + attributes: + retrigger: true index: product: firefox worker: diff --git a/taskcluster/ci/test/kind.yml b/taskcluster/ci/test/kind.yml index ad7679ed248d..484b53979e51 100644 --- a/taskcluster/ci/test/kind.yml +++ b/taskcluster/ci/test/kind.yml @@ -35,8 +35,9 @@ jobs-from: - web-platform.yml - xpcshell.yml - job-defaults: + attributes: + retrigger: true require-signed-extensions: by-release-type: release|esr.*: true diff --git a/taskcluster/ci/valgrind/kind.yml b/taskcluster/ci/valgrind/kind.yml index 4690d6f57631..ed69c490fb1e 100644 --- a/taskcluster/ci/valgrind/kind.yml +++ b/taskcluster/ci/valgrind/kind.yml @@ -14,6 +14,10 @@ transforms: - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms +job-defaults: + attributes: + retrigger: true + jobs: linux64-valgrind/opt: description: "Linux64 Valgrind Opt" diff --git a/taskcluster/ci/webrender/kind.yml b/taskcluster/ci/webrender/kind.yml index b7e23b5beee1..5d9614acd4c0 100644 --- a/taskcluster/ci/webrender/kind.yml +++ b/taskcluster/ci/webrender/kind.yml @@ -13,6 +13,8 @@ transforms: - taskgraph.transforms.task:transforms job-defaults: + attributes: + retrigger: true run-on-projects: ['mozilla-beta', 'trunk', 'try'] treeherder: tier: 1 diff --git a/taskcluster/docs/attributes.rst b/taskcluster/docs/attributes.rst index 51eabeec67c6..0c186656e639 100644 --- a/taskcluster/docs/attributes.rst +++ b/taskcluster/docs/attributes.rst @@ -307,3 +307,7 @@ If a task set this boolean attribute to `true`, it will be processed by the code review bot, the task will ran for every new Phabricator diff. Any supported and detected issue will be automatically reported on the Phabricator revision. + +retrigger +========= +Whether the task can be retriggered, or if it needs to be re-run. diff --git a/taskcluster/taskgraph/actions/registry.py b/taskcluster/taskgraph/actions/registry.py index 44020f808c2e..0913a547d8be 100644 --- a/taskcluster/taskgraph/actions/registry.py +++ b/taskcluster/taskgraph/actions/registry.py @@ -240,6 +240,7 @@ def register_callback_action(name, title, symbol, description, order=10000, mem['registered'] = True callbacks[cb_name] = cb + return cb return register_callback diff --git a/taskcluster/taskgraph/actions/retrigger.py b/taskcluster/taskgraph/actions/retrigger.py index 7485621b627f..51d38f120c40 100644 --- a/taskcluster/taskgraph/actions/retrigger.py +++ b/taskcluster/taskgraph/actions/retrigger.py @@ -6,6 +6,8 @@ from __future__ import absolute_import, print_function, unicode_literals +import sys + import logging import textwrap @@ -61,7 +63,7 @@ def retrigger_decision_action(parameters, graph_config, input, task_group_id, ta 'Create a clone of the task.' ), order=19, # must be greater than other orders in this file, as this is the fallback version - context=[{}], + context=[{'retrigger': 'true'}], schema={ 'type': 'object', 'properties': { @@ -84,6 +86,48 @@ def retrigger_decision_action(parameters, graph_config, input, task_group_id, ta } } ) +@register_callback_action( + title='Retrigger (disabled)', + name='retrigger', + cb_name='retrigger-disabled', + symbol='rt', + generic=True, + description=( + 'Create a clone of the task.\n\n' + 'This type of task should typically be re-run instead of re-triggered.' + ), + order=20, # must be greater than other orders in this file, as this is the fallback version + context=[{}], + schema={ + 'type': 'object', + 'properties': { + 'downstream': { + 'type': 'boolean', + 'description': ( + 'If true, downstream tasks from this one will be cloned as well. ' + 'The dependencies will be updated to work with the new task at the root.' + ), + 'default': False, + }, + 'times': { + 'type': 'integer', + 'default': 1, + 'minimum': 1, + 'maximum': 100, + 'title': 'Times', + 'description': 'How many times to run each task.', + }, + 'force': { + 'type': 'boolean', + 'default': False, + 'description': ( + 'This task should not be re-triggered. ' + 'This can be overridden by passing `true` here.' + ), + }, + } + } +) def retrigger_action(parameters, graph_config, input, task_group_id, task_id): decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( parameters, graph_config) @@ -94,6 +138,15 @@ def retrigger_action(parameters, graph_config, input, task_group_id, task_id): with_downstream = ' ' to_run = [label] + if not input.get('force', None) and not full_task_graph[label].attributes.get('retrigger'): + logger.info( + "Not retriggering task {}, task should not be retrigged " + "and force not specified.".format( + label + ) + ) + sys.exit(1) + if input.get('downstream'): to_run = full_task_graph.graph.transitive_closure(set(to_run), reverse=True).nodes to_run = to_run & set(label_to_taskid.keys()) diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py index fda325acd8bf..2b8cff41e76d 100644 --- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -1780,10 +1780,13 @@ def build_task(config, tasks): task['priority'] = get_default_priority(config.graph_config, config.params['project']) tags = task.get('tags', {}) + attributes = task.get('attributes', {}) + tags.update({ 'createdForUser': config.params['owner'], 'kind': config.kind, 'label': task['label'], + 'retrigger': 'true' if attributes.get('retrigger', False) else 'false' }) task_def = { @@ -1818,7 +1821,6 @@ def build_task(config, tasks): # add the payload and adjust anything else as required (e.g., scopes) payload_builders[task['worker']['implementation']](config, task, task_def) - attributes = task.get('attributes', {}) # Resolve run-on-projects build_platform = attributes.get('build_platform') resolve_keyed_by(task, 'run-on-projects', item_name=task['label'],