Bug 1465117 - Add additional options to backfilling action task r=dustin,jmaher

MozReview-Commit-ID: FMGjhQbg4im

--HG--
extra : rebase_source : a03e351836c7356c6912af85be9108921e50aabd
This commit is contained in:
Brian Stack 2018-06-01 16:44:16 -07:00
parent 53a10471cf
commit 3672e158f8
4 changed files with 52 additions and 23 deletions

View File

@ -272,7 +272,6 @@ class MachCommands(MachCommandBase):
help='Action callback name (Python function name)')
def test_action_callback(self, **options):
import taskgraph.parameters
from taskgraph.util.taskcluster import get_task_definition
import taskgraph.actions
import yaml
@ -288,12 +287,6 @@ class MachCommands(MachCommandBase):
try:
self.setup_logging()
task_id = options['task_id']
if options['task']:
task = load_data(options['task'])
elif task_id:
task = get_task_definition(task_id)
else:
task = None
if options['input']:
input = load_data(options['input'])
@ -308,7 +301,6 @@ class MachCommands(MachCommandBase):
return taskgraph.actions.trigger_action_callback(
task_group_id=options['task_group_id'],
task_id=task_id,
task=task,
input=input,
callback=options['callback'],
parameters=parameters,

View File

@ -12,7 +12,8 @@ import requests
from requests.exceptions import HTTPError
from .registry import register_callback_action
from .util import find_decision_task, create_tasks
from .util import find_decision_task, create_task_from_def, fix_task_dependencies
from slugid import nice as slugid
from taskgraph.util.taskcluster import get_artifact_from_index
from taskgraph.taskgraph import TaskGraph
@ -44,6 +45,24 @@ logger = logging.getLogger(__name__)
'title': 'Depth',
'description': ('The number of previous pushes before the current '
'push to attempt to trigger this task on.')
},
'inclusive': {
'type': 'boolean',
'default': False,
'title': 'Inclusive Range',
'description': ('If true, the backfill will also retrigger the task '
'on the selected push.')
},
'addGeckoProfile': {
'type': 'boolean',
'default': False,
'title': 'Add Gecko Profile',
'description': 'If true, appends --geckoProfile to mozharness options.'
},
'testPath': {
'type': 'string',
'title': 'Test Path',
'description': 'If specified, set MOZHARNESS_TEST_PATHS to this value.'
}
},
'additionalProperties': False
@ -53,8 +72,9 @@ logger = logging.getLogger(__name__)
def backfill_action(parameters, graph_config, input, task_group_id, task_id, task):
label = task['metadata']['name']
pushes = []
depth = input.get('depth', 5)
end_id = int(parameters['pushlog_id']) - 1
inclusive_tweak = 1 if input.get('inclusive') else 0
depth = input.get('depth', 5) + inclusive_tweak
end_id = int(parameters['pushlog_id']) - (1 - inclusive_tweak)
while True:
start_id = max(end_id - depth, 0)
@ -90,8 +110,21 @@ def backfill_action(parameters, graph_config, input, task_group_id, task_id, tas
continue
if label in full_task_graph.tasks.keys():
create_tasks(
[label], full_task_graph, label_to_taskid,
push_params, push_decision_task_id, push)
task_def = fix_task_dependencies(full_task_graph.tasks[label], label_to_taskid)
task_def['taskGroupId'] = push_decision_task_id
if input.get('addGeckoProfile'):
mh_options = task_def['payload'].setdefault('env', {}) \
.get('MOZHARNESS_OPTIONS', '')
task_def['payload']['env']['MOZHARNESS_OPTIONS'] = mh_options + ' --geckoProfile'
task_def['extra']['treeherder']['symbol'] += '-p'
if input.get('testPath'):
env = task_def['payload'].setdefault('env', {})
env['MOZHARNESS_TEST_PATHS'] = input.get('testPath')
task_def['extra']['treeherder']['symbol'] += '-b'
new_task_id = slugid()
create_task_from_def(new_task_id, task_def, parameters['level'])
else:
logging.info('Could not find {} on {}. Skipping.'.format(label, push))

View File

@ -11,9 +11,8 @@ import logging
from slugid import nice as slugid
from .util import (create_task_from_def, fetch_graph_and_labels)
from .util import (create_task_from_def, fetch_graph_and_labels, fix_task_dependencies)
from .registry import register_callback_action
from taskgraph.util.parameterization import resolve_task_references
TASKCLUSTER_QUEUE_URL = "https://queue.taskcluster.net/v1/task"
@ -84,13 +83,7 @@ def mochitest_retrigger_action(parameters, graph_config, input, task_group_id, t
parameters, graph_config)
pre_task = full_task_graph.tasks[task['metadata']['name']]
# fix up the task's dependencies, similar to how optimization would
# have done in the decision
dependencies = {name: label_to_taskid[label]
for name, label in pre_task.dependencies.iteritems()}
new_task_definition = resolve_task_references(pre_task.label, pre_task.task, dependencies)
new_task_definition.setdefault('dependencies', []).extend(dependencies.itervalues())
new_task_definition = fix_task_dependencies(pre_task, label_to_taskid)
# don't want to run mozharness tests, want a custom mach command instead
new_task_definition['payload']['command'] += ['--no-run-tests']

View File

@ -18,6 +18,7 @@ from taskgraph.decision import write_artifact
from taskgraph.taskgraph import TaskGraph
from taskgraph.optimize import optimize_task_graph
from taskgraph.util.taskcluster import get_session, find_task_id, get_artifact, list_tasks
from taskgraph.util.parameterization import resolve_task_references
logger = logging.getLogger(__name__)
@ -105,6 +106,16 @@ def create_task_from_def(task_id, task_def, level):
create.create_task(session, task_id, label, task_def)
def fix_task_dependencies(task_def, label_to_taskid):
"""fix up the task's dependencies, similar to how optimization would
have done in the decision"""
dependencies = {name: label_to_taskid[label]
for name, label in task_def.dependencies.iteritems()}
new_task_definition = resolve_task_references(task_def.label, task_def.task, dependencies)
new_task_definition.setdefault('dependencies', []).extend(dependencies.itervalues())
return new_task_definition
def update_parent(task, graph):
task.task.setdefault('extra', {})['parent'] = os.environ.get('TASK_ID', '')
return task