mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-15 14:25:52 +00:00
6db7f5787e
This uses the run_on_projects attribute introduced earlier for most branches, adjusts the `ash` method to handle that branch as the legacy implementation did, and updates try syntax to match builds as well as tests. In the process, this enables optimizing target tasks, meaning that tasks specifically requested in the try syntax might be optimized. While this is probably not ideal, it matches the existing behavior of try (where `-j all` is the default but all jobs are set to run only when certain files have been modified). This change can be reverted later, in a more advanced version of try. MozReview-Commit-ID: 5FYeUTAsafr --HG-- extra : rebase_source : b358e0e7cd8a401c50009e63dd55c59489c9b75b
159 lines
5.3 KiB
Python
159 lines
5.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
import os
|
|
import json
|
|
import logging
|
|
import yaml
|
|
|
|
from .generator import TaskGraphGenerator
|
|
from .create import create_tasks
|
|
from .parameters import Parameters
|
|
from .target_tasks import get_method
|
|
|
|
from taskgraph.util.templates import Templates
|
|
from taskgraph.util.time import (
|
|
json_time_from_now,
|
|
current_json_time,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
ARTIFACTS_DIR = 'artifacts'
|
|
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
|
|
|
|
# For each project, this gives a set of parameters specific to the project.
|
|
# See `taskcluster/docs/parameters.rst` for information on parameters.
|
|
PER_PROJECT_PARAMETERS = {
|
|
'try': {
|
|
'target_tasks_method': 'try_option_syntax',
|
|
# Always perform optimization. This makes it difficult to use try
|
|
# pushes to run a task that would otherwise be optimized, but is a
|
|
# compromise to avoid essentially disabling optimization in try.
|
|
'optimize_target_tasks': True,
|
|
},
|
|
|
|
'ash': {
|
|
'target_tasks_method': 'ash_tasks',
|
|
'optimize_target_tasks': True,
|
|
},
|
|
|
|
# the default parameters are used for projects that do not match above.
|
|
'default': {
|
|
'target_tasks_method': 'default',
|
|
'optimize_target_tasks': True,
|
|
}
|
|
}
|
|
|
|
|
|
def taskgraph_decision(options):
|
|
"""
|
|
Run the decision task. This function implements `mach taskgraph decision`,
|
|
and is responsible for
|
|
|
|
* processing decision task command-line options into parameters
|
|
* running task-graph generation exactly the same way the other `mach
|
|
taskgraph` commands do
|
|
* generating a set of artifacts to memorialize the graph
|
|
* calling TaskCluster APIs to create the graph
|
|
"""
|
|
|
|
parameters = get_decision_parameters(options)
|
|
|
|
# create a TaskGraphGenerator instance
|
|
target_tasks_method = parameters.get('target_tasks_method', 'all_tasks')
|
|
target_tasks_method = get_method(target_tasks_method)
|
|
tgg = TaskGraphGenerator(
|
|
root_dir=options['root'],
|
|
parameters=parameters,
|
|
target_tasks_method=target_tasks_method)
|
|
|
|
# write out the parameters used to generate this graph
|
|
write_artifact('parameters.yml', dict(**parameters))
|
|
|
|
# write out the yml file for action tasks
|
|
write_artifact('action.yml', get_action_yml(parameters))
|
|
|
|
# write out the full graph for reference
|
|
write_artifact('full-task-graph.json', tgg.full_task_graph.to_json())
|
|
|
|
# write out the target task set to allow reproducing this as input
|
|
write_artifact('target-tasks.json', tgg.target_task_set.tasks.keys())
|
|
|
|
# write out the optimized task graph to describe what will actually happen,
|
|
# and the map of labels to taskids
|
|
write_artifact('task-graph.json', tgg.optimized_task_graph.to_json())
|
|
write_artifact('label-to-taskid.json', tgg.label_to_taskid)
|
|
|
|
# actually create the graph
|
|
create_tasks(tgg.optimized_task_graph, tgg.label_to_taskid)
|
|
|
|
|
|
def get_decision_parameters(options):
|
|
"""
|
|
Load parameters from the command-line options for 'taskgraph decision'.
|
|
This also applies per-project parameters, based on the given project.
|
|
|
|
"""
|
|
parameters = {n: options[n] for n in [
|
|
'base_repository',
|
|
'head_repository',
|
|
'head_rev',
|
|
'head_ref',
|
|
'message',
|
|
'project',
|
|
'pushlog_id',
|
|
'pushdate',
|
|
'owner',
|
|
'level',
|
|
'triggered_by',
|
|
'target_tasks_method',
|
|
] if n in options}
|
|
|
|
project = parameters['project']
|
|
try:
|
|
parameters.update(PER_PROJECT_PARAMETERS[project])
|
|
except KeyError:
|
|
logger.warning("using default project parameters; add {} to "
|
|
"PER_PROJECT_PARAMETERS in {} to customize behavior "
|
|
"for this project".format(project, __file__))
|
|
parameters.update(PER_PROJECT_PARAMETERS['default'])
|
|
|
|
# `target_tasks_method` has higher precedence than `project` parameters
|
|
if options.get('target_tasks_method'):
|
|
parameters['target_tasks_method'] = options['target_tasks_method']
|
|
|
|
return Parameters(parameters)
|
|
|
|
|
|
def write_artifact(filename, data):
|
|
logger.info('writing artifact file `{}`'.format(filename))
|
|
if not os.path.isdir(ARTIFACTS_DIR):
|
|
os.mkdir(ARTIFACTS_DIR)
|
|
path = os.path.join(ARTIFACTS_DIR, filename)
|
|
if filename.endswith('.yml'):
|
|
with open(path, 'w') as f:
|
|
yaml.safe_dump(data, f, allow_unicode=True, default_flow_style=False)
|
|
elif filename.endswith('.json'):
|
|
with open(path, 'w') as f:
|
|
json.dump(data, f, sort_keys=True, indent=2, separators=(',', ': '))
|
|
else:
|
|
raise TypeError("Don't know how to write to {}".format(filename))
|
|
|
|
|
|
def get_action_yml(parameters):
|
|
templates = Templates(os.path.join(GECKO, "taskcluster/taskgraph"))
|
|
action_parameters = parameters.copy()
|
|
action_parameters.update({
|
|
"decision_task_id": "{{decision_task_id}}",
|
|
"task_labels": "{{task_labels}}",
|
|
"from_now": json_time_from_now,
|
|
"now": current_json_time()
|
|
})
|
|
return templates.load('action.yml', action_parameters)
|